cctally 1.91.0 → 1.92.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/README.md +4 -2
- package/bin/_cctally_cache.py +903 -74
- package/bin/_cctally_config.py +57 -0
- package/bin/_cctally_core.py +94 -14
- package/bin/_cctally_dashboard.py +217 -19
- package/bin/_cctally_dashboard_conversation.py +170 -20
- package/bin/_cctally_dashboard_envelope.py +2 -0
- package/bin/_cctally_db.py +481 -19
- package/bin/_cctally_doctor.py +18 -1
- package/bin/_cctally_journal.py +1156 -21
- package/bin/_cctally_journal_repair.py +6 -0
- package/bin/_cctally_parser.py +26 -0
- package/bin/_cctally_quota.py +171 -55
- package/bin/_cctally_record.py +13 -1
- package/bin/_cctally_rederive.py +4 -0
- package/bin/_cctally_statusline.py +6 -6
- package/bin/_cctally_store.py +1061 -40
- package/bin/_cctally_transcript.py +32 -2
- package/bin/_cctally_tui.py +54 -6
- package/bin/_lib_cache_report.py +8 -3
- package/bin/_lib_codex_conversation.py +851 -81
- package/bin/_lib_codex_conversation_query.py +2031 -96
- package/bin/_lib_codex_find_projection.py +517 -0
- package/bin/_lib_codex_harness_preamble.py +176 -0
- package/bin/_lib_codex_hooks.py +5 -3
- package/bin/_lib_codex_js_scan.py +254 -0
- package/bin/_lib_codex_landmarks.py +309 -0
- package/bin/_lib_codex_title_clean.py +116 -0
- package/bin/_lib_conversation_dispatch.py +168 -22
- package/bin/_lib_conversation_query.py +62 -2
- package/bin/_lib_conversation_watch.py +4 -2
- package/bin/_lib_doctor.py +64 -0
- package/bin/_lib_quota_alert_axes.py +31 -34
- package/bin/_lib_stats_damage.py +523 -0
- package/bin/_lib_stats_publish.py +243 -0
- package/bin/cctally +17 -3
- package/dashboard/static/assets/index-Dat-mza6.js +97 -0
- package/dashboard/static/assets/{index-Dwirao3Y.css → index-DnWdv8um.css} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +8 -1
- package/dashboard/static/assets/index-CILAoEja.js +0 -90
package/bin/_cctally_cache.py
CHANGED
|
@@ -127,6 +127,7 @@ def _cctally():
|
|
|
127
127
|
# for ``eprint`` is deleted.
|
|
128
128
|
import _cctally_core
|
|
129
129
|
from _cctally_core import eprint
|
|
130
|
+
from _lib_codex_find_projection import CODEX_FIND_PROJECTION_VERSION
|
|
130
131
|
from _lib_source_identity import source_root_key
|
|
131
132
|
# #416 spec §4.2: the pure tolerance-anchored reset kernel. `_lib_quota` imports
|
|
132
133
|
# only `_lib_accounts` (a stdlib leaf), so binding it here is circular-safe.
|
|
@@ -227,6 +228,12 @@ _CONV_INSERT_SQL = (
|
|
|
227
228
|
" search_tool,search_thinking)"
|
|
228
229
|
" VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
|
229
230
|
)
|
|
231
|
+
_CONV_ACCOUNT_INSERT_SQL = _CONV_INSERT_SQL.replace(
|
|
232
|
+
"search_tool,search_thinking)", "search_tool,search_thinking,account_key)"
|
|
233
|
+
).replace(
|
|
234
|
+
"?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
235
|
+
"?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
236
|
+
)
|
|
230
237
|
|
|
231
238
|
# #193: last non-null write wins (ai-title carries no timestamp; see spec S1). NO
|
|
232
239
|
# byte_offset guard — it can't order a cross-file resumed session. Ordering is
|
|
@@ -383,7 +390,7 @@ SESSION_ENTRY_UPSERT_SQL_REWALK = (
|
|
|
383
390
|
+ _SESSION_ENTRY_REWALK_PHYSICAL_GUARD)
|
|
384
391
|
|
|
385
392
|
|
|
386
|
-
def _conv_row_tuple(m, path_str):
|
|
393
|
+
def _conv_row_tuple(m, path_str, account_key=None, *, include_account=False):
|
|
387
394
|
"""Flatten a ``MessageRow`` into the ``_CONV_INSERT_SQL`` column order.
|
|
388
395
|
|
|
389
396
|
The #177 enrichment fields (stop_reason / attribution_skill /
|
|
@@ -393,7 +400,7 @@ def _conv_row_tuple(m, path_str):
|
|
|
393
400
|
one tuple. #217 S1 / U7a: the documented-dead ``search_aux`` column is gone
|
|
394
401
|
from the live schema (dropped by migration 016); the split
|
|
395
402
|
``search_tool``/``search_thinking`` columns carry the non-prose index."""
|
|
396
|
-
|
|
403
|
+
row = (
|
|
397
404
|
m.session_id, m.uuid, m.parent_uuid, path_str, m.byte_offset,
|
|
398
405
|
m.timestamp_utc, m.entry_type, m.text, m.blocks_json, m.model,
|
|
399
406
|
m.msg_id, m.req_id, m.cwd, m.git_branch, m.is_sidechain,
|
|
@@ -401,6 +408,7 @@ def _conv_row_tuple(m, path_str):
|
|
|
401
408
|
m.stop_reason, m.attribution_skill, m.attribution_plugin,
|
|
402
409
|
m.search_tool, m.search_thinking,
|
|
403
410
|
)
|
|
411
|
+
return row + ((account_key,) if include_account else ())
|
|
404
412
|
|
|
405
413
|
|
|
406
414
|
def _iter_sync_entries(
|
|
@@ -1300,15 +1308,24 @@ def _codex_provider_roots() -> list[CodexProviderRoot]:
|
|
|
1300
1308
|
roots: list[CodexProviderRoot] = []
|
|
1301
1309
|
seen: set[pathlib.Path] = set()
|
|
1302
1310
|
for configured in _cctally()._codex_home_roots():
|
|
1303
|
-
|
|
1311
|
+
try:
|
|
1312
|
+
provider_root = _canonical_codex_path(configured)
|
|
1313
|
+
except OSError:
|
|
1314
|
+
# An unreadable configured root is unknown filesystem state. Leave
|
|
1315
|
+
# it out of discovery; the prune scope still retains its configured
|
|
1316
|
+
# identity and will refuse destructive cleanup for that root.
|
|
1317
|
+
continue
|
|
1304
1318
|
if provider_root in seen:
|
|
1305
1319
|
continue
|
|
1306
1320
|
sessions = configured / "sessions"
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1321
|
+
try:
|
|
1322
|
+
if sessions.is_dir():
|
|
1323
|
+
walk_root = sessions
|
|
1324
|
+
elif configured.is_dir():
|
|
1325
|
+
walk_root = configured
|
|
1326
|
+
else:
|
|
1327
|
+
continue
|
|
1328
|
+
except OSError:
|
|
1312
1329
|
continue
|
|
1313
1330
|
seen.add(provider_root)
|
|
1314
1331
|
roots.append(CodexProviderRoot(
|
|
@@ -1438,6 +1455,214 @@ def _discover_codex_files_with_roots() -> list[CodexDiscoveredFile]:
|
|
|
1438
1455
|
return discovered
|
|
1439
1456
|
|
|
1440
1457
|
|
|
1458
|
+
CODEX_ORPHAN_PRUNE_REFUSED_KEY = "codex_orphan_prune_refused"
|
|
1459
|
+
_CODEX_ROOT_RECOGNITION_TYPES = frozenset({
|
|
1460
|
+
"session_meta", "turn_context", "event_msg", "response_item", "world_state",
|
|
1461
|
+
"compacted",
|
|
1462
|
+
})
|
|
1463
|
+
_CODEX_ROOT_RECOGNITION_MAX_FILES = 32
|
|
1464
|
+
_CODEX_ROOT_RECOGNITION_MAX_BYTES = 64 * 1024
|
|
1465
|
+
_CODEX_ROOT_RECOGNITION_MAX_LINES = 64
|
|
1466
|
+
|
|
1467
|
+
|
|
1468
|
+
def _codex_file_is_recognizable(path: pathlib.Path) -> bool:
|
|
1469
|
+
"""Boundedly prove that one JSONL contains a known Codex record envelope.
|
|
1470
|
+
|
|
1471
|
+
Unknown records remain ingestible for forward compatibility, but they are
|
|
1472
|
+
not strong enough evidence to authorize deletion of retained history.
|
|
1473
|
+
"""
|
|
1474
|
+
consumed = 0
|
|
1475
|
+
lines = 0
|
|
1476
|
+
try:
|
|
1477
|
+
with path.open("rb") as fh:
|
|
1478
|
+
while (
|
|
1479
|
+
lines < _CODEX_ROOT_RECOGNITION_MAX_LINES
|
|
1480
|
+
and consumed < _CODEX_ROOT_RECOGNITION_MAX_BYTES
|
|
1481
|
+
):
|
|
1482
|
+
remaining = _CODEX_ROOT_RECOGNITION_MAX_BYTES - consumed
|
|
1483
|
+
raw = fh.readline(remaining + 1)
|
|
1484
|
+
if not raw:
|
|
1485
|
+
break
|
|
1486
|
+
lines += 1
|
|
1487
|
+
consumed += len(raw)
|
|
1488
|
+
if len(raw) > remaining or not raw.endswith(b"\n"):
|
|
1489
|
+
break
|
|
1490
|
+
try:
|
|
1491
|
+
obj = json.loads(
|
|
1492
|
+
raw,
|
|
1493
|
+
parse_constant=_lib_jsonl._reject_nonfinite_json_constant,
|
|
1494
|
+
)
|
|
1495
|
+
except (UnicodeDecodeError, ValueError, TypeError):
|
|
1496
|
+
continue
|
|
1497
|
+
if (
|
|
1498
|
+
isinstance(obj, dict)
|
|
1499
|
+
and obj.get("type") in _CODEX_ROOT_RECOGNITION_TYPES
|
|
1500
|
+
and isinstance(obj.get("payload"), dict)
|
|
1501
|
+
):
|
|
1502
|
+
return True
|
|
1503
|
+
except OSError:
|
|
1504
|
+
return False
|
|
1505
|
+
return False
|
|
1506
|
+
|
|
1507
|
+
|
|
1508
|
+
@dataclass(frozen=True)
|
|
1509
|
+
class _CodexPruneScope:
|
|
1510
|
+
"""Filesystem evidence that may authorize one whole-tree orphan prune.
|
|
1511
|
+
|
|
1512
|
+
A configured root is only *recognized* after a bounded probe finds at least
|
|
1513
|
+
one known Codex record envelope under it. A missing, empty, or unrelated
|
|
1514
|
+
directory therefore remains protected. When a current root is recognized,
|
|
1515
|
+
roots no longer present in the configured set may still be removed — that
|
|
1516
|
+
preserves issue #108's deliberate A -> B cache scoping. Within a recognized
|
|
1517
|
+
root, a missing file is deletion evidence only while its parent directory is
|
|
1518
|
+
still readable; losing an entire mounted subtree fails closed.
|
|
1519
|
+
"""
|
|
1520
|
+
|
|
1521
|
+
configured_root_keys: frozenset[str]
|
|
1522
|
+
recognized_root_keys: frozenset[str]
|
|
1523
|
+
|
|
1524
|
+
def refusal_reason(self, source_path: str, root_key: str | None) -> str | None:
|
|
1525
|
+
if not self.recognized_root_keys:
|
|
1526
|
+
return "no_recognizable_roots"
|
|
1527
|
+
if (
|
|
1528
|
+
root_key in self.configured_root_keys
|
|
1529
|
+
and root_key not in self.recognized_root_keys
|
|
1530
|
+
):
|
|
1531
|
+
return "configured_root_unrecognizable"
|
|
1532
|
+
if root_key in self.recognized_root_keys:
|
|
1533
|
+
parent = pathlib.Path(source_path).parent
|
|
1534
|
+
try:
|
|
1535
|
+
if not parent.is_dir() or not os.access(parent, os.R_OK | os.X_OK):
|
|
1536
|
+
return "source_parent_unavailable"
|
|
1537
|
+
except OSError:
|
|
1538
|
+
return "source_parent_unavailable"
|
|
1539
|
+
return None
|
|
1540
|
+
|
|
1541
|
+
def root_refusal_reason(self, root_key: str) -> str | None:
|
|
1542
|
+
if not self.recognized_root_keys:
|
|
1543
|
+
return "no_recognizable_roots"
|
|
1544
|
+
if (
|
|
1545
|
+
root_key in self.configured_root_keys
|
|
1546
|
+
and root_key not in self.recognized_root_keys
|
|
1547
|
+
):
|
|
1548
|
+
return "configured_root_unrecognizable"
|
|
1549
|
+
return None
|
|
1550
|
+
|
|
1551
|
+
|
|
1552
|
+
def _codex_prune_scope(files: list[CodexDiscoveredFile]) -> _CodexPruneScope:
|
|
1553
|
+
configured: set[str] = set()
|
|
1554
|
+
for raw_root in _cctally()._codex_home_roots():
|
|
1555
|
+
try:
|
|
1556
|
+
provider_root = _canonical_codex_path(raw_root)
|
|
1557
|
+
configured.add(source_root_key(str(provider_root)))
|
|
1558
|
+
except (OSError, TypeError, ValueError):
|
|
1559
|
+
continue
|
|
1560
|
+
recognized: set[str] = set()
|
|
1561
|
+
checked_per_root: dict[str, int] = {}
|
|
1562
|
+
# Discovery is stable oldest-first for Codex's dated rollout paths. Probe
|
|
1563
|
+
# from the tail so a bounded check sees the current envelope generation,
|
|
1564
|
+
# rather than spending its entire budget on a large legacy prefix. A root
|
|
1565
|
+
# containing only legacy/unknown envelopes still fails closed.
|
|
1566
|
+
for item in reversed(files):
|
|
1567
|
+
if item.source_root_key in recognized:
|
|
1568
|
+
continue
|
|
1569
|
+
checked = checked_per_root.get(item.source_root_key, 0)
|
|
1570
|
+
if checked >= _CODEX_ROOT_RECOGNITION_MAX_FILES:
|
|
1571
|
+
continue
|
|
1572
|
+
checked_per_root[item.source_root_key] = checked + 1
|
|
1573
|
+
if _codex_file_is_recognizable(item.source_path):
|
|
1574
|
+
recognized.add(item.source_root_key)
|
|
1575
|
+
return _CodexPruneScope(
|
|
1576
|
+
configured_root_keys=frozenset(configured),
|
|
1577
|
+
recognized_root_keys=frozenset(recognized),
|
|
1578
|
+
)
|
|
1579
|
+
|
|
1580
|
+
|
|
1581
|
+
def _partition_codex_prune_candidates(
|
|
1582
|
+
scope: _CodexPruneScope,
|
|
1583
|
+
sources: list[tuple[str, str | None]],
|
|
1584
|
+
root_keys: set[str],
|
|
1585
|
+
) -> tuple[
|
|
1586
|
+
list[tuple[str, str | None]], list[tuple[str, str | None]], set[str], set[str]
|
|
1587
|
+
]:
|
|
1588
|
+
safe_sources: list[tuple[str, str | None]] = []
|
|
1589
|
+
refused_sources: list[tuple[str, str | None]] = []
|
|
1590
|
+
for source in sources:
|
|
1591
|
+
target = refused_sources if scope.refusal_reason(*source) else safe_sources
|
|
1592
|
+
target.append(source)
|
|
1593
|
+
safe_roots: set[str] = set()
|
|
1594
|
+
refused_roots: set[str] = set()
|
|
1595
|
+
for root_key in root_keys:
|
|
1596
|
+
target = refused_roots if scope.root_refusal_reason(root_key) else safe_roots
|
|
1597
|
+
target.add(root_key)
|
|
1598
|
+
return safe_sources, refused_sources, safe_roots, refused_roots
|
|
1599
|
+
|
|
1600
|
+
|
|
1601
|
+
def _codex_prune_refusal_record(
|
|
1602
|
+
conn: sqlite3.Connection,
|
|
1603
|
+
*,
|
|
1604
|
+
store: str,
|
|
1605
|
+
scope: _CodexPruneScope,
|
|
1606
|
+
refused_sources: list[tuple[str, str | None]],
|
|
1607
|
+
refused_root_keys: set[str],
|
|
1608
|
+
) -> None:
|
|
1609
|
+
"""Persist a privacy-safe refusal signal in the store that declined."""
|
|
1610
|
+
now = (
|
|
1611
|
+
_cctally_core._command_as_of()
|
|
1612
|
+
.astimezone(dt.timezone.utc)
|
|
1613
|
+
.isoformat(timespec="seconds")
|
|
1614
|
+
.replace("+00:00", "Z")
|
|
1615
|
+
)
|
|
1616
|
+
since = now
|
|
1617
|
+
row = conn.execute(
|
|
1618
|
+
"SELECT value FROM cache_meta WHERE key=?",
|
|
1619
|
+
(CODEX_ORPHAN_PRUNE_REFUSED_KEY,),
|
|
1620
|
+
).fetchone()
|
|
1621
|
+
if row and row[0]:
|
|
1622
|
+
try:
|
|
1623
|
+
previous = json.loads(row[0])
|
|
1624
|
+
if isinstance(previous, dict) and isinstance(previous.get("since"), str):
|
|
1625
|
+
since = previous["since"]
|
|
1626
|
+
except (TypeError, ValueError):
|
|
1627
|
+
pass
|
|
1628
|
+
reasons = sorted({
|
|
1629
|
+
reason
|
|
1630
|
+
for path, root_key in refused_sources
|
|
1631
|
+
if (reason := scope.refusal_reason(path, root_key)) is not None
|
|
1632
|
+
} | {
|
|
1633
|
+
reason
|
|
1634
|
+
for root_key in refused_root_keys
|
|
1635
|
+
if (reason := scope.root_refusal_reason(root_key)) is not None
|
|
1636
|
+
})
|
|
1637
|
+
record = {
|
|
1638
|
+
"schemaVersion": 1,
|
|
1639
|
+
"store": store,
|
|
1640
|
+
"since": since,
|
|
1641
|
+
"at": now,
|
|
1642
|
+
"reasons": reasons,
|
|
1643
|
+
"configuredRootCount": len(scope.configured_root_keys),
|
|
1644
|
+
"recognizedRootCount": len(scope.recognized_root_keys),
|
|
1645
|
+
"preservedFileCount": len({path for path, _root in refused_sources}),
|
|
1646
|
+
"preservedRootCount": len(refused_root_keys),
|
|
1647
|
+
}
|
|
1648
|
+
conn.execute(
|
|
1649
|
+
"INSERT INTO cache_meta(key,value) VALUES(?,?) "
|
|
1650
|
+
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
1651
|
+
(CODEX_ORPHAN_PRUNE_REFUSED_KEY, json.dumps(record, sort_keys=True)),
|
|
1652
|
+
)
|
|
1653
|
+
eprint(
|
|
1654
|
+
f"[codex-{store}] orphan prune refused: preserved "
|
|
1655
|
+
f"{record['preservedFileCount']} tracked file(s); "
|
|
1656
|
+
"configured Codex roots were not sufficient deletion evidence"
|
|
1657
|
+
)
|
|
1658
|
+
|
|
1659
|
+
|
|
1660
|
+
def _clear_codex_prune_refusal(conn: sqlite3.Connection) -> None:
|
|
1661
|
+
conn.execute(
|
|
1662
|
+
"DELETE FROM cache_meta WHERE key=?", (CODEX_ORPHAN_PRUNE_REFUSED_KEY,)
|
|
1663
|
+
)
|
|
1664
|
+
|
|
1665
|
+
|
|
1441
1666
|
def _qualify_codex_targets(only_paths: "set[str]") -> list[CodexDiscoveredFile]:
|
|
1442
1667
|
"""Resolve each requested path through the ordered configured roots exactly
|
|
1443
1668
|
as full discovery would (spec §5.1) — producing the same per-file facts a
|
|
@@ -1603,7 +1828,12 @@ def _clear_codex_derived_rows(conn: sqlite3.Connection) -> bool:
|
|
|
1603
1828
|
|
|
1604
1829
|
|
|
1605
1830
|
def _bump_codex_physical_mutation_seq(conn: sqlite3.Connection) -> None:
|
|
1606
|
-
"""Advance the
|
|
1831
|
+
"""Advance the shared Codex physical-state invalidation token in this txn.
|
|
1832
|
+
|
|
1833
|
+
Dashboard/TUI snapshot versions and the quota projection certificate both
|
|
1834
|
+
consume this value. Every writer that changes their cache.db inputs must
|
|
1835
|
+
advance it in the same transaction as that change.
|
|
1836
|
+
"""
|
|
1607
1837
|
conn.execute(
|
|
1608
1838
|
"INSERT INTO cache_meta(key, value) VALUES "
|
|
1609
1839
|
"('codex_physical_mutation_seq', '1') "
|
|
@@ -1622,6 +1852,10 @@ _CODEX_MSG_INSERT_SQL = (
|
|
|
1622
1852
|
"INSERT OR IGNORE INTO codex_conversation_messages (" + _CODEX_NORM_COLS + ") "
|
|
1623
1853
|
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
|
1624
1854
|
)
|
|
1855
|
+
_CODEX_ACCOUNT_MSG_INSERT_SQL = (
|
|
1856
|
+
"INSERT OR IGNORE INTO codex_conversation_messages (" + _CODEX_NORM_COLS
|
|
1857
|
+
+ ", account_key) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
|
1858
|
+
)
|
|
1625
1859
|
|
|
1626
1860
|
|
|
1627
1861
|
def _codex_conversation_project_attribution(
|
|
@@ -1679,17 +1913,29 @@ def _load_codex_normalized_rows(
|
|
|
1679
1913
|
]
|
|
1680
1914
|
|
|
1681
1915
|
|
|
1682
|
-
def _insert_codex_normalized_rows(
|
|
1916
|
+
def _insert_codex_normalized_rows(
|
|
1917
|
+
conn: sqlite3.Connection,
|
|
1918
|
+
rows: list,
|
|
1919
|
+
touches: list,
|
|
1920
|
+
account_by_physical: "dict[tuple[str, int], str | None] | None" = None,
|
|
1921
|
+
) -> None:
|
|
1683
1922
|
"""Insert normalized rows (INSERT OR IGNORE on the physical key) + their file
|
|
1684
1923
|
touches (message linkage resolved via (source_path, line_offset))."""
|
|
1685
1924
|
if rows:
|
|
1686
|
-
|
|
1925
|
+
with_accounts = account_by_physical is not None
|
|
1926
|
+
account_by_physical = account_by_physical or {}
|
|
1927
|
+
conn.executemany(
|
|
1928
|
+
_CODEX_ACCOUNT_MSG_INSERT_SQL if with_accounts else _CODEX_MSG_INSERT_SQL,
|
|
1929
|
+
[
|
|
1687
1930
|
(r.conversation_key, r.source_root_key, r.source_path, r.line_offset,
|
|
1688
1931
|
r.timestamp_utc, r.turn_id, r.call_id, r.kind, r.event_type,
|
|
1689
1932
|
r.record_family, r.model, r.text, r.content_digest, r.content_len,
|
|
1690
1933
|
r.detail_json, r.search_tool, r.search_thinking)
|
|
1934
|
+
+ ((account_by_physical.get((r.source_path, r.line_offset)),)
|
|
1935
|
+
if with_accounts else ())
|
|
1691
1936
|
for r in rows
|
|
1692
|
-
|
|
1937
|
+
],
|
|
1938
|
+
)
|
|
1693
1939
|
for touch in touches:
|
|
1694
1940
|
conn.execute(
|
|
1695
1941
|
"INSERT OR IGNORE INTO codex_conversation_file_touches "
|
|
@@ -1764,26 +2010,125 @@ def _replay_codex_normalization(conn: sqlite3.Connection) -> None:
|
|
|
1764
2010
|
the plain rowid alias make a re-run byte-idempotent."""
|
|
1765
2011
|
kern = _lib_codex_conversation
|
|
1766
2012
|
events_by_file: dict[str, list] = {}
|
|
2013
|
+
accounts_by_file: dict[str, dict[tuple[str, int], str | None]] = {}
|
|
1767
2014
|
order: list[str] = []
|
|
2015
|
+
has_account = "account_key" in {
|
|
2016
|
+
str(info[1])
|
|
2017
|
+
for info in conn.execute("PRAGMA table_info(codex_conversation_events)")
|
|
2018
|
+
}
|
|
2019
|
+
account_select = ", account_key" if has_account else ""
|
|
1768
2020
|
for row in conn.execute(
|
|
1769
2021
|
"SELECT source_path, line_offset, source_root_key, conversation_key, "
|
|
1770
2022
|
"native_thread_id, root_thread_id, parent_thread_id, timestamp_utc, "
|
|
1771
2023
|
"record_type, event_type, turn_id, call_id, payload_json "
|
|
2024
|
+
+ account_select + " "
|
|
1772
2025
|
"FROM codex_conversation_events "
|
|
1773
2026
|
"ORDER BY source_path ASC, line_offset ASC"
|
|
1774
2027
|
):
|
|
1775
|
-
event = _lib_jsonl.CodexPhysicalEvent(*row)
|
|
2028
|
+
event = _lib_jsonl.CodexPhysicalEvent(*(row[:-1] if has_account else row))
|
|
1776
2029
|
if event.source_path not in events_by_file:
|
|
1777
2030
|
events_by_file[event.source_path] = []
|
|
2031
|
+
accounts_by_file[event.source_path] = {}
|
|
1778
2032
|
order.append(event.source_path)
|
|
1779
2033
|
events_by_file[event.source_path].append(event)
|
|
2034
|
+
if has_account:
|
|
2035
|
+
accounts_by_file[event.source_path][
|
|
2036
|
+
(event.source_path, int(event.line_offset))
|
|
2037
|
+
] = row[-1]
|
|
1780
2038
|
affected: set = set()
|
|
1781
2039
|
for source_path in order:
|
|
1782
2040
|
result = kern.normalize_codex_events(
|
|
1783
2041
|
events_by_file[source_path], initial=kern.CodexStickyState())
|
|
1784
|
-
_insert_codex_normalized_rows(
|
|
2042
|
+
_insert_codex_normalized_rows(
|
|
2043
|
+
conn, result.rows, result.touches,
|
|
2044
|
+
accounts_by_file[source_path] if has_account else None,
|
|
2045
|
+
)
|
|
1785
2046
|
affected.update(r.conversation_key for r in result.rows)
|
|
1786
2047
|
_recompute_codex_rollups(conn, affected)
|
|
2048
|
+
# Migration 025 still replays this helper against legacy cache.db fixtures,
|
|
2049
|
+
# where the conversations-only #482 projection table does not exist. Live
|
|
2050
|
+
# conversations.db replays must refresh the projection, but the historical
|
|
2051
|
+
# cache migration must remain byte-stable and cannot create a newer store's
|
|
2052
|
+
# derivation out of order.
|
|
2053
|
+
projection_exists = conn.execute(
|
|
2054
|
+
"SELECT 1 FROM sqlite_master "
|
|
2055
|
+
"WHERE type='table' AND name='codex_find_projection'"
|
|
2056
|
+
).fetchone() is not None
|
|
2057
|
+
if projection_exists:
|
|
2058
|
+
import _lib_codex_conversation_query as query
|
|
2059
|
+
query.materialize_codex_find_projection(conn, affected)
|
|
2060
|
+
|
|
2061
|
+
|
|
2062
|
+
def run_codex_find_projection_backfill(
|
|
2063
|
+
conn: sqlite3.Connection,
|
|
2064
|
+
*,
|
|
2065
|
+
batch_size: int = 400,
|
|
2066
|
+
) -> dict[str, object]:
|
|
2067
|
+
"""Consume one resumable #482 projection batch from retained rows.
|
|
2068
|
+
|
|
2069
|
+
The caller owns the Codex conversations provider flock. Progress is keyed
|
|
2070
|
+
by the normalized message rowid, but a selected conversation is always
|
|
2071
|
+
rebuilt as a whole so native folds never straddle a batch boundary.
|
|
2072
|
+
"""
|
|
2073
|
+
pending = conn.execute(
|
|
2074
|
+
"SELECT 1 FROM cache_meta "
|
|
2075
|
+
"WHERE key='codex_find_projection_backfill_pending'"
|
|
2076
|
+
).fetchone()
|
|
2077
|
+
if pending is None:
|
|
2078
|
+
complete = conn.execute(
|
|
2079
|
+
"SELECT 1 FROM cache_meta "
|
|
2080
|
+
"WHERE key='codex_find_projection_complete_version' AND value=?",
|
|
2081
|
+
(str(CODEX_FIND_PROJECTION_VERSION),),
|
|
2082
|
+
).fetchone() is not None
|
|
2083
|
+
return {"processed": 0, "complete": complete}
|
|
2084
|
+
row = conn.execute(
|
|
2085
|
+
"SELECT value FROM cache_meta "
|
|
2086
|
+
"WHERE key='codex_find_projection_backfill_cursor'"
|
|
2087
|
+
).fetchone()
|
|
2088
|
+
try:
|
|
2089
|
+
cursor = int(row[0]) if row is not None else 0
|
|
2090
|
+
except (TypeError, ValueError):
|
|
2091
|
+
cursor = 0
|
|
2092
|
+
selected_rows = conn.execute(
|
|
2093
|
+
"SELECT id,conversation_key FROM codex_conversation_messages "
|
|
2094
|
+
"WHERE id>? ORDER BY id LIMIT ?",
|
|
2095
|
+
(cursor, max(1, batch_size)),
|
|
2096
|
+
).fetchall()
|
|
2097
|
+
selected = {conversation_key for _message_id, conversation_key in selected_rows}
|
|
2098
|
+
if selected_rows:
|
|
2099
|
+
import _lib_codex_conversation_query as query
|
|
2100
|
+
query.materialize_codex_find_projection(conn, selected)
|
|
2101
|
+
# Advance only over raw rows actually consumed by this batch. A
|
|
2102
|
+
# conversation may own later, interleaved row ids; jumping to its MAX
|
|
2103
|
+
# would skip another conversation whose first row lies in between and
|
|
2104
|
+
# could falsely certify the projection complete.
|
|
2105
|
+
cursor = int(selected_rows[-1][0])
|
|
2106
|
+
conn.execute(
|
|
2107
|
+
"INSERT OR REPLACE INTO cache_meta(key,value) VALUES"
|
|
2108
|
+
"('codex_find_projection_backfill_cursor',?)",
|
|
2109
|
+
(str(cursor),),
|
|
2110
|
+
)
|
|
2111
|
+
remaining = conn.execute(
|
|
2112
|
+
"SELECT 1 FROM codex_conversation_messages WHERE id>? LIMIT 1",
|
|
2113
|
+
(cursor,),
|
|
2114
|
+
).fetchone()
|
|
2115
|
+
complete = remaining is None
|
|
2116
|
+
if complete:
|
|
2117
|
+
conn.execute(
|
|
2118
|
+
"INSERT OR REPLACE INTO cache_meta(key,value) VALUES(?,?)",
|
|
2119
|
+
(
|
|
2120
|
+
"codex_find_projection_complete_version",
|
|
2121
|
+
str(CODEX_FIND_PROJECTION_VERSION),
|
|
2122
|
+
),
|
|
2123
|
+
)
|
|
2124
|
+
conn.execute(
|
|
2125
|
+
"DELETE FROM cache_meta WHERE key IN "
|
|
2126
|
+
"('codex_find_projection_backfill_pending',"
|
|
2127
|
+
" 'codex_find_projection_backfill_cursor',"
|
|
2128
|
+
" 'codex_find_projection_backfill_version')"
|
|
2129
|
+
)
|
|
2130
|
+
conn.commit()
|
|
2131
|
+
return {"processed": len(selected), "complete": complete}
|
|
1787
2132
|
|
|
1788
2133
|
|
|
1789
2134
|
def _repair_codex_turn_ids_for_source(
|
|
@@ -1832,27 +2177,18 @@ def _repair_codex_turn_ids_for_source(
|
|
|
1832
2177
|
return affected
|
|
1833
2178
|
|
|
1834
2179
|
|
|
1835
|
-
def
|
|
2180
|
+
def _collect_retained_codex_paths_and_roots(
|
|
1836
2181
|
conn: sqlite3.Connection,
|
|
1837
|
-
current_file_identities: set[tuple[str, str]],
|
|
1838
|
-
active_root_keys: set[str],
|
|
1839
2182
|
) -> tuple[list[tuple[str, str | None]], set[str]]:
|
|
1840
|
-
"""Return
|
|
2183
|
+
"""Return every retained real source identity and provider root.
|
|
1841
2184
|
|
|
1842
2185
|
A failed/partial prior write can leave any S1 child family without its
|
|
1843
|
-
terminal ``codex_session_files`` row.
|
|
1844
|
-
every physical family,
|
|
1845
|
-
|
|
2186
|
+
terminal ``codex_session_files`` row. Safety decisions must therefore see
|
|
2187
|
+
every physical family, including a still-discovered path whose root no
|
|
2188
|
+
longer contains recognizable Codex data. Relative fixture rows have no
|
|
2189
|
+
on-disk authority and remain outside filesystem pruning.
|
|
1846
2190
|
"""
|
|
1847
|
-
|
|
1848
|
-
stale_root_keys: set[str] = set()
|
|
1849
|
-
current_paths = {path for path, _root_key in current_file_identities}
|
|
1850
|
-
terminal_file_identities = {
|
|
1851
|
-
(path, root_key)
|
|
1852
|
-
for path, root_key in conn.execute(
|
|
1853
|
-
"SELECT path, source_root_key FROM codex_session_files"
|
|
1854
|
-
)
|
|
1855
|
-
}
|
|
2191
|
+
retained_identities: set[tuple[str, str | None]] = set()
|
|
1856
2192
|
family_queries = (
|
|
1857
2193
|
"SELECT path, source_root_key FROM codex_session_files",
|
|
1858
2194
|
"SELECT source_path, source_root_key FROM codex_session_entries",
|
|
@@ -1862,29 +2198,59 @@ def _collect_inactive_codex_paths_and_roots(
|
|
|
1862
2198
|
)
|
|
1863
2199
|
for query in family_queries:
|
|
1864
2200
|
for source_path, root_key in conn.execute(query):
|
|
1865
|
-
|
|
1866
|
-
if (
|
|
1867
|
-
not os.path.isabs(source_path)
|
|
1868
|
-
or identity in current_file_identities
|
|
1869
|
-
# An old terminal file at a currently discovered path must
|
|
1870
|
-
# reach the normal requalification loop, which resets every
|
|
1871
|
-
# family as one file transaction and records the reset stat.
|
|
1872
|
-
or (
|
|
1873
|
-
source_path in current_paths
|
|
1874
|
-
and identity in terminal_file_identities
|
|
1875
|
-
)
|
|
1876
|
-
):
|
|
2201
|
+
if not os.path.isabs(source_path):
|
|
1877
2202
|
continue
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
2203
|
+
retained_identities.add((source_path, root_key))
|
|
2204
|
+
retained_root_keys = {
|
|
2205
|
+
root_key for _path, root_key in retained_identities if root_key is not None
|
|
2206
|
+
}
|
|
2207
|
+
retained_root_keys.update(
|
|
1882
2208
|
root_key
|
|
1883
2209
|
for (root_key,) in conn.execute(
|
|
1884
2210
|
"SELECT source_root_key FROM codex_source_roots"
|
|
1885
2211
|
)
|
|
1886
|
-
if root_key not in active_root_keys
|
|
1887
2212
|
)
|
|
2213
|
+
return (
|
|
2214
|
+
sorted(retained_identities, key=lambda item: (item[0], item[1] or "")),
|
|
2215
|
+
retained_root_keys,
|
|
2216
|
+
)
|
|
2217
|
+
|
|
2218
|
+
|
|
2219
|
+
def _collect_inactive_codex_paths_and_roots(
|
|
2220
|
+
conn: sqlite3.Connection,
|
|
2221
|
+
current_file_identities: set[tuple[str, str]],
|
|
2222
|
+
active_root_keys: set[str],
|
|
2223
|
+
) -> tuple[list[tuple[str, str | None]], set[str]]:
|
|
2224
|
+
"""Return stale retained source identities and candidate root keys."""
|
|
2225
|
+
retained_identities, retained_root_keys = (
|
|
2226
|
+
_collect_retained_codex_paths_and_roots(conn)
|
|
2227
|
+
)
|
|
2228
|
+
current_paths = {path for path, _root_key in current_file_identities}
|
|
2229
|
+
terminal_file_identities = {
|
|
2230
|
+
(path, root_key)
|
|
2231
|
+
for path, root_key in conn.execute(
|
|
2232
|
+
"SELECT path, source_root_key FROM codex_session_files"
|
|
2233
|
+
)
|
|
2234
|
+
}
|
|
2235
|
+
stale_identities = {
|
|
2236
|
+
identity
|
|
2237
|
+
for identity in retained_identities
|
|
2238
|
+
if (
|
|
2239
|
+
identity not in current_file_identities
|
|
2240
|
+
# An old terminal file at a currently discovered path must reach
|
|
2241
|
+
# the normal requalification loop. The caller separately protects
|
|
2242
|
+
# that loop when the configured root is unrecognizable.
|
|
2243
|
+
and not (
|
|
2244
|
+
identity[0] in current_paths
|
|
2245
|
+
and identity in terminal_file_identities
|
|
2246
|
+
)
|
|
2247
|
+
)
|
|
2248
|
+
}
|
|
2249
|
+
stale_root_keys = {
|
|
2250
|
+
root_key
|
|
2251
|
+
for root_key in retained_root_keys
|
|
2252
|
+
if root_key not in active_root_keys
|
|
2253
|
+
}
|
|
1888
2254
|
return sorted(stale_identities, key=lambda item: (item[0], item[1] or "")), stale_root_keys
|
|
1889
2255
|
|
|
1890
2256
|
|
|
@@ -3793,7 +4159,9 @@ def _reingest_parse_file(jp, path_str):
|
|
|
3793
4159
|
return rows
|
|
3794
4160
|
|
|
3795
4161
|
|
|
3796
|
-
def _resumable_reingest_conversation_messages(
|
|
4162
|
+
def _resumable_reingest_conversation_messages(
|
|
4163
|
+
conn, active_account_key: "str | None" = None,
|
|
4164
|
+
):
|
|
3797
4165
|
"""#179: resumable, lock-friendly replacement for the old global
|
|
3798
4166
|
``clear_conversation_messages`` + offset-0 ``backfill_conversation_messages``
|
|
3799
4167
|
reingest, which re-armed the whole ~2.5min rebuild on any interrupt. Walks
|
|
@@ -3841,6 +4209,30 @@ def _resumable_reingest_conversation_messages(conn):
|
|
|
3841
4209
|
conn.commit()
|
|
3842
4210
|
continue
|
|
3843
4211
|
try:
|
|
4212
|
+
# #347 enrichment replays must not erase first-stamped transcript
|
|
4213
|
+
# ownership. Snapshot by physical offset before the destructive
|
|
4214
|
+
# per-source replace and carry it onto the re-parsed row.
|
|
4215
|
+
account_capable = "account_key" in {
|
|
4216
|
+
str(info[1])
|
|
4217
|
+
for info in conn.execute("PRAGMA table_info(conversation_messages)")
|
|
4218
|
+
}
|
|
4219
|
+
account_by_offset = {
|
|
4220
|
+
int(offset): account_key
|
|
4221
|
+
for offset, account_key in conn.execute(
|
|
4222
|
+
"SELECT byte_offset,account_key FROM conversation_messages "
|
|
4223
|
+
"WHERE source_path=?",
|
|
4224
|
+
(path_str,),
|
|
4225
|
+
)
|
|
4226
|
+
} if account_capable else {}
|
|
4227
|
+
if account_capable:
|
|
4228
|
+
rows = [
|
|
4229
|
+
row + (
|
|
4230
|
+
account_by_offset[int(row[4])]
|
|
4231
|
+
if int(row[4]) in account_by_offset
|
|
4232
|
+
else active_account_key,
|
|
4233
|
+
)
|
|
4234
|
+
for row in rows
|
|
4235
|
+
]
|
|
3844
4236
|
# #217 S2 / I-3 (P1-4): conversation_file_touches is derived state
|
|
3845
4237
|
# keyed by conversation_messages.id, and this per-source reingest
|
|
3846
4238
|
# DELETEs + re-inserts the file's message rows (bumping autoincrement
|
|
@@ -3855,7 +4247,11 @@ def _resumable_reingest_conversation_messages(conn):
|
|
|
3855
4247
|
conn.execute("DELETE FROM conversation_messages WHERE source_path=?",
|
|
3856
4248
|
(path_str,))
|
|
3857
4249
|
if rows:
|
|
3858
|
-
conn.executemany(
|
|
4250
|
+
conn.executemany(
|
|
4251
|
+
_CONV_ACCOUNT_INSERT_SQL if account_capable
|
|
4252
|
+
else _CONV_INSERT_SQL,
|
|
4253
|
+
rows,
|
|
4254
|
+
)
|
|
3859
4255
|
# Refill scoped to this file's just-reinserted physical keys
|
|
3860
4256
|
# (col 3=source_path, col 4=byte_offset per _conv_row_tuple).
|
|
3861
4257
|
_fill_file_touches(
|
|
@@ -4860,6 +5256,12 @@ class CodexIngestStats:
|
|
|
4860
5256
|
# Count of cached files dropped because they fall outside the CURRENT
|
|
4861
5257
|
# $CODEX_HOME root set (issue #108 — a prior-root purge, not a delta).
|
|
4862
5258
|
files_pruned: int = 0
|
|
5259
|
+
# #485: a whole-tree walk may discover no trustworthy current Codex root.
|
|
5260
|
+
# In that state absence is not deletion evidence, so ordinary orphan
|
|
5261
|
+
# pruning is refused and retained rows stay untouched. These fields make
|
|
5262
|
+
# the destructive decision directly observable to callers and tests.
|
|
5263
|
+
prune_refused: bool = False
|
|
5264
|
+
prune_refused_files: int = 0
|
|
4863
5265
|
lock_contended: bool = False
|
|
4864
5266
|
# #279 S2 F1 parse-health counters — folded from each file's
|
|
4865
5267
|
# _CodexIterState after its drain. Same vocabulary as the iterator
|
|
@@ -5456,6 +5858,7 @@ def sync_codex_cache(
|
|
|
5456
5858
|
_cache_root_keys,
|
|
5457
5859
|
)
|
|
5458
5860
|
seq_before = codex_physical_mutation_seq(conn)
|
|
5861
|
+
rebuild_prune_scope: _CodexPruneScope | None = None
|
|
5459
5862
|
|
|
5460
5863
|
# #416 spec D1: which rollouts were ALREADY ingested before this rebuild
|
|
5461
5864
|
# cleared the cursor table. A rebuild re-reads their bytes, and bytes
|
|
@@ -5485,6 +5888,46 @@ def sync_codex_cache(
|
|
|
5485
5888
|
str(_canonical_codex_path(pathlib.Path(str(_path))))))
|
|
5486
5889
|
except (ValueError, TypeError, OSError):
|
|
5487
5890
|
continue
|
|
5891
|
+
if rebuild and not targeted:
|
|
5892
|
+
# #485: rebuild used to clear first and discover second. With an
|
|
5893
|
+
# empty/missing/misconfigured root that made the explicit repair
|
|
5894
|
+
# command itself destroy the only retained copy. Discover before
|
|
5895
|
+
# the clear and require the same positive root evidence as ordinary
|
|
5896
|
+
# pruning whenever real absolute Codex rows already exist.
|
|
5897
|
+
preflight_files = _discover_codex_files_with_roots()
|
|
5898
|
+
preflight_scope = _codex_prune_scope(preflight_files)
|
|
5899
|
+
rebuild_prune_scope = preflight_scope
|
|
5900
|
+
preflight_sources, _preflight_root_keys = (
|
|
5901
|
+
_collect_retained_codex_paths_and_roots(conn)
|
|
5902
|
+
)
|
|
5903
|
+
preflight_source_root_keys = {
|
|
5904
|
+
root_key
|
|
5905
|
+
for _path, root_key in preflight_sources
|
|
5906
|
+
if root_key is not None
|
|
5907
|
+
}
|
|
5908
|
+
(
|
|
5909
|
+
_safe_sources,
|
|
5910
|
+
refused_sources,
|
|
5911
|
+
_safe_roots,
|
|
5912
|
+
refused_root_keys,
|
|
5913
|
+
) = _partition_codex_prune_candidates(
|
|
5914
|
+
preflight_scope, preflight_sources, preflight_source_root_keys
|
|
5915
|
+
)
|
|
5916
|
+
if refused_sources or refused_root_keys:
|
|
5917
|
+
_codex_prune_refusal_record(
|
|
5918
|
+
conn,
|
|
5919
|
+
store="cache",
|
|
5920
|
+
scope=preflight_scope,
|
|
5921
|
+
refused_sources=refused_sources,
|
|
5922
|
+
refused_root_keys=refused_root_keys,
|
|
5923
|
+
)
|
|
5924
|
+
conn.commit()
|
|
5925
|
+
stats.files_total = len(preflight_files)
|
|
5926
|
+
stats.prune_refused = True
|
|
5927
|
+
stats.prune_refused_files = len({
|
|
5928
|
+
path for path, _root in refused_sources
|
|
5929
|
+
})
|
|
5930
|
+
return stats
|
|
5488
5931
|
if rebuild:
|
|
5489
5932
|
# Clear INSIDE the lock — see sync_cache() for the full
|
|
5490
5933
|
# rationale. Done before the existing SELECT so delta
|
|
@@ -5590,19 +6033,13 @@ def sync_codex_cache(
|
|
|
5590
6033
|
stats.files_total = len(files)
|
|
5591
6034
|
_p_disc.set_count(len(files))
|
|
5592
6035
|
|
|
5593
|
-
# Scope the cache to the CURRENT root set
|
|
5594
|
-
#
|
|
5595
|
-
#
|
|
5596
|
-
#
|
|
5597
|
-
#
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
# re-derivable, so honoring the override beats retaining unreachable
|
|
5601
|
-
# rows. Done INSIDE the lock and committed BEFORE the existing-SELECT
|
|
5602
|
-
# + parse loop so no cache.db write lock is held across the read-heavy
|
|
5603
|
-
# ingest (same invariant as the --rebuild clear above). Concurrent
|
|
5604
|
-
# processes with different $CODEX_HOME would prune each other; the
|
|
5605
|
-
# flock serializes them and that is a pathological configuration.
|
|
6036
|
+
# Scope the cache to the CURRENT root set (issue #108), but only from
|
|
6037
|
+
# positive filesystem evidence (#485). A recognized new root still
|
|
6038
|
+
# authorizes the deliberate A -> B scope switch. No recognized root,
|
|
6039
|
+
# or one configured-but-unrecognizable sibling, is unknown filesystem
|
|
6040
|
+
# state rather than proof of deletion and therefore fails closed.
|
|
6041
|
+
protected_paths: set[str] = set()
|
|
6042
|
+
ordinary_prune_scope: _CodexPruneScope | None = None
|
|
5606
6043
|
if not rebuild and not targeted: # --rebuild already cleared; targeted bypasses
|
|
5607
6044
|
current_file_identities = {
|
|
5608
6045
|
(str(item.source_path), item.source_root_key) for item in files
|
|
@@ -5619,12 +6056,39 @@ def sync_codex_cache(
|
|
|
5619
6056
|
orphan_sources, orphan_root_keys = _collect_inactive_codex_paths_and_roots(
|
|
5620
6057
|
conn, current_file_identities, active_root_keys,
|
|
5621
6058
|
)
|
|
5622
|
-
|
|
6059
|
+
prune_scope = _codex_prune_scope(files)
|
|
6060
|
+
ordinary_prune_scope = prune_scope
|
|
6061
|
+
(
|
|
6062
|
+
safe_sources,
|
|
6063
|
+
_refused_orphan_sources,
|
|
6064
|
+
safe_root_keys,
|
|
6065
|
+
_refused_orphan_root_keys,
|
|
6066
|
+
) = _partition_codex_prune_candidates(
|
|
6067
|
+
prune_scope, orphan_sources, orphan_root_keys
|
|
6068
|
+
)
|
|
6069
|
+
retained_sources, _retained_root_keys = (
|
|
6070
|
+
_collect_retained_codex_paths_and_roots(conn)
|
|
6071
|
+
)
|
|
6072
|
+
retained_source_root_keys = {
|
|
6073
|
+
root_key
|
|
6074
|
+
for _path, root_key in retained_sources
|
|
6075
|
+
if root_key is not None
|
|
6076
|
+
}
|
|
6077
|
+
(
|
|
6078
|
+
_safe_retained_sources,
|
|
6079
|
+
refused_sources,
|
|
6080
|
+
_safe_retained_roots,
|
|
6081
|
+
refused_root_keys,
|
|
6082
|
+
) = _partition_codex_prune_candidates(
|
|
6083
|
+
prune_scope, retained_sources, retained_source_root_keys
|
|
6084
|
+
)
|
|
6085
|
+
protected_paths = {path for path, _root in refused_sources}
|
|
6086
|
+
if safe_sources or safe_root_keys:
|
|
5623
6087
|
before_prune = conn.total_changes
|
|
5624
6088
|
# #294 S6: capture the conversation keys the orphan rows belong to
|
|
5625
6089
|
# BEFORE deleting, so the rollups can be repaired/deleted after.
|
|
5626
6090
|
orphan_keys: set = set()
|
|
5627
|
-
for orphan_path, orphan_root_key in
|
|
6091
|
+
for orphan_path, orphan_root_key in safe_sources:
|
|
5628
6092
|
orphan_keys.update(
|
|
5629
6093
|
row[0] for row in conn.execute(
|
|
5630
6094
|
"SELECT DISTINCT conversation_key FROM codex_conversation_messages "
|
|
@@ -5637,7 +6101,7 @@ def sync_codex_cache(
|
|
|
5637
6101
|
)
|
|
5638
6102
|
_prune_inactive_codex_source_roots(
|
|
5639
6103
|
conn, active_root_keys,
|
|
5640
|
-
candidate_root_keys=
|
|
6104
|
+
candidate_root_keys=safe_root_keys,
|
|
5641
6105
|
)
|
|
5642
6106
|
# Recompute-affected-or-delete the rollups the prune touched (§3.2):
|
|
5643
6107
|
# a conversation with no surviving rows loses its rollup, one that
|
|
@@ -5645,8 +6109,24 @@ def sync_codex_cache(
|
|
|
5645
6109
|
_recompute_codex_rollups(conn, orphan_keys)
|
|
5646
6110
|
if conn.total_changes != before_prune:
|
|
5647
6111
|
_bump_codex_physical_mutation_seq(conn)
|
|
6112
|
+
stats.files_pruned = len({path for path, _root in safe_sources})
|
|
6113
|
+
if refused_sources or refused_root_keys:
|
|
6114
|
+
_codex_prune_refusal_record(
|
|
6115
|
+
conn,
|
|
6116
|
+
store="cache",
|
|
6117
|
+
scope=prune_scope,
|
|
6118
|
+
refused_sources=refused_sources,
|
|
6119
|
+
refused_root_keys=refused_root_keys,
|
|
6120
|
+
)
|
|
6121
|
+
stats.prune_refused = True
|
|
6122
|
+
stats.prune_refused_files = len({
|
|
6123
|
+
path for path, _root in refused_sources
|
|
6124
|
+
})
|
|
6125
|
+
if (
|
|
6126
|
+
safe_sources or safe_root_keys
|
|
6127
|
+
or refused_sources or refused_root_keys
|
|
6128
|
+
):
|
|
5648
6129
|
conn.commit()
|
|
5649
|
-
stats.files_pruned = len({path for path, _root in orphan_sources})
|
|
5650
6130
|
|
|
5651
6131
|
# This SELECT does NOT open an implicit transaction (Python's
|
|
5652
6132
|
# sqlite3 module only BEGINs on DML). Do NOT add any INSERT/
|
|
@@ -5808,6 +6288,12 @@ def sync_codex_cache(
|
|
|
5808
6288
|
break
|
|
5809
6289
|
jp = discovered.source_path
|
|
5810
6290
|
path_str = str(jp)
|
|
6291
|
+
if path_str in protected_paths:
|
|
6292
|
+
# The file is still visible, but its configured root did not
|
|
6293
|
+
# yield recognizable Codex data. Treat that as unknown
|
|
6294
|
+
# filesystem state: neither truncation/requalification reset
|
|
6295
|
+
# nor append ingestion may replace the retained generation.
|
|
6296
|
+
continue
|
|
5811
6297
|
try:
|
|
5812
6298
|
st = jp.stat()
|
|
5813
6299
|
except OSError as exc:
|
|
@@ -6543,6 +7029,17 @@ def sync_codex_cache(
|
|
|
6543
7029
|
and stats.files_failed == 0
|
|
6544
7030
|
and stats.files_deferred_torn == 0
|
|
6545
7031
|
):
|
|
7032
|
+
recovery_scope = rebuild_prune_scope or ordinary_prune_scope
|
|
7033
|
+
if (
|
|
7034
|
+
recovery_scope is not None
|
|
7035
|
+
and recovery_scope.recognized_root_keys
|
|
7036
|
+
and not stats.prune_refused
|
|
7037
|
+
and (
|
|
7038
|
+
stats.files_processed + stats.files_skipped_unchanged
|
|
7039
|
+
== stats.files_total
|
|
7040
|
+
)
|
|
7041
|
+
):
|
|
7042
|
+
_clear_codex_prune_refusal(conn)
|
|
6546
7043
|
if replay_pending:
|
|
6547
7044
|
conn.execute("DELETE FROM cache_meta WHERE key = ?",
|
|
6548
7045
|
(CODEX_REPLAY_FROM_ZERO_KEY,))
|
|
@@ -8095,6 +8592,160 @@ def open_conversations_db(*, attach_cache: bool = True) -> sqlite3.Connection:
|
|
|
8095
8592
|
return _conversations_open_guarded(attach_cache=attach_cache)
|
|
8096
8593
|
|
|
8097
8594
|
|
|
8595
|
+
def scope_conversations_db_to_account(
|
|
8596
|
+
conn: sqlite3.Connection, account_key: str,
|
|
8597
|
+
) -> None:
|
|
8598
|
+
"""Narrow one conversation read connection to ``account_key`` (#347).
|
|
8599
|
+
|
|
8600
|
+
The query kernels intentionally retain their byte-frozen unqualified SQL.
|
|
8601
|
+
A qualified request gets a fresh SQLite connection and this helper shadows
|
|
8602
|
+
every transcript/accounting leaf with TEMP views on that connection only.
|
|
8603
|
+
SQLite resolves TEMP before main/attached schemas, so browse, facets,
|
|
8604
|
+
search, detail, outline, export, payload, media, find, and cost/token joins
|
|
8605
|
+
all inherit the same physical-row predicate without a parallel query stack.
|
|
8606
|
+
|
|
8607
|
+
The two rollups are rebuilt into TEMP tables from the filtered leaves. No
|
|
8608
|
+
persistent row is changed and an unqualified connection takes none of this
|
|
8609
|
+
path, preserving the existing SQL plans and serialized bytes.
|
|
8610
|
+
"""
|
|
8611
|
+
import _lib_accounts
|
|
8612
|
+
|
|
8613
|
+
key = str(account_key or "").strip()
|
|
8614
|
+
if not key:
|
|
8615
|
+
raise ValueError("account_key is required")
|
|
8616
|
+
if key == _lib_accounts.UNATTRIBUTED:
|
|
8617
|
+
stored_predicate = "COALESCE(account_key,'unattributed')='unattributed'"
|
|
8618
|
+
else:
|
|
8619
|
+
# The value lives in a one-row TEMP table and reaches every view through
|
|
8620
|
+
# a subquery; never interpolate an opaque account key into DDL.
|
|
8621
|
+
stored_predicate = (
|
|
8622
|
+
"COALESCE(account_key,'unattributed')="
|
|
8623
|
+
"(SELECT account_key FROM _conversation_account_scope)"
|
|
8624
|
+
)
|
|
8625
|
+
|
|
8626
|
+
conn.execute(
|
|
8627
|
+
"CREATE TEMP TABLE _conversation_account_scope "
|
|
8628
|
+
"(account_key TEXT NOT NULL PRIMARY KEY)"
|
|
8629
|
+
)
|
|
8630
|
+
conn.execute(
|
|
8631
|
+
"INSERT INTO _conversation_account_scope(account_key) VALUES(?)", (key,)
|
|
8632
|
+
)
|
|
8633
|
+
|
|
8634
|
+
conn.executescript(
|
|
8635
|
+
f"""
|
|
8636
|
+
CREATE TEMP VIEW conversation_messages AS
|
|
8637
|
+
SELECT * FROM main.conversation_messages WHERE {stored_predicate};
|
|
8638
|
+
CREATE TEMP VIEW conversation_ai_titles AS
|
|
8639
|
+
SELECT t.* FROM main.conversation_ai_titles t
|
|
8640
|
+
WHERE EXISTS (
|
|
8641
|
+
SELECT 1 FROM conversation_messages m
|
|
8642
|
+
WHERE m.source_path=t.source_path
|
|
8643
|
+
AND m.byte_offset=t.byte_offset
|
|
8644
|
+
);
|
|
8645
|
+
CREATE TEMP VIEW conversation_file_touches AS
|
|
8646
|
+
SELECT t.* FROM main.conversation_file_touches t
|
|
8647
|
+
WHERE EXISTS (
|
|
8648
|
+
SELECT 1 FROM conversation_messages m WHERE m.id=t.message_id
|
|
8649
|
+
);
|
|
8650
|
+
CREATE TEMP VIEW session_entries AS
|
|
8651
|
+
SELECT * FROM cache_db.session_entries WHERE {stored_predicate};
|
|
8652
|
+
-- A file-level project path cannot be partitioned when one JSONL
|
|
8653
|
+
-- switches accounts. Scoped anonymization therefore uses only the
|
|
8654
|
+
-- already-filtered physical message CWDs.
|
|
8655
|
+
CREATE TEMP VIEW session_files AS
|
|
8656
|
+
SELECT * FROM cache_db.session_files WHERE 0;
|
|
8657
|
+
|
|
8658
|
+
CREATE TEMP VIEW codex_conversation_events AS
|
|
8659
|
+
SELECT * FROM main.codex_conversation_events WHERE {stored_predicate};
|
|
8660
|
+
CREATE TEMP VIEW codex_conversation_messages AS
|
|
8661
|
+
SELECT * FROM main.codex_conversation_messages WHERE {stored_predicate};
|
|
8662
|
+
CREATE TEMP VIEW codex_conversation_file_touches AS
|
|
8663
|
+
SELECT t.* FROM main.codex_conversation_file_touches t
|
|
8664
|
+
WHERE EXISTS (
|
|
8665
|
+
SELECT 1 FROM codex_conversation_messages m WHERE m.id=t.message_id
|
|
8666
|
+
);
|
|
8667
|
+
CREATE TEMP VIEW codex_find_projection AS
|
|
8668
|
+
SELECT p.* FROM main.codex_find_projection p
|
|
8669
|
+
WHERE EXISTS (
|
|
8670
|
+
SELECT 1 FROM codex_conversation_messages m WHERE m.id=p.message_id
|
|
8671
|
+
);
|
|
8672
|
+
CREATE TEMP VIEW codex_session_entries AS
|
|
8673
|
+
SELECT * FROM cache_db.codex_session_entries WHERE {stored_predicate};
|
|
8674
|
+
|
|
8675
|
+
-- Thread metadata has conversation-level provenance and can predate an
|
|
8676
|
+
-- in-file account switch. Do not expose it in a scoped connection;
|
|
8677
|
+
-- the Codex rollup degrades explicitly to unassigned rather than leaking
|
|
8678
|
+
-- another account's cwd/git metadata.
|
|
8679
|
+
CREATE TEMP VIEW codex_conversation_threads AS
|
|
8680
|
+
SELECT * FROM cache_db.codex_conversation_threads WHERE 0;
|
|
8681
|
+
CREATE TEMP VIEW codex_source_roots AS
|
|
8682
|
+
SELECT * FROM cache_db.codex_source_roots WHERE 0;
|
|
8683
|
+
|
|
8684
|
+
CREATE TEMP TABLE conversation_sessions AS
|
|
8685
|
+
SELECT * FROM main.conversation_sessions WHERE 0;
|
|
8686
|
+
CREATE TEMP TABLE codex_conversation_rollups (
|
|
8687
|
+
conversation_key TEXT NOT NULL PRIMARY KEY,
|
|
8688
|
+
source_root_key TEXT NOT NULL,
|
|
8689
|
+
parent_thread_id TEXT,
|
|
8690
|
+
item_count INTEGER NOT NULL DEFAULT 0,
|
|
8691
|
+
started_utc TEXT,
|
|
8692
|
+
last_activity_utc TEXT,
|
|
8693
|
+
project_key TEXT,
|
|
8694
|
+
project_label TEXT,
|
|
8695
|
+
models_json TEXT,
|
|
8696
|
+
title TEXT
|
|
8697
|
+
);
|
|
8698
|
+
"""
|
|
8699
|
+
)
|
|
8700
|
+
_recompute_conversation_sessions(conn)
|
|
8701
|
+
codex_keys = {
|
|
8702
|
+
row[0]
|
|
8703
|
+
for row in conn.execute(
|
|
8704
|
+
"SELECT DISTINCT conversation_key FROM codex_conversation_messages"
|
|
8705
|
+
)
|
|
8706
|
+
if row[0]
|
|
8707
|
+
}
|
|
8708
|
+
safe_project_attribution: dict[str, tuple[str | None, str | None]] = {}
|
|
8709
|
+
for conversation_key in codex_keys:
|
|
8710
|
+
persisted = conn.execute(
|
|
8711
|
+
"SELECT project_key,project_label "
|
|
8712
|
+
"FROM main.codex_conversation_rollups WHERE conversation_key=?",
|
|
8713
|
+
(conversation_key,),
|
|
8714
|
+
).fetchone()
|
|
8715
|
+
if persisted is not None:
|
|
8716
|
+
safe_project_attribution[conversation_key] = persisted
|
|
8717
|
+
continue
|
|
8718
|
+
thread = conn.execute(
|
|
8719
|
+
"SELECT source_root_key,cwd,git_json "
|
|
8720
|
+
"FROM cache_db.codex_conversation_threads WHERE conversation_key=?",
|
|
8721
|
+
(conversation_key,),
|
|
8722
|
+
).fetchone()
|
|
8723
|
+
if thread is not None:
|
|
8724
|
+
safe_project_attribution[conversation_key] = (
|
|
8725
|
+
_codex_conversation_project_attribution(*thread)
|
|
8726
|
+
)
|
|
8727
|
+
_recompute_codex_rollups(conn, codex_keys)
|
|
8728
|
+
# Project identity is safe conversation-level enrichment: it is already
|
|
8729
|
+
# visible on the unqualified rail and contains only an opaque key plus the
|
|
8730
|
+
# derived display label. Preserve those two fields for conversations that
|
|
8731
|
+
# survived the physical-row account predicate, without exposing the
|
|
8732
|
+
# conversation-level cwd/git, source-root path, title, or thread topology
|
|
8733
|
+
# that produced them (#497).
|
|
8734
|
+
conn.executemany(
|
|
8735
|
+
"UPDATE codex_conversation_rollups SET project_key=?,project_label=? "
|
|
8736
|
+
"WHERE conversation_key=?",
|
|
8737
|
+
[
|
|
8738
|
+
(project_key, project_label, conversation_key)
|
|
8739
|
+
for conversation_key, (project_key, project_label)
|
|
8740
|
+
in safe_project_attribution.items()
|
|
8741
|
+
],
|
|
8742
|
+
)
|
|
8743
|
+
# TEMP rollup writes open a transaction. Close it before a long-lived
|
|
8744
|
+
# account-scoped SSE reader starts watching so later provider-writer commits
|
|
8745
|
+
# are visible to the dynamic leaf views and never contend on this setup work.
|
|
8746
|
+
conn.commit()
|
|
8747
|
+
|
|
8748
|
+
|
|
8098
8749
|
def _open_conversations_db_for_recovery(
|
|
8099
8750
|
*, attach_cache: bool = True,
|
|
8100
8751
|
) -> sqlite3.Connection:
|
|
@@ -8669,6 +9320,7 @@ def _prepare_claude_conversation_maintenance(
|
|
|
8669
9320
|
*,
|
|
8670
9321
|
rebuild: bool,
|
|
8671
9322
|
targeted: bool,
|
|
9323
|
+
active_account_key: "str | None" = None,
|
|
8672
9324
|
) -> bool:
|
|
8673
9325
|
"""Consume transcript-only upgrade work under the conversation flock.
|
|
8674
9326
|
|
|
@@ -8760,7 +9412,9 @@ def _prepare_claude_conversation_maintenance(
|
|
|
8760
9412
|
"'conversation_background_mcp_reingest_pending')"
|
|
8761
9413
|
).fetchone() is not None
|
|
8762
9414
|
if reingest:
|
|
8763
|
-
_resumable_reingest_conversation_messages(
|
|
9415
|
+
_resumable_reingest_conversation_messages(
|
|
9416
|
+
conn, active_account_key=active_account_key,
|
|
9417
|
+
)
|
|
8764
9418
|
_set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
|
|
8765
9419
|
conn.commit()
|
|
8766
9420
|
|
|
@@ -8806,6 +9460,20 @@ def sync_claude_conversations(
|
|
|
8806
9460
|
stats.lock_contended = True
|
|
8807
9461
|
return stats
|
|
8808
9462
|
|
|
9463
|
+
# #347 observe-and-stamp mirrors the accounting ingest boundary. A
|
|
9464
|
+
# torn credential read is undecided, not unattributed: defer before any
|
|
9465
|
+
# maintenance/rebuild mutation so the cursor remains replayable.
|
|
9466
|
+
import _lib_accounts
|
|
9467
|
+
claude_identity = _cctally_core._resolve_active_claude_identity()
|
|
9468
|
+
if claude_identity.get("status") == "torn":
|
|
9469
|
+
stats.files_deferred_torn += 1
|
|
9470
|
+
stats.deferred_reason = "identity_torn"
|
|
9471
|
+
return stats
|
|
9472
|
+
active_key = claude_identity["account_key"]
|
|
9473
|
+
active_account_key = (
|
|
9474
|
+
None if active_key == _lib_accounts.UNATTRIBUTED else active_key
|
|
9475
|
+
)
|
|
9476
|
+
|
|
8809
9477
|
targeted = only_paths is not None
|
|
8810
9478
|
pending_rebuild = conn.execute(
|
|
8811
9479
|
"SELECT 1 FROM cache_meta "
|
|
@@ -8835,10 +9503,19 @@ def sync_claude_conversations(
|
|
|
8835
9503
|
|
|
8836
9504
|
_report_conversation_progress(progress, "prepare", stats)
|
|
8837
9505
|
maintenance_replayed = _prepare_claude_conversation_maintenance(
|
|
8838
|
-
conn, rebuild=rebuild, targeted=targeted
|
|
9506
|
+
conn, rebuild=rebuild, targeted=targeted,
|
|
9507
|
+
active_account_key=active_account_key,
|
|
8839
9508
|
)
|
|
8840
9509
|
|
|
9510
|
+
rebuild_account_stamps: dict[tuple[str, int], str | None] = {}
|
|
8841
9511
|
if rebuild:
|
|
9512
|
+
rebuild_account_stamps = {
|
|
9513
|
+
(str(path), int(offset)): account_key
|
|
9514
|
+
for path, offset, account_key in conn.execute(
|
|
9515
|
+
"SELECT source_path,byte_offset,account_key "
|
|
9516
|
+
"FROM conversation_messages"
|
|
9517
|
+
)
|
|
9518
|
+
}
|
|
8842
9519
|
clear_conversation_messages(conn)
|
|
8843
9520
|
conn.execute("DELETE FROM conversation_ai_titles")
|
|
8844
9521
|
conn.execute("DELETE FROM conversation_sessions")
|
|
@@ -8915,7 +9592,16 @@ def sync_claude_conversations(
|
|
|
8915
9592
|
include_cost=False,
|
|
8916
9593
|
):
|
|
8917
9594
|
if mrow is not None:
|
|
8918
|
-
|
|
9595
|
+
account_key = rebuild_account_stamps.get(
|
|
9596
|
+
(path_str, int(mrow.byte_offset)),
|
|
9597
|
+
active_account_key,
|
|
9598
|
+
)
|
|
9599
|
+
conv_rows.append(
|
|
9600
|
+
_conv_row_tuple(
|
|
9601
|
+
mrow, path_str, account_key,
|
|
9602
|
+
include_account=True,
|
|
9603
|
+
)
|
|
9604
|
+
)
|
|
8919
9605
|
if ai is not None:
|
|
8920
9606
|
ai_rows.append(
|
|
8921
9607
|
(ai.session_id, ai.ai_title, path_str, ai.byte_offset)
|
|
@@ -8952,7 +9638,7 @@ def sync_claude_conversations(
|
|
|
8952
9638
|
)
|
|
8953
9639
|
stats.files_reset_truncated += 1
|
|
8954
9640
|
if conv_rows:
|
|
8955
|
-
conn.executemany(
|
|
9641
|
+
conn.executemany(_CONV_ACCOUNT_INSERT_SQL, conv_rows)
|
|
8956
9642
|
_fill_file_touches(
|
|
8957
9643
|
conn, scope=[(row[3], row[4]) for row in conv_rows]
|
|
8958
9644
|
)
|
|
@@ -9059,6 +9745,7 @@ def sync_codex_conversations(
|
|
|
9059
9745
|
"""Delta-sync Codex events/search rows into conversations.db (#320)."""
|
|
9060
9746
|
stats = CodexIngestStats()
|
|
9061
9747
|
did_from_zero_replay = False
|
|
9748
|
+
rebuild_account_stamps: dict[tuple[str, int], str | None] = {}
|
|
9062
9749
|
_cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
|
|
9063
9750
|
_cctally_core.CONVERSATIONS_LOCK_CODEX_PATH.touch()
|
|
9064
9751
|
lock_fh = open(_cctally_core.CONVERSATIONS_LOCK_CODEX_PATH, "w")
|
|
@@ -9106,7 +9793,51 @@ def sync_codex_conversations(
|
|
|
9106
9793
|
return stats
|
|
9107
9794
|
rebuild = (
|
|
9108
9795
|
rebuild or pending_rebuild or contract_rebuild or codex_replay_pending)
|
|
9796
|
+
rebuild_prune_scope: _CodexPruneScope | None = None
|
|
9109
9797
|
if rebuild:
|
|
9798
|
+
rebuild_account_stamps = {
|
|
9799
|
+
(str(path), int(offset)): account_key
|
|
9800
|
+
for path, offset, account_key in conn.execute(
|
|
9801
|
+
"SELECT source_path,line_offset,account_key "
|
|
9802
|
+
"FROM codex_conversation_events"
|
|
9803
|
+
)
|
|
9804
|
+
}
|
|
9805
|
+
# #485: establish positive root evidence before the destructive
|
|
9806
|
+
# clear. A rebuild against an empty or missing CODEX_HOME must keep
|
|
9807
|
+
# the retained transcript store and its pending replay marker.
|
|
9808
|
+
preflight_files = _discover_codex_files_with_roots()
|
|
9809
|
+
preflight_scope = _codex_prune_scope(preflight_files)
|
|
9810
|
+
rebuild_prune_scope = preflight_scope
|
|
9811
|
+
preflight_sources = [
|
|
9812
|
+
(path, root_key)
|
|
9813
|
+
for path, root_key in conn.execute(
|
|
9814
|
+
"SELECT path,source_root_key "
|
|
9815
|
+
"FROM codex_conversation_source_files"
|
|
9816
|
+
)
|
|
9817
|
+
]
|
|
9818
|
+
(
|
|
9819
|
+
_safe_sources,
|
|
9820
|
+
refused_sources,
|
|
9821
|
+
_safe_roots,
|
|
9822
|
+
_refused_roots,
|
|
9823
|
+
) = _partition_codex_prune_candidates(
|
|
9824
|
+
preflight_scope, preflight_sources, set()
|
|
9825
|
+
)
|
|
9826
|
+
if refused_sources:
|
|
9827
|
+
_codex_prune_refusal_record(
|
|
9828
|
+
conn,
|
|
9829
|
+
store="conversations",
|
|
9830
|
+
scope=preflight_scope,
|
|
9831
|
+
refused_sources=refused_sources,
|
|
9832
|
+
refused_root_keys=set(),
|
|
9833
|
+
)
|
|
9834
|
+
conn.commit()
|
|
9835
|
+
stats.files_total = len(preflight_files)
|
|
9836
|
+
stats.prune_refused = True
|
|
9837
|
+
stats.prune_refused_files = len({
|
|
9838
|
+
path for path, _root in refused_sources
|
|
9839
|
+
})
|
|
9840
|
+
return stats
|
|
9110
9841
|
conn.execute(
|
|
9111
9842
|
"INSERT OR REPLACE INTO cache_meta(key,value) VALUES(?,?)",
|
|
9112
9843
|
("conversation_rebuild_codex_pending", "1"),
|
|
@@ -9152,9 +9883,36 @@ def sync_codex_conversations(
|
|
|
9152
9883
|
if current_size < prev[0]:
|
|
9153
9884
|
stats.deferred_reason = "truncation"
|
|
9154
9885
|
return stats
|
|
9886
|
+
protected_paths: set[str] = set()
|
|
9887
|
+
ordinary_prune_scope: _CodexPruneScope | None = None
|
|
9155
9888
|
if only_paths is None:
|
|
9156
9889
|
active_paths = {str(item.source_path) for item in files}
|
|
9157
|
-
|
|
9890
|
+
prune_scope = _codex_prune_scope(files)
|
|
9891
|
+
ordinary_prune_scope = prune_scope
|
|
9892
|
+
stale_sources = [
|
|
9893
|
+
(stale_path, existing[stale_path][3])
|
|
9894
|
+
for stale_path in sorted(set(existing) - active_paths)
|
|
9895
|
+
]
|
|
9896
|
+
safe_sources, _refused_stale, _safe_roots, _refused_roots = (
|
|
9897
|
+
_partition_codex_prune_candidates(
|
|
9898
|
+
prune_scope, stale_sources, set()
|
|
9899
|
+
)
|
|
9900
|
+
)
|
|
9901
|
+
retained_sources = [
|
|
9902
|
+
(path, values[3])
|
|
9903
|
+
for path, values in existing.items()
|
|
9904
|
+
if os.path.isabs(path)
|
|
9905
|
+
]
|
|
9906
|
+
(
|
|
9907
|
+
_safe_retained,
|
|
9908
|
+
refused_sources,
|
|
9909
|
+
_safe_retained_roots,
|
|
9910
|
+
_refused_retained_roots,
|
|
9911
|
+
) = _partition_codex_prune_candidates(
|
|
9912
|
+
prune_scope, retained_sources, set()
|
|
9913
|
+
)
|
|
9914
|
+
protected_paths = {path for path, _root in refused_sources}
|
|
9915
|
+
for stale_path, _stale_root_key in safe_sources:
|
|
9158
9916
|
affected = {
|
|
9159
9917
|
row[0] for row in conn.execute(
|
|
9160
9918
|
"SELECT DISTINCT conversation_key "
|
|
@@ -9179,11 +9937,30 @@ def sync_codex_conversations(
|
|
|
9179
9937
|
(stale_path,),
|
|
9180
9938
|
)
|
|
9181
9939
|
_recompute_codex_rollups(conn, affected)
|
|
9940
|
+
import _lib_codex_conversation_query as query
|
|
9941
|
+
query.materialize_codex_find_projection(conn, affected)
|
|
9942
|
+
stats.files_pruned = len({path for path, _root in safe_sources})
|
|
9943
|
+
if refused_sources:
|
|
9944
|
+
_codex_prune_refusal_record(
|
|
9945
|
+
conn,
|
|
9946
|
+
store="conversations",
|
|
9947
|
+
scope=prune_scope,
|
|
9948
|
+
refused_sources=refused_sources,
|
|
9949
|
+
refused_root_keys=set(),
|
|
9950
|
+
)
|
|
9951
|
+
stats.prune_refused = True
|
|
9952
|
+
stats.prune_refused_files = len({
|
|
9953
|
+
path for path, _root in refused_sources
|
|
9954
|
+
})
|
|
9182
9955
|
conn.commit()
|
|
9183
9956
|
|
|
9184
9957
|
for discovered in files:
|
|
9185
9958
|
jp = discovered.source_path
|
|
9186
9959
|
path_str = str(jp)
|
|
9960
|
+
if path_str in protected_paths:
|
|
9961
|
+
# Visible but unrecognizable input is not evidence that the
|
|
9962
|
+
# retained transcript generation should be reset or replaced.
|
|
9963
|
+
continue
|
|
9187
9964
|
try:
|
|
9188
9965
|
st = jp.stat()
|
|
9189
9966
|
except OSError:
|
|
@@ -9219,6 +9996,31 @@ def sync_codex_conversations(
|
|
|
9219
9996
|
initial_conversation = prev[10] if prev else None
|
|
9220
9997
|
initial_turn = prev[11] if prev else None
|
|
9221
9998
|
|
|
9999
|
+
# #347 uses the accounting cache's journal-backed file-range map as
|
|
10000
|
+
# the Codex transcript oracle. It is already attached read-only,
|
|
10001
|
+
# and core ingest runs first, so rebuilds replay the original
|
|
10002
|
+
# decision instead of consulting the currently-active auth.json.
|
|
10003
|
+
try:
|
|
10004
|
+
file_identity = codex_file_identity(discovered)
|
|
10005
|
+
incarnation_row = conn.execute(
|
|
10006
|
+
"SELECT incarnation FROM cache_db.codex_file_incarnations "
|
|
10007
|
+
"WHERE file_identity=?",
|
|
10008
|
+
(file_identity,),
|
|
10009
|
+
).fetchone()
|
|
10010
|
+
incarnation = int(incarnation_row[0]) if incarnation_row else 1
|
|
10011
|
+
account_ranges = [
|
|
10012
|
+
(int(off), key)
|
|
10013
|
+
for off, key in conn.execute(
|
|
10014
|
+
"SELECT from_offset,account_key "
|
|
10015
|
+
"FROM cache_db.codex_file_accounts "
|
|
10016
|
+
"WHERE file_identity=? AND incarnation=? "
|
|
10017
|
+
"ORDER BY from_offset",
|
|
10018
|
+
(file_identity, incarnation),
|
|
10019
|
+
)
|
|
10020
|
+
]
|
|
10021
|
+
except sqlite3.OperationalError:
|
|
10022
|
+
account_ranges = []
|
|
10023
|
+
|
|
9222
10024
|
state = _CodexIterState(
|
|
9223
10025
|
session_id=initial_session_id,
|
|
9224
10026
|
model=initial_model,
|
|
@@ -9241,6 +10043,7 @@ def sync_codex_conversations(
|
|
|
9241
10043
|
)
|
|
9242
10044
|
events = []
|
|
9243
10045
|
event_rows = []
|
|
10046
|
+
account_by_physical: dict[tuple[str, int], str | None] = {}
|
|
9244
10047
|
yielded = 0
|
|
9245
10048
|
try:
|
|
9246
10049
|
with open(jp, "rb") as fh:
|
|
@@ -9256,6 +10059,16 @@ def sync_codex_conversations(
|
|
|
9256
10059
|
):
|
|
9257
10060
|
event = emission.event
|
|
9258
10061
|
events.append(event)
|
|
10062
|
+
covered, account_key = codex_account_for_offset(
|
|
10063
|
+
account_ranges, int(event.line_offset)
|
|
10064
|
+
)
|
|
10065
|
+
if not covered:
|
|
10066
|
+
account_key = rebuild_account_stamps.get(
|
|
10067
|
+
(event.source_path, int(event.line_offset))
|
|
10068
|
+
)
|
|
10069
|
+
account_by_physical[
|
|
10070
|
+
(event.source_path, int(event.line_offset))
|
|
10071
|
+
] = account_key
|
|
9259
10072
|
event_rows.append((
|
|
9260
10073
|
event.source_path,
|
|
9261
10074
|
event.line_offset,
|
|
@@ -9270,6 +10083,7 @@ def sync_codex_conversations(
|
|
|
9270
10083
|
event.turn_id,
|
|
9271
10084
|
event.call_id,
|
|
9272
10085
|
event.payload_json,
|
|
10086
|
+
account_key,
|
|
9273
10087
|
))
|
|
9274
10088
|
if emission.accounting is not None:
|
|
9275
10089
|
yielded += 1
|
|
@@ -9327,12 +10141,13 @@ def sync_codex_conversations(
|
|
|
9327
10141
|
"INSERT OR IGNORE INTO codex_conversation_events "
|
|
9328
10142
|
"(source_path,line_offset,source_root_key,conversation_key,"
|
|
9329
10143
|
"native_thread_id,root_thread_id,parent_thread_id,"
|
|
9330
|
-
"timestamp_utc,record_type,event_type,turn_id,call_id,payload_json
|
|
9331
|
-
"VALUES (
|
|
10144
|
+
"timestamp_utc,record_type,event_type,turn_id,call_id,payload_json,"
|
|
10145
|
+
"account_key) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
9332
10146
|
event_rows,
|
|
9333
10147
|
)
|
|
9334
10148
|
_insert_codex_normalized_rows(
|
|
9335
|
-
conn, normalized.rows, normalized.touches
|
|
10149
|
+
conn, normalized.rows, normalized.touches,
|
|
10150
|
+
account_by_physical,
|
|
9336
10151
|
)
|
|
9337
10152
|
affected_keys.update(
|
|
9338
10153
|
row.conversation_key for row in normalized.rows
|
|
@@ -9345,6 +10160,8 @@ def sync_codex_conversations(
|
|
|
9345
10160
|
_repair_codex_turn_ids_for_source(conn, path_str)
|
|
9346
10161
|
)
|
|
9347
10162
|
_recompute_codex_rollups(conn, affected_keys)
|
|
10163
|
+
import _lib_codex_conversation_query as query
|
|
10164
|
+
query.materialize_codex_find_projection(conn, affected_keys)
|
|
9348
10165
|
terminal = state.thread
|
|
9349
10166
|
conn.execute(
|
|
9350
10167
|
"INSERT INTO codex_conversation_source_files "
|
|
@@ -9392,8 +10209,20 @@ def sync_codex_conversations(
|
|
|
9392
10209
|
stats.files_failed += 1
|
|
9393
10210
|
_report_conversation_progress(progress, "ingest", stats)
|
|
9394
10211
|
|
|
10212
|
+
run_codex_find_projection_backfill(conn)
|
|
9395
10213
|
_report_conversation_progress(progress, "finalize", stats)
|
|
9396
10214
|
if only_paths is None and stats.files_failed == 0:
|
|
10215
|
+
recovery_scope = rebuild_prune_scope or ordinary_prune_scope
|
|
10216
|
+
if (
|
|
10217
|
+
recovery_scope is not None
|
|
10218
|
+
and recovery_scope.recognized_root_keys
|
|
10219
|
+
and not stats.prune_refused
|
|
10220
|
+
and (
|
|
10221
|
+
stats.files_processed + stats.files_skipped_unchanged
|
|
10222
|
+
== stats.files_total
|
|
10223
|
+
)
|
|
10224
|
+
):
|
|
10225
|
+
_clear_codex_prune_refusal(conn)
|
|
9397
10226
|
conn.execute(
|
|
9398
10227
|
"INSERT OR REPLACE INTO cache_meta(key,value) VALUES(?,?)",
|
|
9399
10228
|
(
|