cctally 1.90.1 → 1.92.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -0
- package/README.md +2 -2
- package/bin/_cctally_cache.py +863 -74
- package/bin/_cctally_config.py +57 -0
- package/bin/_cctally_core.py +53 -8
- package/bin/_cctally_dashboard.py +146 -5
- package/bin/_cctally_dashboard_conversation.py +164 -18
- package/bin/_cctally_dashboard_envelope.py +69 -12
- package/bin/_cctally_dashboard_sources.py +27 -1
- package/bin/_cctally_db.py +372 -10
- package/bin/_cctally_doctor.py +18 -1
- package/bin/_cctally_journal.py +535 -13
- package/bin/_cctally_journal_repair.py +6 -0
- package/bin/_cctally_parser.py +6 -0
- package/bin/_cctally_quota.py +171 -55
- package/bin/_cctally_record.py +13 -1
- package/bin/_cctally_rederive.py +4 -0
- package/bin/_cctally_store.py +311 -6
- package/bin/_cctally_transcript.py +32 -2
- package/bin/_lib_cache_report.py +8 -3
- package/bin/_lib_cache_report_wire.py +8 -20
- package/bin/_lib_codex_conversation.py +959 -81
- package/bin/_lib_codex_conversation_query.py +2792 -167
- package/bin/_lib_codex_find_projection.py +370 -0
- package/bin/_lib_codex_harness_preamble.py +176 -0
- package/bin/_lib_codex_hooks.py +5 -3
- package/bin/_lib_codex_js_scan.py +254 -0
- package/bin/_lib_codex_landmarks.py +309 -0
- package/bin/_lib_codex_reasoning_headings.py +73 -0
- package/bin/_lib_codex_segments.py +259 -0
- package/bin/_lib_codex_title_clean.py +116 -0
- package/bin/_lib_conversation_dispatch.py +153 -21
- package/bin/_lib_conversation_watch.py +4 -2
- package/bin/_lib_dashboard_sources.py +33 -32
- package/bin/_lib_doctor.py +64 -0
- package/bin/_lib_quota_alert_axes.py +31 -34
- package/bin/_lib_stats_damage.py +523 -0
- package/bin/cctally +5 -0
- package/dashboard/static/assets/index-BEzzJtUd.js +97 -0
- package/dashboard/static/assets/index-DnWdv8um.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +9 -1
- package/dashboard/static/assets/index-Bar8-S1i.css +0 -1
- package/dashboard/static/assets/index-CRogVlEC.js +0 -92
package/bin/_cctally_cache.py
CHANGED
|
@@ -227,6 +227,12 @@ _CONV_INSERT_SQL = (
|
|
|
227
227
|
" search_tool,search_thinking)"
|
|
228
228
|
" VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
|
229
229
|
)
|
|
230
|
+
_CONV_ACCOUNT_INSERT_SQL = _CONV_INSERT_SQL.replace(
|
|
231
|
+
"search_tool,search_thinking)", "search_tool,search_thinking,account_key)"
|
|
232
|
+
).replace(
|
|
233
|
+
"?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
234
|
+
"?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
235
|
+
)
|
|
230
236
|
|
|
231
237
|
# #193: last non-null write wins (ai-title carries no timestamp; see spec S1). NO
|
|
232
238
|
# byte_offset guard — it can't order a cross-file resumed session. Ordering is
|
|
@@ -383,7 +389,7 @@ SESSION_ENTRY_UPSERT_SQL_REWALK = (
|
|
|
383
389
|
+ _SESSION_ENTRY_REWALK_PHYSICAL_GUARD)
|
|
384
390
|
|
|
385
391
|
|
|
386
|
-
def _conv_row_tuple(m, path_str):
|
|
392
|
+
def _conv_row_tuple(m, path_str, account_key=None, *, include_account=False):
|
|
387
393
|
"""Flatten a ``MessageRow`` into the ``_CONV_INSERT_SQL`` column order.
|
|
388
394
|
|
|
389
395
|
The #177 enrichment fields (stop_reason / attribution_skill /
|
|
@@ -393,7 +399,7 @@ def _conv_row_tuple(m, path_str):
|
|
|
393
399
|
one tuple. #217 S1 / U7a: the documented-dead ``search_aux`` column is gone
|
|
394
400
|
from the live schema (dropped by migration 016); the split
|
|
395
401
|
``search_tool``/``search_thinking`` columns carry the non-prose index."""
|
|
396
|
-
|
|
402
|
+
row = (
|
|
397
403
|
m.session_id, m.uuid, m.parent_uuid, path_str, m.byte_offset,
|
|
398
404
|
m.timestamp_utc, m.entry_type, m.text, m.blocks_json, m.model,
|
|
399
405
|
m.msg_id, m.req_id, m.cwd, m.git_branch, m.is_sidechain,
|
|
@@ -401,6 +407,7 @@ def _conv_row_tuple(m, path_str):
|
|
|
401
407
|
m.stop_reason, m.attribution_skill, m.attribution_plugin,
|
|
402
408
|
m.search_tool, m.search_thinking,
|
|
403
409
|
)
|
|
410
|
+
return row + ((account_key,) if include_account else ())
|
|
404
411
|
|
|
405
412
|
|
|
406
413
|
def _iter_sync_entries(
|
|
@@ -1300,15 +1307,24 @@ def _codex_provider_roots() -> list[CodexProviderRoot]:
|
|
|
1300
1307
|
roots: list[CodexProviderRoot] = []
|
|
1301
1308
|
seen: set[pathlib.Path] = set()
|
|
1302
1309
|
for configured in _cctally()._codex_home_roots():
|
|
1303
|
-
|
|
1310
|
+
try:
|
|
1311
|
+
provider_root = _canonical_codex_path(configured)
|
|
1312
|
+
except OSError:
|
|
1313
|
+
# An unreadable configured root is unknown filesystem state. Leave
|
|
1314
|
+
# it out of discovery; the prune scope still retains its configured
|
|
1315
|
+
# identity and will refuse destructive cleanup for that root.
|
|
1316
|
+
continue
|
|
1304
1317
|
if provider_root in seen:
|
|
1305
1318
|
continue
|
|
1306
1319
|
sessions = configured / "sessions"
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1320
|
+
try:
|
|
1321
|
+
if sessions.is_dir():
|
|
1322
|
+
walk_root = sessions
|
|
1323
|
+
elif configured.is_dir():
|
|
1324
|
+
walk_root = configured
|
|
1325
|
+
else:
|
|
1326
|
+
continue
|
|
1327
|
+
except OSError:
|
|
1312
1328
|
continue
|
|
1313
1329
|
seen.add(provider_root)
|
|
1314
1330
|
roots.append(CodexProviderRoot(
|
|
@@ -1438,6 +1454,214 @@ def _discover_codex_files_with_roots() -> list[CodexDiscoveredFile]:
|
|
|
1438
1454
|
return discovered
|
|
1439
1455
|
|
|
1440
1456
|
|
|
1457
|
+
CODEX_ORPHAN_PRUNE_REFUSED_KEY = "codex_orphan_prune_refused"
|
|
1458
|
+
_CODEX_ROOT_RECOGNITION_TYPES = frozenset({
|
|
1459
|
+
"session_meta", "turn_context", "event_msg", "response_item", "world_state",
|
|
1460
|
+
"compacted",
|
|
1461
|
+
})
|
|
1462
|
+
_CODEX_ROOT_RECOGNITION_MAX_FILES = 32
|
|
1463
|
+
_CODEX_ROOT_RECOGNITION_MAX_BYTES = 64 * 1024
|
|
1464
|
+
_CODEX_ROOT_RECOGNITION_MAX_LINES = 64
|
|
1465
|
+
|
|
1466
|
+
|
|
1467
|
+
def _codex_file_is_recognizable(path: pathlib.Path) -> bool:
|
|
1468
|
+
"""Boundedly prove that one JSONL contains a known Codex record envelope.
|
|
1469
|
+
|
|
1470
|
+
Unknown records remain ingestible for forward compatibility, but they are
|
|
1471
|
+
not strong enough evidence to authorize deletion of retained history.
|
|
1472
|
+
"""
|
|
1473
|
+
consumed = 0
|
|
1474
|
+
lines = 0
|
|
1475
|
+
try:
|
|
1476
|
+
with path.open("rb") as fh:
|
|
1477
|
+
while (
|
|
1478
|
+
lines < _CODEX_ROOT_RECOGNITION_MAX_LINES
|
|
1479
|
+
and consumed < _CODEX_ROOT_RECOGNITION_MAX_BYTES
|
|
1480
|
+
):
|
|
1481
|
+
remaining = _CODEX_ROOT_RECOGNITION_MAX_BYTES - consumed
|
|
1482
|
+
raw = fh.readline(remaining + 1)
|
|
1483
|
+
if not raw:
|
|
1484
|
+
break
|
|
1485
|
+
lines += 1
|
|
1486
|
+
consumed += len(raw)
|
|
1487
|
+
if len(raw) > remaining or not raw.endswith(b"\n"):
|
|
1488
|
+
break
|
|
1489
|
+
try:
|
|
1490
|
+
obj = json.loads(
|
|
1491
|
+
raw,
|
|
1492
|
+
parse_constant=_lib_jsonl._reject_nonfinite_json_constant,
|
|
1493
|
+
)
|
|
1494
|
+
except (UnicodeDecodeError, ValueError, TypeError):
|
|
1495
|
+
continue
|
|
1496
|
+
if (
|
|
1497
|
+
isinstance(obj, dict)
|
|
1498
|
+
and obj.get("type") in _CODEX_ROOT_RECOGNITION_TYPES
|
|
1499
|
+
and isinstance(obj.get("payload"), dict)
|
|
1500
|
+
):
|
|
1501
|
+
return True
|
|
1502
|
+
except OSError:
|
|
1503
|
+
return False
|
|
1504
|
+
return False
|
|
1505
|
+
|
|
1506
|
+
|
|
1507
|
+
@dataclass(frozen=True)
|
|
1508
|
+
class _CodexPruneScope:
|
|
1509
|
+
"""Filesystem evidence that may authorize one whole-tree orphan prune.
|
|
1510
|
+
|
|
1511
|
+
A configured root is only *recognized* after a bounded probe finds at least
|
|
1512
|
+
one known Codex record envelope under it. A missing, empty, or unrelated
|
|
1513
|
+
directory therefore remains protected. When a current root is recognized,
|
|
1514
|
+
roots no longer present in the configured set may still be removed — that
|
|
1515
|
+
preserves issue #108's deliberate A -> B cache scoping. Within a recognized
|
|
1516
|
+
root, a missing file is deletion evidence only while its parent directory is
|
|
1517
|
+
still readable; losing an entire mounted subtree fails closed.
|
|
1518
|
+
"""
|
|
1519
|
+
|
|
1520
|
+
configured_root_keys: frozenset[str]
|
|
1521
|
+
recognized_root_keys: frozenset[str]
|
|
1522
|
+
|
|
1523
|
+
def refusal_reason(self, source_path: str, root_key: str | None) -> str | None:
|
|
1524
|
+
if not self.recognized_root_keys:
|
|
1525
|
+
return "no_recognizable_roots"
|
|
1526
|
+
if (
|
|
1527
|
+
root_key in self.configured_root_keys
|
|
1528
|
+
and root_key not in self.recognized_root_keys
|
|
1529
|
+
):
|
|
1530
|
+
return "configured_root_unrecognizable"
|
|
1531
|
+
if root_key in self.recognized_root_keys:
|
|
1532
|
+
parent = pathlib.Path(source_path).parent
|
|
1533
|
+
try:
|
|
1534
|
+
if not parent.is_dir() or not os.access(parent, os.R_OK | os.X_OK):
|
|
1535
|
+
return "source_parent_unavailable"
|
|
1536
|
+
except OSError:
|
|
1537
|
+
return "source_parent_unavailable"
|
|
1538
|
+
return None
|
|
1539
|
+
|
|
1540
|
+
def root_refusal_reason(self, root_key: str) -> str | None:
|
|
1541
|
+
if not self.recognized_root_keys:
|
|
1542
|
+
return "no_recognizable_roots"
|
|
1543
|
+
if (
|
|
1544
|
+
root_key in self.configured_root_keys
|
|
1545
|
+
and root_key not in self.recognized_root_keys
|
|
1546
|
+
):
|
|
1547
|
+
return "configured_root_unrecognizable"
|
|
1548
|
+
return None
|
|
1549
|
+
|
|
1550
|
+
|
|
1551
|
+
def _codex_prune_scope(files: list[CodexDiscoveredFile]) -> _CodexPruneScope:
|
|
1552
|
+
configured: set[str] = set()
|
|
1553
|
+
for raw_root in _cctally()._codex_home_roots():
|
|
1554
|
+
try:
|
|
1555
|
+
provider_root = _canonical_codex_path(raw_root)
|
|
1556
|
+
configured.add(source_root_key(str(provider_root)))
|
|
1557
|
+
except (OSError, TypeError, ValueError):
|
|
1558
|
+
continue
|
|
1559
|
+
recognized: set[str] = set()
|
|
1560
|
+
checked_per_root: dict[str, int] = {}
|
|
1561
|
+
# Discovery is stable oldest-first for Codex's dated rollout paths. Probe
|
|
1562
|
+
# from the tail so a bounded check sees the current envelope generation,
|
|
1563
|
+
# rather than spending its entire budget on a large legacy prefix. A root
|
|
1564
|
+
# containing only legacy/unknown envelopes still fails closed.
|
|
1565
|
+
for item in reversed(files):
|
|
1566
|
+
if item.source_root_key in recognized:
|
|
1567
|
+
continue
|
|
1568
|
+
checked = checked_per_root.get(item.source_root_key, 0)
|
|
1569
|
+
if checked >= _CODEX_ROOT_RECOGNITION_MAX_FILES:
|
|
1570
|
+
continue
|
|
1571
|
+
checked_per_root[item.source_root_key] = checked + 1
|
|
1572
|
+
if _codex_file_is_recognizable(item.source_path):
|
|
1573
|
+
recognized.add(item.source_root_key)
|
|
1574
|
+
return _CodexPruneScope(
|
|
1575
|
+
configured_root_keys=frozenset(configured),
|
|
1576
|
+
recognized_root_keys=frozenset(recognized),
|
|
1577
|
+
)
|
|
1578
|
+
|
|
1579
|
+
|
|
1580
|
+
def _partition_codex_prune_candidates(
|
|
1581
|
+
scope: _CodexPruneScope,
|
|
1582
|
+
sources: list[tuple[str, str | None]],
|
|
1583
|
+
root_keys: set[str],
|
|
1584
|
+
) -> tuple[
|
|
1585
|
+
list[tuple[str, str | None]], list[tuple[str, str | None]], set[str], set[str]
|
|
1586
|
+
]:
|
|
1587
|
+
safe_sources: list[tuple[str, str | None]] = []
|
|
1588
|
+
refused_sources: list[tuple[str, str | None]] = []
|
|
1589
|
+
for source in sources:
|
|
1590
|
+
target = refused_sources if scope.refusal_reason(*source) else safe_sources
|
|
1591
|
+
target.append(source)
|
|
1592
|
+
safe_roots: set[str] = set()
|
|
1593
|
+
refused_roots: set[str] = set()
|
|
1594
|
+
for root_key in root_keys:
|
|
1595
|
+
target = refused_roots if scope.root_refusal_reason(root_key) else safe_roots
|
|
1596
|
+
target.add(root_key)
|
|
1597
|
+
return safe_sources, refused_sources, safe_roots, refused_roots
|
|
1598
|
+
|
|
1599
|
+
|
|
1600
|
+
def _codex_prune_refusal_record(
|
|
1601
|
+
conn: sqlite3.Connection,
|
|
1602
|
+
*,
|
|
1603
|
+
store: str,
|
|
1604
|
+
scope: _CodexPruneScope,
|
|
1605
|
+
refused_sources: list[tuple[str, str | None]],
|
|
1606
|
+
refused_root_keys: set[str],
|
|
1607
|
+
) -> None:
|
|
1608
|
+
"""Persist a privacy-safe refusal signal in the store that declined."""
|
|
1609
|
+
now = (
|
|
1610
|
+
_cctally_core._command_as_of()
|
|
1611
|
+
.astimezone(dt.timezone.utc)
|
|
1612
|
+
.isoformat(timespec="seconds")
|
|
1613
|
+
.replace("+00:00", "Z")
|
|
1614
|
+
)
|
|
1615
|
+
since = now
|
|
1616
|
+
row = conn.execute(
|
|
1617
|
+
"SELECT value FROM cache_meta WHERE key=?",
|
|
1618
|
+
(CODEX_ORPHAN_PRUNE_REFUSED_KEY,),
|
|
1619
|
+
).fetchone()
|
|
1620
|
+
if row and row[0]:
|
|
1621
|
+
try:
|
|
1622
|
+
previous = json.loads(row[0])
|
|
1623
|
+
if isinstance(previous, dict) and isinstance(previous.get("since"), str):
|
|
1624
|
+
since = previous["since"]
|
|
1625
|
+
except (TypeError, ValueError):
|
|
1626
|
+
pass
|
|
1627
|
+
reasons = sorted({
|
|
1628
|
+
reason
|
|
1629
|
+
for path, root_key in refused_sources
|
|
1630
|
+
if (reason := scope.refusal_reason(path, root_key)) is not None
|
|
1631
|
+
} | {
|
|
1632
|
+
reason
|
|
1633
|
+
for root_key in refused_root_keys
|
|
1634
|
+
if (reason := scope.root_refusal_reason(root_key)) is not None
|
|
1635
|
+
})
|
|
1636
|
+
record = {
|
|
1637
|
+
"schemaVersion": 1,
|
|
1638
|
+
"store": store,
|
|
1639
|
+
"since": since,
|
|
1640
|
+
"at": now,
|
|
1641
|
+
"reasons": reasons,
|
|
1642
|
+
"configuredRootCount": len(scope.configured_root_keys),
|
|
1643
|
+
"recognizedRootCount": len(scope.recognized_root_keys),
|
|
1644
|
+
"preservedFileCount": len({path for path, _root in refused_sources}),
|
|
1645
|
+
"preservedRootCount": len(refused_root_keys),
|
|
1646
|
+
}
|
|
1647
|
+
conn.execute(
|
|
1648
|
+
"INSERT INTO cache_meta(key,value) VALUES(?,?) "
|
|
1649
|
+
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
1650
|
+
(CODEX_ORPHAN_PRUNE_REFUSED_KEY, json.dumps(record, sort_keys=True)),
|
|
1651
|
+
)
|
|
1652
|
+
eprint(
|
|
1653
|
+
f"[codex-{store}] orphan prune refused: preserved "
|
|
1654
|
+
f"{record['preservedFileCount']} tracked file(s); "
|
|
1655
|
+
"configured Codex roots were not sufficient deletion evidence"
|
|
1656
|
+
)
|
|
1657
|
+
|
|
1658
|
+
|
|
1659
|
+
def _clear_codex_prune_refusal(conn: sqlite3.Connection) -> None:
|
|
1660
|
+
conn.execute(
|
|
1661
|
+
"DELETE FROM cache_meta WHERE key=?", (CODEX_ORPHAN_PRUNE_REFUSED_KEY,)
|
|
1662
|
+
)
|
|
1663
|
+
|
|
1664
|
+
|
|
1441
1665
|
def _qualify_codex_targets(only_paths: "set[str]") -> list[CodexDiscoveredFile]:
|
|
1442
1666
|
"""Resolve each requested path through the ordered configured roots exactly
|
|
1443
1667
|
as full discovery would (spec §5.1) — producing the same per-file facts a
|
|
@@ -1603,7 +1827,12 @@ def _clear_codex_derived_rows(conn: sqlite3.Connection) -> bool:
|
|
|
1603
1827
|
|
|
1604
1828
|
|
|
1605
1829
|
def _bump_codex_physical_mutation_seq(conn: sqlite3.Connection) -> None:
|
|
1606
|
-
"""Advance the
|
|
1830
|
+
"""Advance the shared Codex physical-state invalidation token in this txn.
|
|
1831
|
+
|
|
1832
|
+
Dashboard/TUI snapshot versions and the quota projection certificate both
|
|
1833
|
+
consume this value. Every writer that changes their cache.db inputs must
|
|
1834
|
+
advance it in the same transaction as that change.
|
|
1835
|
+
"""
|
|
1607
1836
|
conn.execute(
|
|
1608
1837
|
"INSERT INTO cache_meta(key, value) VALUES "
|
|
1609
1838
|
"('codex_physical_mutation_seq', '1') "
|
|
@@ -1622,6 +1851,10 @@ _CODEX_MSG_INSERT_SQL = (
|
|
|
1622
1851
|
"INSERT OR IGNORE INTO codex_conversation_messages (" + _CODEX_NORM_COLS + ") "
|
|
1623
1852
|
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
|
1624
1853
|
)
|
|
1854
|
+
_CODEX_ACCOUNT_MSG_INSERT_SQL = (
|
|
1855
|
+
"INSERT OR IGNORE INTO codex_conversation_messages (" + _CODEX_NORM_COLS
|
|
1856
|
+
+ ", account_key) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
|
1857
|
+
)
|
|
1625
1858
|
|
|
1626
1859
|
|
|
1627
1860
|
def _codex_conversation_project_attribution(
|
|
@@ -1679,17 +1912,29 @@ def _load_codex_normalized_rows(
|
|
|
1679
1912
|
]
|
|
1680
1913
|
|
|
1681
1914
|
|
|
1682
|
-
def _insert_codex_normalized_rows(
|
|
1915
|
+
def _insert_codex_normalized_rows(
|
|
1916
|
+
conn: sqlite3.Connection,
|
|
1917
|
+
rows: list,
|
|
1918
|
+
touches: list,
|
|
1919
|
+
account_by_physical: "dict[tuple[str, int], str | None] | None" = None,
|
|
1920
|
+
) -> None:
|
|
1683
1921
|
"""Insert normalized rows (INSERT OR IGNORE on the physical key) + their file
|
|
1684
1922
|
touches (message linkage resolved via (source_path, line_offset))."""
|
|
1685
1923
|
if rows:
|
|
1686
|
-
|
|
1924
|
+
with_accounts = account_by_physical is not None
|
|
1925
|
+
account_by_physical = account_by_physical or {}
|
|
1926
|
+
conn.executemany(
|
|
1927
|
+
_CODEX_ACCOUNT_MSG_INSERT_SQL if with_accounts else _CODEX_MSG_INSERT_SQL,
|
|
1928
|
+
[
|
|
1687
1929
|
(r.conversation_key, r.source_root_key, r.source_path, r.line_offset,
|
|
1688
1930
|
r.timestamp_utc, r.turn_id, r.call_id, r.kind, r.event_type,
|
|
1689
1931
|
r.record_family, r.model, r.text, r.content_digest, r.content_len,
|
|
1690
1932
|
r.detail_json, r.search_tool, r.search_thinking)
|
|
1933
|
+
+ ((account_by_physical.get((r.source_path, r.line_offset)),)
|
|
1934
|
+
if with_accounts else ())
|
|
1691
1935
|
for r in rows
|
|
1692
|
-
|
|
1936
|
+
],
|
|
1937
|
+
)
|
|
1693
1938
|
for touch in touches:
|
|
1694
1939
|
conn.execute(
|
|
1695
1940
|
"INSERT OR IGNORE INTO codex_conversation_file_touches "
|
|
@@ -1764,26 +2009,120 @@ def _replay_codex_normalization(conn: sqlite3.Connection) -> None:
|
|
|
1764
2009
|
the plain rowid alias make a re-run byte-idempotent."""
|
|
1765
2010
|
kern = _lib_codex_conversation
|
|
1766
2011
|
events_by_file: dict[str, list] = {}
|
|
2012
|
+
accounts_by_file: dict[str, dict[tuple[str, int], str | None]] = {}
|
|
1767
2013
|
order: list[str] = []
|
|
2014
|
+
has_account = "account_key" in {
|
|
2015
|
+
str(info[1])
|
|
2016
|
+
for info in conn.execute("PRAGMA table_info(codex_conversation_events)")
|
|
2017
|
+
}
|
|
2018
|
+
account_select = ", account_key" if has_account else ""
|
|
1768
2019
|
for row in conn.execute(
|
|
1769
2020
|
"SELECT source_path, line_offset, source_root_key, conversation_key, "
|
|
1770
2021
|
"native_thread_id, root_thread_id, parent_thread_id, timestamp_utc, "
|
|
1771
2022
|
"record_type, event_type, turn_id, call_id, payload_json "
|
|
2023
|
+
+ account_select + " "
|
|
1772
2024
|
"FROM codex_conversation_events "
|
|
1773
2025
|
"ORDER BY source_path ASC, line_offset ASC"
|
|
1774
2026
|
):
|
|
1775
|
-
event = _lib_jsonl.CodexPhysicalEvent(*row)
|
|
2027
|
+
event = _lib_jsonl.CodexPhysicalEvent(*(row[:-1] if has_account else row))
|
|
1776
2028
|
if event.source_path not in events_by_file:
|
|
1777
2029
|
events_by_file[event.source_path] = []
|
|
2030
|
+
accounts_by_file[event.source_path] = {}
|
|
1778
2031
|
order.append(event.source_path)
|
|
1779
2032
|
events_by_file[event.source_path].append(event)
|
|
2033
|
+
if has_account:
|
|
2034
|
+
accounts_by_file[event.source_path][
|
|
2035
|
+
(event.source_path, int(event.line_offset))
|
|
2036
|
+
] = row[-1]
|
|
1780
2037
|
affected: set = set()
|
|
1781
2038
|
for source_path in order:
|
|
1782
2039
|
result = kern.normalize_codex_events(
|
|
1783
2040
|
events_by_file[source_path], initial=kern.CodexStickyState())
|
|
1784
|
-
_insert_codex_normalized_rows(
|
|
2041
|
+
_insert_codex_normalized_rows(
|
|
2042
|
+
conn, result.rows, result.touches,
|
|
2043
|
+
accounts_by_file[source_path] if has_account else None,
|
|
2044
|
+
)
|
|
1785
2045
|
affected.update(r.conversation_key for r in result.rows)
|
|
1786
2046
|
_recompute_codex_rollups(conn, affected)
|
|
2047
|
+
# Migration 025 still replays this helper against legacy cache.db fixtures,
|
|
2048
|
+
# where the conversations-only #482 projection table does not exist. Live
|
|
2049
|
+
# conversations.db replays must refresh the projection, but the historical
|
|
2050
|
+
# cache migration must remain byte-stable and cannot create a newer store's
|
|
2051
|
+
# derivation out of order.
|
|
2052
|
+
projection_exists = conn.execute(
|
|
2053
|
+
"SELECT 1 FROM sqlite_master "
|
|
2054
|
+
"WHERE type='table' AND name='codex_find_projection'"
|
|
2055
|
+
).fetchone() is not None
|
|
2056
|
+
if projection_exists:
|
|
2057
|
+
import _lib_codex_conversation_query as query
|
|
2058
|
+
query.materialize_codex_find_projection(conn, affected)
|
|
2059
|
+
|
|
2060
|
+
|
|
2061
|
+
def run_codex_find_projection_backfill(
|
|
2062
|
+
conn: sqlite3.Connection,
|
|
2063
|
+
*,
|
|
2064
|
+
batch_size: int = 400,
|
|
2065
|
+
) -> dict[str, object]:
|
|
2066
|
+
"""Consume one resumable #482 projection batch from retained rows.
|
|
2067
|
+
|
|
2068
|
+
The caller owns the Codex conversations provider flock. Progress is keyed
|
|
2069
|
+
by the normalized message rowid, but a selected conversation is always
|
|
2070
|
+
rebuilt as a whole so native folds never straddle a batch boundary.
|
|
2071
|
+
"""
|
|
2072
|
+
pending = conn.execute(
|
|
2073
|
+
"SELECT 1 FROM cache_meta "
|
|
2074
|
+
"WHERE key='codex_find_projection_backfill_pending'"
|
|
2075
|
+
).fetchone()
|
|
2076
|
+
if pending is None:
|
|
2077
|
+
complete = conn.execute(
|
|
2078
|
+
"SELECT 1 FROM cache_meta "
|
|
2079
|
+
"WHERE key='codex_find_projection_complete_version' AND value='1'"
|
|
2080
|
+
).fetchone() is not None
|
|
2081
|
+
return {"processed": 0, "complete": complete}
|
|
2082
|
+
row = conn.execute(
|
|
2083
|
+
"SELECT value FROM cache_meta "
|
|
2084
|
+
"WHERE key='codex_find_projection_backfill_cursor'"
|
|
2085
|
+
).fetchone()
|
|
2086
|
+
try:
|
|
2087
|
+
cursor = int(row[0]) if row is not None else 0
|
|
2088
|
+
except (TypeError, ValueError):
|
|
2089
|
+
cursor = 0
|
|
2090
|
+
selected_rows = conn.execute(
|
|
2091
|
+
"SELECT id,conversation_key FROM codex_conversation_messages "
|
|
2092
|
+
"WHERE id>? ORDER BY id LIMIT ?",
|
|
2093
|
+
(cursor, max(1, batch_size)),
|
|
2094
|
+
).fetchall()
|
|
2095
|
+
selected = {conversation_key for _message_id, conversation_key in selected_rows}
|
|
2096
|
+
if selected_rows:
|
|
2097
|
+
import _lib_codex_conversation_query as query
|
|
2098
|
+
query.materialize_codex_find_projection(conn, selected)
|
|
2099
|
+
# Advance only over raw rows actually consumed by this batch. A
|
|
2100
|
+
# conversation may own later, interleaved row ids; jumping to its MAX
|
|
2101
|
+
# would skip another conversation whose first row lies in between and
|
|
2102
|
+
# could falsely certify the projection complete.
|
|
2103
|
+
cursor = int(selected_rows[-1][0])
|
|
2104
|
+
conn.execute(
|
|
2105
|
+
"INSERT OR REPLACE INTO cache_meta(key,value) VALUES"
|
|
2106
|
+
"('codex_find_projection_backfill_cursor',?)",
|
|
2107
|
+
(str(cursor),),
|
|
2108
|
+
)
|
|
2109
|
+
remaining = conn.execute(
|
|
2110
|
+
"SELECT 1 FROM codex_conversation_messages WHERE id>? LIMIT 1",
|
|
2111
|
+
(cursor,),
|
|
2112
|
+
).fetchone()
|
|
2113
|
+
complete = remaining is None
|
|
2114
|
+
if complete:
|
|
2115
|
+
conn.execute(
|
|
2116
|
+
"INSERT OR REPLACE INTO cache_meta(key,value) VALUES"
|
|
2117
|
+
"('codex_find_projection_complete_version','1')"
|
|
2118
|
+
)
|
|
2119
|
+
conn.execute(
|
|
2120
|
+
"DELETE FROM cache_meta WHERE key IN "
|
|
2121
|
+
"('codex_find_projection_backfill_pending',"
|
|
2122
|
+
" 'codex_find_projection_backfill_cursor')"
|
|
2123
|
+
)
|
|
2124
|
+
conn.commit()
|
|
2125
|
+
return {"processed": len(selected), "complete": complete}
|
|
1787
2126
|
|
|
1788
2127
|
|
|
1789
2128
|
def _repair_codex_turn_ids_for_source(
|
|
@@ -1832,27 +2171,18 @@ def _repair_codex_turn_ids_for_source(
|
|
|
1832
2171
|
return affected
|
|
1833
2172
|
|
|
1834
2173
|
|
|
1835
|
-
def
|
|
2174
|
+
def _collect_retained_codex_paths_and_roots(
|
|
1836
2175
|
conn: sqlite3.Connection,
|
|
1837
|
-
current_file_identities: set[tuple[str, str]],
|
|
1838
|
-
active_root_keys: set[str],
|
|
1839
2176
|
) -> tuple[list[tuple[str, str | None]], set[str]]:
|
|
1840
|
-
"""Return
|
|
2177
|
+
"""Return every retained real source identity and provider root.
|
|
1841
2178
|
|
|
1842
2179
|
A failed/partial prior write can leave any S1 child family without its
|
|
1843
|
-
terminal ``codex_session_files`` row.
|
|
1844
|
-
every physical family,
|
|
1845
|
-
|
|
2180
|
+
terminal ``codex_session_files`` row. Safety decisions must therefore see
|
|
2181
|
+
every physical family, including a still-discovered path whose root no
|
|
2182
|
+
longer contains recognizable Codex data. Relative fixture rows have no
|
|
2183
|
+
on-disk authority and remain outside filesystem pruning.
|
|
1846
2184
|
"""
|
|
1847
|
-
|
|
1848
|
-
stale_root_keys: set[str] = set()
|
|
1849
|
-
current_paths = {path for path, _root_key in current_file_identities}
|
|
1850
|
-
terminal_file_identities = {
|
|
1851
|
-
(path, root_key)
|
|
1852
|
-
for path, root_key in conn.execute(
|
|
1853
|
-
"SELECT path, source_root_key FROM codex_session_files"
|
|
1854
|
-
)
|
|
1855
|
-
}
|
|
2185
|
+
retained_identities: set[tuple[str, str | None]] = set()
|
|
1856
2186
|
family_queries = (
|
|
1857
2187
|
"SELECT path, source_root_key FROM codex_session_files",
|
|
1858
2188
|
"SELECT source_path, source_root_key FROM codex_session_entries",
|
|
@@ -1862,29 +2192,59 @@ def _collect_inactive_codex_paths_and_roots(
|
|
|
1862
2192
|
)
|
|
1863
2193
|
for query in family_queries:
|
|
1864
2194
|
for source_path, root_key in conn.execute(query):
|
|
1865
|
-
|
|
1866
|
-
if (
|
|
1867
|
-
not os.path.isabs(source_path)
|
|
1868
|
-
or identity in current_file_identities
|
|
1869
|
-
# An old terminal file at a currently discovered path must
|
|
1870
|
-
# reach the normal requalification loop, which resets every
|
|
1871
|
-
# family as one file transaction and records the reset stat.
|
|
1872
|
-
or (
|
|
1873
|
-
source_path in current_paths
|
|
1874
|
-
and identity in terminal_file_identities
|
|
1875
|
-
)
|
|
1876
|
-
):
|
|
2195
|
+
if not os.path.isabs(source_path):
|
|
1877
2196
|
continue
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
2197
|
+
retained_identities.add((source_path, root_key))
|
|
2198
|
+
retained_root_keys = {
|
|
2199
|
+
root_key for _path, root_key in retained_identities if root_key is not None
|
|
2200
|
+
}
|
|
2201
|
+
retained_root_keys.update(
|
|
1882
2202
|
root_key
|
|
1883
2203
|
for (root_key,) in conn.execute(
|
|
1884
2204
|
"SELECT source_root_key FROM codex_source_roots"
|
|
1885
2205
|
)
|
|
1886
|
-
if root_key not in active_root_keys
|
|
1887
2206
|
)
|
|
2207
|
+
return (
|
|
2208
|
+
sorted(retained_identities, key=lambda item: (item[0], item[1] or "")),
|
|
2209
|
+
retained_root_keys,
|
|
2210
|
+
)
|
|
2211
|
+
|
|
2212
|
+
|
|
2213
|
+
def _collect_inactive_codex_paths_and_roots(
|
|
2214
|
+
conn: sqlite3.Connection,
|
|
2215
|
+
current_file_identities: set[tuple[str, str]],
|
|
2216
|
+
active_root_keys: set[str],
|
|
2217
|
+
) -> tuple[list[tuple[str, str | None]], set[str]]:
|
|
2218
|
+
"""Return stale retained source identities and candidate root keys."""
|
|
2219
|
+
retained_identities, retained_root_keys = (
|
|
2220
|
+
_collect_retained_codex_paths_and_roots(conn)
|
|
2221
|
+
)
|
|
2222
|
+
current_paths = {path for path, _root_key in current_file_identities}
|
|
2223
|
+
terminal_file_identities = {
|
|
2224
|
+
(path, root_key)
|
|
2225
|
+
for path, root_key in conn.execute(
|
|
2226
|
+
"SELECT path, source_root_key FROM codex_session_files"
|
|
2227
|
+
)
|
|
2228
|
+
}
|
|
2229
|
+
stale_identities = {
|
|
2230
|
+
identity
|
|
2231
|
+
for identity in retained_identities
|
|
2232
|
+
if (
|
|
2233
|
+
identity not in current_file_identities
|
|
2234
|
+
# An old terminal file at a currently discovered path must reach
|
|
2235
|
+
# the normal requalification loop. The caller separately protects
|
|
2236
|
+
# that loop when the configured root is unrecognizable.
|
|
2237
|
+
and not (
|
|
2238
|
+
identity[0] in current_paths
|
|
2239
|
+
and identity in terminal_file_identities
|
|
2240
|
+
)
|
|
2241
|
+
)
|
|
2242
|
+
}
|
|
2243
|
+
stale_root_keys = {
|
|
2244
|
+
root_key
|
|
2245
|
+
for root_key in retained_root_keys
|
|
2246
|
+
if root_key not in active_root_keys
|
|
2247
|
+
}
|
|
1888
2248
|
return sorted(stale_identities, key=lambda item: (item[0], item[1] or "")), stale_root_keys
|
|
1889
2249
|
|
|
1890
2250
|
|
|
@@ -3793,7 +4153,9 @@ def _reingest_parse_file(jp, path_str):
|
|
|
3793
4153
|
return rows
|
|
3794
4154
|
|
|
3795
4155
|
|
|
3796
|
-
def _resumable_reingest_conversation_messages(
|
|
4156
|
+
def _resumable_reingest_conversation_messages(
|
|
4157
|
+
conn, active_account_key: "str | None" = None,
|
|
4158
|
+
):
|
|
3797
4159
|
"""#179: resumable, lock-friendly replacement for the old global
|
|
3798
4160
|
``clear_conversation_messages`` + offset-0 ``backfill_conversation_messages``
|
|
3799
4161
|
reingest, which re-armed the whole ~2.5min rebuild on any interrupt. Walks
|
|
@@ -3841,6 +4203,30 @@ def _resumable_reingest_conversation_messages(conn):
|
|
|
3841
4203
|
conn.commit()
|
|
3842
4204
|
continue
|
|
3843
4205
|
try:
|
|
4206
|
+
# #347 enrichment replays must not erase first-stamped transcript
|
|
4207
|
+
# ownership. Snapshot by physical offset before the destructive
|
|
4208
|
+
# per-source replace and carry it onto the re-parsed row.
|
|
4209
|
+
account_capable = "account_key" in {
|
|
4210
|
+
str(info[1])
|
|
4211
|
+
for info in conn.execute("PRAGMA table_info(conversation_messages)")
|
|
4212
|
+
}
|
|
4213
|
+
account_by_offset = {
|
|
4214
|
+
int(offset): account_key
|
|
4215
|
+
for offset, account_key in conn.execute(
|
|
4216
|
+
"SELECT byte_offset,account_key FROM conversation_messages "
|
|
4217
|
+
"WHERE source_path=?",
|
|
4218
|
+
(path_str,),
|
|
4219
|
+
)
|
|
4220
|
+
} if account_capable else {}
|
|
4221
|
+
if account_capable:
|
|
4222
|
+
rows = [
|
|
4223
|
+
row + (
|
|
4224
|
+
account_by_offset[int(row[4])]
|
|
4225
|
+
if int(row[4]) in account_by_offset
|
|
4226
|
+
else active_account_key,
|
|
4227
|
+
)
|
|
4228
|
+
for row in rows
|
|
4229
|
+
]
|
|
3844
4230
|
# #217 S2 / I-3 (P1-4): conversation_file_touches is derived state
|
|
3845
4231
|
# keyed by conversation_messages.id, and this per-source reingest
|
|
3846
4232
|
# DELETEs + re-inserts the file's message rows (bumping autoincrement
|
|
@@ -3855,7 +4241,11 @@ def _resumable_reingest_conversation_messages(conn):
|
|
|
3855
4241
|
conn.execute("DELETE FROM conversation_messages WHERE source_path=?",
|
|
3856
4242
|
(path_str,))
|
|
3857
4243
|
if rows:
|
|
3858
|
-
conn.executemany(
|
|
4244
|
+
conn.executemany(
|
|
4245
|
+
_CONV_ACCOUNT_INSERT_SQL if account_capable
|
|
4246
|
+
else _CONV_INSERT_SQL,
|
|
4247
|
+
rows,
|
|
4248
|
+
)
|
|
3859
4249
|
# Refill scoped to this file's just-reinserted physical keys
|
|
3860
4250
|
# (col 3=source_path, col 4=byte_offset per _conv_row_tuple).
|
|
3861
4251
|
_fill_file_touches(
|
|
@@ -4860,6 +5250,12 @@ class CodexIngestStats:
|
|
|
4860
5250
|
# Count of cached files dropped because they fall outside the CURRENT
|
|
4861
5251
|
# $CODEX_HOME root set (issue #108 — a prior-root purge, not a delta).
|
|
4862
5252
|
files_pruned: int = 0
|
|
5253
|
+
# #485: a whole-tree walk may discover no trustworthy current Codex root.
|
|
5254
|
+
# In that state absence is not deletion evidence, so ordinary orphan
|
|
5255
|
+
# pruning is refused and retained rows stay untouched. These fields make
|
|
5256
|
+
# the destructive decision directly observable to callers and tests.
|
|
5257
|
+
prune_refused: bool = False
|
|
5258
|
+
prune_refused_files: int = 0
|
|
4863
5259
|
lock_contended: bool = False
|
|
4864
5260
|
# #279 S2 F1 parse-health counters — folded from each file's
|
|
4865
5261
|
# _CodexIterState after its drain. Same vocabulary as the iterator
|
|
@@ -5456,6 +5852,7 @@ def sync_codex_cache(
|
|
|
5456
5852
|
_cache_root_keys,
|
|
5457
5853
|
)
|
|
5458
5854
|
seq_before = codex_physical_mutation_seq(conn)
|
|
5855
|
+
rebuild_prune_scope: _CodexPruneScope | None = None
|
|
5459
5856
|
|
|
5460
5857
|
# #416 spec D1: which rollouts were ALREADY ingested before this rebuild
|
|
5461
5858
|
# cleared the cursor table. A rebuild re-reads their bytes, and bytes
|
|
@@ -5485,6 +5882,46 @@ def sync_codex_cache(
|
|
|
5485
5882
|
str(_canonical_codex_path(pathlib.Path(str(_path))))))
|
|
5486
5883
|
except (ValueError, TypeError, OSError):
|
|
5487
5884
|
continue
|
|
5885
|
+
if rebuild and not targeted:
|
|
5886
|
+
# #485: rebuild used to clear first and discover second. With an
|
|
5887
|
+
# empty/missing/misconfigured root that made the explicit repair
|
|
5888
|
+
# command itself destroy the only retained copy. Discover before
|
|
5889
|
+
# the clear and require the same positive root evidence as ordinary
|
|
5890
|
+
# pruning whenever real absolute Codex rows already exist.
|
|
5891
|
+
preflight_files = _discover_codex_files_with_roots()
|
|
5892
|
+
preflight_scope = _codex_prune_scope(preflight_files)
|
|
5893
|
+
rebuild_prune_scope = preflight_scope
|
|
5894
|
+
preflight_sources, _preflight_root_keys = (
|
|
5895
|
+
_collect_retained_codex_paths_and_roots(conn)
|
|
5896
|
+
)
|
|
5897
|
+
preflight_source_root_keys = {
|
|
5898
|
+
root_key
|
|
5899
|
+
for _path, root_key in preflight_sources
|
|
5900
|
+
if root_key is not None
|
|
5901
|
+
}
|
|
5902
|
+
(
|
|
5903
|
+
_safe_sources,
|
|
5904
|
+
refused_sources,
|
|
5905
|
+
_safe_roots,
|
|
5906
|
+
refused_root_keys,
|
|
5907
|
+
) = _partition_codex_prune_candidates(
|
|
5908
|
+
preflight_scope, preflight_sources, preflight_source_root_keys
|
|
5909
|
+
)
|
|
5910
|
+
if refused_sources or refused_root_keys:
|
|
5911
|
+
_codex_prune_refusal_record(
|
|
5912
|
+
conn,
|
|
5913
|
+
store="cache",
|
|
5914
|
+
scope=preflight_scope,
|
|
5915
|
+
refused_sources=refused_sources,
|
|
5916
|
+
refused_root_keys=refused_root_keys,
|
|
5917
|
+
)
|
|
5918
|
+
conn.commit()
|
|
5919
|
+
stats.files_total = len(preflight_files)
|
|
5920
|
+
stats.prune_refused = True
|
|
5921
|
+
stats.prune_refused_files = len({
|
|
5922
|
+
path for path, _root in refused_sources
|
|
5923
|
+
})
|
|
5924
|
+
return stats
|
|
5488
5925
|
if rebuild:
|
|
5489
5926
|
# Clear INSIDE the lock — see sync_cache() for the full
|
|
5490
5927
|
# rationale. Done before the existing SELECT so delta
|
|
@@ -5590,19 +6027,13 @@ def sync_codex_cache(
|
|
|
5590
6027
|
stats.files_total = len(files)
|
|
5591
6028
|
_p_disc.set_count(len(files))
|
|
5592
6029
|
|
|
5593
|
-
# Scope the cache to the CURRENT root set
|
|
5594
|
-
#
|
|
5595
|
-
#
|
|
5596
|
-
#
|
|
5597
|
-
#
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
# re-derivable, so honoring the override beats retaining unreachable
|
|
5601
|
-
# rows. Done INSIDE the lock and committed BEFORE the existing-SELECT
|
|
5602
|
-
# + parse loop so no cache.db write lock is held across the read-heavy
|
|
5603
|
-
# ingest (same invariant as the --rebuild clear above). Concurrent
|
|
5604
|
-
# processes with different $CODEX_HOME would prune each other; the
|
|
5605
|
-
# flock serializes them and that is a pathological configuration.
|
|
6030
|
+
# Scope the cache to the CURRENT root set (issue #108), but only from
|
|
6031
|
+
# positive filesystem evidence (#485). A recognized new root still
|
|
6032
|
+
# authorizes the deliberate A -> B scope switch. No recognized root,
|
|
6033
|
+
# or one configured-but-unrecognizable sibling, is unknown filesystem
|
|
6034
|
+
# state rather than proof of deletion and therefore fails closed.
|
|
6035
|
+
protected_paths: set[str] = set()
|
|
6036
|
+
ordinary_prune_scope: _CodexPruneScope | None = None
|
|
5606
6037
|
if not rebuild and not targeted: # --rebuild already cleared; targeted bypasses
|
|
5607
6038
|
current_file_identities = {
|
|
5608
6039
|
(str(item.source_path), item.source_root_key) for item in files
|
|
@@ -5619,12 +6050,39 @@ def sync_codex_cache(
|
|
|
5619
6050
|
orphan_sources, orphan_root_keys = _collect_inactive_codex_paths_and_roots(
|
|
5620
6051
|
conn, current_file_identities, active_root_keys,
|
|
5621
6052
|
)
|
|
5622
|
-
|
|
6053
|
+
prune_scope = _codex_prune_scope(files)
|
|
6054
|
+
ordinary_prune_scope = prune_scope
|
|
6055
|
+
(
|
|
6056
|
+
safe_sources,
|
|
6057
|
+
_refused_orphan_sources,
|
|
6058
|
+
safe_root_keys,
|
|
6059
|
+
_refused_orphan_root_keys,
|
|
6060
|
+
) = _partition_codex_prune_candidates(
|
|
6061
|
+
prune_scope, orphan_sources, orphan_root_keys
|
|
6062
|
+
)
|
|
6063
|
+
retained_sources, _retained_root_keys = (
|
|
6064
|
+
_collect_retained_codex_paths_and_roots(conn)
|
|
6065
|
+
)
|
|
6066
|
+
retained_source_root_keys = {
|
|
6067
|
+
root_key
|
|
6068
|
+
for _path, root_key in retained_sources
|
|
6069
|
+
if root_key is not None
|
|
6070
|
+
}
|
|
6071
|
+
(
|
|
6072
|
+
_safe_retained_sources,
|
|
6073
|
+
refused_sources,
|
|
6074
|
+
_safe_retained_roots,
|
|
6075
|
+
refused_root_keys,
|
|
6076
|
+
) = _partition_codex_prune_candidates(
|
|
6077
|
+
prune_scope, retained_sources, retained_source_root_keys
|
|
6078
|
+
)
|
|
6079
|
+
protected_paths = {path for path, _root in refused_sources}
|
|
6080
|
+
if safe_sources or safe_root_keys:
|
|
5623
6081
|
before_prune = conn.total_changes
|
|
5624
6082
|
# #294 S6: capture the conversation keys the orphan rows belong to
|
|
5625
6083
|
# BEFORE deleting, so the rollups can be repaired/deleted after.
|
|
5626
6084
|
orphan_keys: set = set()
|
|
5627
|
-
for orphan_path, orphan_root_key in
|
|
6085
|
+
for orphan_path, orphan_root_key in safe_sources:
|
|
5628
6086
|
orphan_keys.update(
|
|
5629
6087
|
row[0] for row in conn.execute(
|
|
5630
6088
|
"SELECT DISTINCT conversation_key FROM codex_conversation_messages "
|
|
@@ -5637,7 +6095,7 @@ def sync_codex_cache(
|
|
|
5637
6095
|
)
|
|
5638
6096
|
_prune_inactive_codex_source_roots(
|
|
5639
6097
|
conn, active_root_keys,
|
|
5640
|
-
candidate_root_keys=
|
|
6098
|
+
candidate_root_keys=safe_root_keys,
|
|
5641
6099
|
)
|
|
5642
6100
|
# Recompute-affected-or-delete the rollups the prune touched (§3.2):
|
|
5643
6101
|
# a conversation with no surviving rows loses its rollup, one that
|
|
@@ -5645,8 +6103,24 @@ def sync_codex_cache(
|
|
|
5645
6103
|
_recompute_codex_rollups(conn, orphan_keys)
|
|
5646
6104
|
if conn.total_changes != before_prune:
|
|
5647
6105
|
_bump_codex_physical_mutation_seq(conn)
|
|
6106
|
+
stats.files_pruned = len({path for path, _root in safe_sources})
|
|
6107
|
+
if refused_sources or refused_root_keys:
|
|
6108
|
+
_codex_prune_refusal_record(
|
|
6109
|
+
conn,
|
|
6110
|
+
store="cache",
|
|
6111
|
+
scope=prune_scope,
|
|
6112
|
+
refused_sources=refused_sources,
|
|
6113
|
+
refused_root_keys=refused_root_keys,
|
|
6114
|
+
)
|
|
6115
|
+
stats.prune_refused = True
|
|
6116
|
+
stats.prune_refused_files = len({
|
|
6117
|
+
path for path, _root in refused_sources
|
|
6118
|
+
})
|
|
6119
|
+
if (
|
|
6120
|
+
safe_sources or safe_root_keys
|
|
6121
|
+
or refused_sources or refused_root_keys
|
|
6122
|
+
):
|
|
5648
6123
|
conn.commit()
|
|
5649
|
-
stats.files_pruned = len({path for path, _root in orphan_sources})
|
|
5650
6124
|
|
|
5651
6125
|
# This SELECT does NOT open an implicit transaction (Python's
|
|
5652
6126
|
# sqlite3 module only BEGINs on DML). Do NOT add any INSERT/
|
|
@@ -5808,6 +6282,12 @@ def sync_codex_cache(
|
|
|
5808
6282
|
break
|
|
5809
6283
|
jp = discovered.source_path
|
|
5810
6284
|
path_str = str(jp)
|
|
6285
|
+
if path_str in protected_paths:
|
|
6286
|
+
# The file is still visible, but its configured root did not
|
|
6287
|
+
# yield recognizable Codex data. Treat that as unknown
|
|
6288
|
+
# filesystem state: neither truncation/requalification reset
|
|
6289
|
+
# nor append ingestion may replace the retained generation.
|
|
6290
|
+
continue
|
|
5811
6291
|
try:
|
|
5812
6292
|
st = jp.stat()
|
|
5813
6293
|
except OSError as exc:
|
|
@@ -6543,6 +7023,17 @@ def sync_codex_cache(
|
|
|
6543
7023
|
and stats.files_failed == 0
|
|
6544
7024
|
and stats.files_deferred_torn == 0
|
|
6545
7025
|
):
|
|
7026
|
+
recovery_scope = rebuild_prune_scope or ordinary_prune_scope
|
|
7027
|
+
if (
|
|
7028
|
+
recovery_scope is not None
|
|
7029
|
+
and recovery_scope.recognized_root_keys
|
|
7030
|
+
and not stats.prune_refused
|
|
7031
|
+
and (
|
|
7032
|
+
stats.files_processed + stats.files_skipped_unchanged
|
|
7033
|
+
== stats.files_total
|
|
7034
|
+
)
|
|
7035
|
+
):
|
|
7036
|
+
_clear_codex_prune_refusal(conn)
|
|
6546
7037
|
if replay_pending:
|
|
6547
7038
|
conn.execute("DELETE FROM cache_meta WHERE key = ?",
|
|
6548
7039
|
(CODEX_REPLAY_FROM_ZERO_KEY,))
|
|
@@ -8095,6 +8586,126 @@ def open_conversations_db(*, attach_cache: bool = True) -> sqlite3.Connection:
|
|
|
8095
8586
|
return _conversations_open_guarded(attach_cache=attach_cache)
|
|
8096
8587
|
|
|
8097
8588
|
|
|
8589
|
+
def scope_conversations_db_to_account(
|
|
8590
|
+
conn: sqlite3.Connection, account_key: str,
|
|
8591
|
+
) -> None:
|
|
8592
|
+
"""Narrow one conversation read connection to ``account_key`` (#347).
|
|
8593
|
+
|
|
8594
|
+
The query kernels intentionally retain their byte-frozen unqualified SQL.
|
|
8595
|
+
A qualified request gets a fresh SQLite connection and this helper shadows
|
|
8596
|
+
every transcript/accounting leaf with TEMP views on that connection only.
|
|
8597
|
+
SQLite resolves TEMP before main/attached schemas, so browse, facets,
|
|
8598
|
+
search, detail, outline, export, payload, media, find, and cost/token joins
|
|
8599
|
+
all inherit the same physical-row predicate without a parallel query stack.
|
|
8600
|
+
|
|
8601
|
+
The two rollups are rebuilt into TEMP tables from the filtered leaves. No
|
|
8602
|
+
persistent row is changed and an unqualified connection takes none of this
|
|
8603
|
+
path, preserving the existing SQL plans and serialized bytes.
|
|
8604
|
+
"""
|
|
8605
|
+
import _lib_accounts
|
|
8606
|
+
|
|
8607
|
+
key = str(account_key or "").strip()
|
|
8608
|
+
if not key:
|
|
8609
|
+
raise ValueError("account_key is required")
|
|
8610
|
+
if key == _lib_accounts.UNATTRIBUTED:
|
|
8611
|
+
stored_predicate = "COALESCE(account_key,'unattributed')='unattributed'"
|
|
8612
|
+
else:
|
|
8613
|
+
# The value lives in a one-row TEMP table and reaches every view through
|
|
8614
|
+
# a subquery; never interpolate an opaque account key into DDL.
|
|
8615
|
+
stored_predicate = (
|
|
8616
|
+
"COALESCE(account_key,'unattributed')="
|
|
8617
|
+
"(SELECT account_key FROM _conversation_account_scope)"
|
|
8618
|
+
)
|
|
8619
|
+
|
|
8620
|
+
conn.execute(
|
|
8621
|
+
"CREATE TEMP TABLE _conversation_account_scope "
|
|
8622
|
+
"(account_key TEXT NOT NULL PRIMARY KEY)"
|
|
8623
|
+
)
|
|
8624
|
+
conn.execute(
|
|
8625
|
+
"INSERT INTO _conversation_account_scope(account_key) VALUES(?)", (key,)
|
|
8626
|
+
)
|
|
8627
|
+
|
|
8628
|
+
conn.executescript(
|
|
8629
|
+
f"""
|
|
8630
|
+
CREATE TEMP VIEW conversation_messages AS
|
|
8631
|
+
SELECT * FROM main.conversation_messages WHERE {stored_predicate};
|
|
8632
|
+
CREATE TEMP VIEW conversation_ai_titles AS
|
|
8633
|
+
SELECT t.* FROM main.conversation_ai_titles t
|
|
8634
|
+
WHERE EXISTS (
|
|
8635
|
+
SELECT 1 FROM conversation_messages m
|
|
8636
|
+
WHERE m.source_path=t.source_path
|
|
8637
|
+
AND m.byte_offset=t.byte_offset
|
|
8638
|
+
);
|
|
8639
|
+
CREATE TEMP VIEW conversation_file_touches AS
|
|
8640
|
+
SELECT t.* FROM main.conversation_file_touches t
|
|
8641
|
+
WHERE EXISTS (
|
|
8642
|
+
SELECT 1 FROM conversation_messages m WHERE m.id=t.message_id
|
|
8643
|
+
);
|
|
8644
|
+
CREATE TEMP VIEW session_entries AS
|
|
8645
|
+
SELECT * FROM cache_db.session_entries WHERE {stored_predicate};
|
|
8646
|
+
-- A file-level project path cannot be partitioned when one JSONL
|
|
8647
|
+
-- switches accounts. Scoped anonymization therefore uses only the
|
|
8648
|
+
-- already-filtered physical message CWDs.
|
|
8649
|
+
CREATE TEMP VIEW session_files AS
|
|
8650
|
+
SELECT * FROM cache_db.session_files WHERE 0;
|
|
8651
|
+
|
|
8652
|
+
CREATE TEMP VIEW codex_conversation_events AS
|
|
8653
|
+
SELECT * FROM main.codex_conversation_events WHERE {stored_predicate};
|
|
8654
|
+
CREATE TEMP VIEW codex_conversation_messages AS
|
|
8655
|
+
SELECT * FROM main.codex_conversation_messages WHERE {stored_predicate};
|
|
8656
|
+
CREATE TEMP VIEW codex_conversation_file_touches AS
|
|
8657
|
+
SELECT t.* FROM main.codex_conversation_file_touches t
|
|
8658
|
+
WHERE EXISTS (
|
|
8659
|
+
SELECT 1 FROM codex_conversation_messages m WHERE m.id=t.message_id
|
|
8660
|
+
);
|
|
8661
|
+
CREATE TEMP VIEW codex_find_projection AS
|
|
8662
|
+
SELECT p.* FROM main.codex_find_projection p
|
|
8663
|
+
WHERE EXISTS (
|
|
8664
|
+
SELECT 1 FROM codex_conversation_messages m WHERE m.id=p.message_id
|
|
8665
|
+
);
|
|
8666
|
+
CREATE TEMP VIEW codex_session_entries AS
|
|
8667
|
+
SELECT * FROM cache_db.codex_session_entries WHERE {stored_predicate};
|
|
8668
|
+
|
|
8669
|
+
-- Thread metadata has conversation-level provenance and can predate an
|
|
8670
|
+
-- in-file account switch. Do not expose it in a scoped connection;
|
|
8671
|
+
-- the Codex rollup degrades explicitly to unassigned rather than leaking
|
|
8672
|
+
-- another account's cwd/git metadata.
|
|
8673
|
+
CREATE TEMP VIEW codex_conversation_threads AS
|
|
8674
|
+
SELECT * FROM cache_db.codex_conversation_threads WHERE 0;
|
|
8675
|
+
CREATE TEMP VIEW codex_source_roots AS
|
|
8676
|
+
SELECT * FROM cache_db.codex_source_roots WHERE 0;
|
|
8677
|
+
|
|
8678
|
+
CREATE TEMP TABLE conversation_sessions AS
|
|
8679
|
+
SELECT * FROM main.conversation_sessions WHERE 0;
|
|
8680
|
+
CREATE TEMP TABLE codex_conversation_rollups (
|
|
8681
|
+
conversation_key TEXT NOT NULL PRIMARY KEY,
|
|
8682
|
+
source_root_key TEXT NOT NULL,
|
|
8683
|
+
parent_thread_id TEXT,
|
|
8684
|
+
item_count INTEGER NOT NULL DEFAULT 0,
|
|
8685
|
+
started_utc TEXT,
|
|
8686
|
+
last_activity_utc TEXT,
|
|
8687
|
+
project_key TEXT,
|
|
8688
|
+
project_label TEXT,
|
|
8689
|
+
models_json TEXT,
|
|
8690
|
+
title TEXT
|
|
8691
|
+
);
|
|
8692
|
+
"""
|
|
8693
|
+
)
|
|
8694
|
+
_recompute_conversation_sessions(conn)
|
|
8695
|
+
codex_keys = {
|
|
8696
|
+
row[0]
|
|
8697
|
+
for row in conn.execute(
|
|
8698
|
+
"SELECT DISTINCT conversation_key FROM codex_conversation_messages"
|
|
8699
|
+
)
|
|
8700
|
+
if row[0]
|
|
8701
|
+
}
|
|
8702
|
+
_recompute_codex_rollups(conn, codex_keys)
|
|
8703
|
+
# TEMP rollup writes open a transaction. Close it before a long-lived
|
|
8704
|
+
# account-scoped SSE reader starts watching so later provider-writer commits
|
|
8705
|
+
# are visible to the dynamic leaf views and never contend on this setup work.
|
|
8706
|
+
conn.commit()
|
|
8707
|
+
|
|
8708
|
+
|
|
8098
8709
|
def _open_conversations_db_for_recovery(
|
|
8099
8710
|
*, attach_cache: bool = True,
|
|
8100
8711
|
) -> sqlite3.Connection:
|
|
@@ -8669,6 +9280,7 @@ def _prepare_claude_conversation_maintenance(
|
|
|
8669
9280
|
*,
|
|
8670
9281
|
rebuild: bool,
|
|
8671
9282
|
targeted: bool,
|
|
9283
|
+
active_account_key: "str | None" = None,
|
|
8672
9284
|
) -> bool:
|
|
8673
9285
|
"""Consume transcript-only upgrade work under the conversation flock.
|
|
8674
9286
|
|
|
@@ -8760,7 +9372,9 @@ def _prepare_claude_conversation_maintenance(
|
|
|
8760
9372
|
"'conversation_background_mcp_reingest_pending')"
|
|
8761
9373
|
).fetchone() is not None
|
|
8762
9374
|
if reingest:
|
|
8763
|
-
_resumable_reingest_conversation_messages(
|
|
9375
|
+
_resumable_reingest_conversation_messages(
|
|
9376
|
+
conn, active_account_key=active_account_key,
|
|
9377
|
+
)
|
|
8764
9378
|
_set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
|
|
8765
9379
|
conn.commit()
|
|
8766
9380
|
|
|
@@ -8806,6 +9420,20 @@ def sync_claude_conversations(
|
|
|
8806
9420
|
stats.lock_contended = True
|
|
8807
9421
|
return stats
|
|
8808
9422
|
|
|
9423
|
+
# #347 observe-and-stamp mirrors the accounting ingest boundary. A
|
|
9424
|
+
# torn credential read is undecided, not unattributed: defer before any
|
|
9425
|
+
# maintenance/rebuild mutation so the cursor remains replayable.
|
|
9426
|
+
import _lib_accounts
|
|
9427
|
+
claude_identity = _cctally_core._resolve_active_claude_identity()
|
|
9428
|
+
if claude_identity.get("status") == "torn":
|
|
9429
|
+
stats.files_deferred_torn += 1
|
|
9430
|
+
stats.deferred_reason = "identity_torn"
|
|
9431
|
+
return stats
|
|
9432
|
+
active_key = claude_identity["account_key"]
|
|
9433
|
+
active_account_key = (
|
|
9434
|
+
None if active_key == _lib_accounts.UNATTRIBUTED else active_key
|
|
9435
|
+
)
|
|
9436
|
+
|
|
8809
9437
|
targeted = only_paths is not None
|
|
8810
9438
|
pending_rebuild = conn.execute(
|
|
8811
9439
|
"SELECT 1 FROM cache_meta "
|
|
@@ -8835,10 +9463,19 @@ def sync_claude_conversations(
|
|
|
8835
9463
|
|
|
8836
9464
|
_report_conversation_progress(progress, "prepare", stats)
|
|
8837
9465
|
maintenance_replayed = _prepare_claude_conversation_maintenance(
|
|
8838
|
-
conn, rebuild=rebuild, targeted=targeted
|
|
9466
|
+
conn, rebuild=rebuild, targeted=targeted,
|
|
9467
|
+
active_account_key=active_account_key,
|
|
8839
9468
|
)
|
|
8840
9469
|
|
|
9470
|
+
rebuild_account_stamps: dict[tuple[str, int], str | None] = {}
|
|
8841
9471
|
if rebuild:
|
|
9472
|
+
rebuild_account_stamps = {
|
|
9473
|
+
(str(path), int(offset)): account_key
|
|
9474
|
+
for path, offset, account_key in conn.execute(
|
|
9475
|
+
"SELECT source_path,byte_offset,account_key "
|
|
9476
|
+
"FROM conversation_messages"
|
|
9477
|
+
)
|
|
9478
|
+
}
|
|
8842
9479
|
clear_conversation_messages(conn)
|
|
8843
9480
|
conn.execute("DELETE FROM conversation_ai_titles")
|
|
8844
9481
|
conn.execute("DELETE FROM conversation_sessions")
|
|
@@ -8915,7 +9552,16 @@ def sync_claude_conversations(
|
|
|
8915
9552
|
include_cost=False,
|
|
8916
9553
|
):
|
|
8917
9554
|
if mrow is not None:
|
|
8918
|
-
|
|
9555
|
+
account_key = rebuild_account_stamps.get(
|
|
9556
|
+
(path_str, int(mrow.byte_offset)),
|
|
9557
|
+
active_account_key,
|
|
9558
|
+
)
|
|
9559
|
+
conv_rows.append(
|
|
9560
|
+
_conv_row_tuple(
|
|
9561
|
+
mrow, path_str, account_key,
|
|
9562
|
+
include_account=True,
|
|
9563
|
+
)
|
|
9564
|
+
)
|
|
8919
9565
|
if ai is not None:
|
|
8920
9566
|
ai_rows.append(
|
|
8921
9567
|
(ai.session_id, ai.ai_title, path_str, ai.byte_offset)
|
|
@@ -8952,7 +9598,7 @@ def sync_claude_conversations(
|
|
|
8952
9598
|
)
|
|
8953
9599
|
stats.files_reset_truncated += 1
|
|
8954
9600
|
if conv_rows:
|
|
8955
|
-
conn.executemany(
|
|
9601
|
+
conn.executemany(_CONV_ACCOUNT_INSERT_SQL, conv_rows)
|
|
8956
9602
|
_fill_file_touches(
|
|
8957
9603
|
conn, scope=[(row[3], row[4]) for row in conv_rows]
|
|
8958
9604
|
)
|
|
@@ -9059,6 +9705,7 @@ def sync_codex_conversations(
|
|
|
9059
9705
|
"""Delta-sync Codex events/search rows into conversations.db (#320)."""
|
|
9060
9706
|
stats = CodexIngestStats()
|
|
9061
9707
|
did_from_zero_replay = False
|
|
9708
|
+
rebuild_account_stamps: dict[tuple[str, int], str | None] = {}
|
|
9062
9709
|
_cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
|
|
9063
9710
|
_cctally_core.CONVERSATIONS_LOCK_CODEX_PATH.touch()
|
|
9064
9711
|
lock_fh = open(_cctally_core.CONVERSATIONS_LOCK_CODEX_PATH, "w")
|
|
@@ -9106,7 +9753,51 @@ def sync_codex_conversations(
|
|
|
9106
9753
|
return stats
|
|
9107
9754
|
rebuild = (
|
|
9108
9755
|
rebuild or pending_rebuild or contract_rebuild or codex_replay_pending)
|
|
9756
|
+
rebuild_prune_scope: _CodexPruneScope | None = None
|
|
9109
9757
|
if rebuild:
|
|
9758
|
+
rebuild_account_stamps = {
|
|
9759
|
+
(str(path), int(offset)): account_key
|
|
9760
|
+
for path, offset, account_key in conn.execute(
|
|
9761
|
+
"SELECT source_path,line_offset,account_key "
|
|
9762
|
+
"FROM codex_conversation_events"
|
|
9763
|
+
)
|
|
9764
|
+
}
|
|
9765
|
+
# #485: establish positive root evidence before the destructive
|
|
9766
|
+
# clear. A rebuild against an empty or missing CODEX_HOME must keep
|
|
9767
|
+
# the retained transcript store and its pending replay marker.
|
|
9768
|
+
preflight_files = _discover_codex_files_with_roots()
|
|
9769
|
+
preflight_scope = _codex_prune_scope(preflight_files)
|
|
9770
|
+
rebuild_prune_scope = preflight_scope
|
|
9771
|
+
preflight_sources = [
|
|
9772
|
+
(path, root_key)
|
|
9773
|
+
for path, root_key in conn.execute(
|
|
9774
|
+
"SELECT path,source_root_key "
|
|
9775
|
+
"FROM codex_conversation_source_files"
|
|
9776
|
+
)
|
|
9777
|
+
]
|
|
9778
|
+
(
|
|
9779
|
+
_safe_sources,
|
|
9780
|
+
refused_sources,
|
|
9781
|
+
_safe_roots,
|
|
9782
|
+
_refused_roots,
|
|
9783
|
+
) = _partition_codex_prune_candidates(
|
|
9784
|
+
preflight_scope, preflight_sources, set()
|
|
9785
|
+
)
|
|
9786
|
+
if refused_sources:
|
|
9787
|
+
_codex_prune_refusal_record(
|
|
9788
|
+
conn,
|
|
9789
|
+
store="conversations",
|
|
9790
|
+
scope=preflight_scope,
|
|
9791
|
+
refused_sources=refused_sources,
|
|
9792
|
+
refused_root_keys=set(),
|
|
9793
|
+
)
|
|
9794
|
+
conn.commit()
|
|
9795
|
+
stats.files_total = len(preflight_files)
|
|
9796
|
+
stats.prune_refused = True
|
|
9797
|
+
stats.prune_refused_files = len({
|
|
9798
|
+
path for path, _root in refused_sources
|
|
9799
|
+
})
|
|
9800
|
+
return stats
|
|
9110
9801
|
conn.execute(
|
|
9111
9802
|
"INSERT OR REPLACE INTO cache_meta(key,value) VALUES(?,?)",
|
|
9112
9803
|
("conversation_rebuild_codex_pending", "1"),
|
|
@@ -9152,9 +9843,36 @@ def sync_codex_conversations(
|
|
|
9152
9843
|
if current_size < prev[0]:
|
|
9153
9844
|
stats.deferred_reason = "truncation"
|
|
9154
9845
|
return stats
|
|
9846
|
+
protected_paths: set[str] = set()
|
|
9847
|
+
ordinary_prune_scope: _CodexPruneScope | None = None
|
|
9155
9848
|
if only_paths is None:
|
|
9156
9849
|
active_paths = {str(item.source_path) for item in files}
|
|
9157
|
-
|
|
9850
|
+
prune_scope = _codex_prune_scope(files)
|
|
9851
|
+
ordinary_prune_scope = prune_scope
|
|
9852
|
+
stale_sources = [
|
|
9853
|
+
(stale_path, existing[stale_path][3])
|
|
9854
|
+
for stale_path in sorted(set(existing) - active_paths)
|
|
9855
|
+
]
|
|
9856
|
+
safe_sources, _refused_stale, _safe_roots, _refused_roots = (
|
|
9857
|
+
_partition_codex_prune_candidates(
|
|
9858
|
+
prune_scope, stale_sources, set()
|
|
9859
|
+
)
|
|
9860
|
+
)
|
|
9861
|
+
retained_sources = [
|
|
9862
|
+
(path, values[3])
|
|
9863
|
+
for path, values in existing.items()
|
|
9864
|
+
if os.path.isabs(path)
|
|
9865
|
+
]
|
|
9866
|
+
(
|
|
9867
|
+
_safe_retained,
|
|
9868
|
+
refused_sources,
|
|
9869
|
+
_safe_retained_roots,
|
|
9870
|
+
_refused_retained_roots,
|
|
9871
|
+
) = _partition_codex_prune_candidates(
|
|
9872
|
+
prune_scope, retained_sources, set()
|
|
9873
|
+
)
|
|
9874
|
+
protected_paths = {path for path, _root in refused_sources}
|
|
9875
|
+
for stale_path, _stale_root_key in safe_sources:
|
|
9158
9876
|
affected = {
|
|
9159
9877
|
row[0] for row in conn.execute(
|
|
9160
9878
|
"SELECT DISTINCT conversation_key "
|
|
@@ -9179,11 +9897,30 @@ def sync_codex_conversations(
|
|
|
9179
9897
|
(stale_path,),
|
|
9180
9898
|
)
|
|
9181
9899
|
_recompute_codex_rollups(conn, affected)
|
|
9900
|
+
import _lib_codex_conversation_query as query
|
|
9901
|
+
query.materialize_codex_find_projection(conn, affected)
|
|
9902
|
+
stats.files_pruned = len({path for path, _root in safe_sources})
|
|
9903
|
+
if refused_sources:
|
|
9904
|
+
_codex_prune_refusal_record(
|
|
9905
|
+
conn,
|
|
9906
|
+
store="conversations",
|
|
9907
|
+
scope=prune_scope,
|
|
9908
|
+
refused_sources=refused_sources,
|
|
9909
|
+
refused_root_keys=set(),
|
|
9910
|
+
)
|
|
9911
|
+
stats.prune_refused = True
|
|
9912
|
+
stats.prune_refused_files = len({
|
|
9913
|
+
path for path, _root in refused_sources
|
|
9914
|
+
})
|
|
9182
9915
|
conn.commit()
|
|
9183
9916
|
|
|
9184
9917
|
for discovered in files:
|
|
9185
9918
|
jp = discovered.source_path
|
|
9186
9919
|
path_str = str(jp)
|
|
9920
|
+
if path_str in protected_paths:
|
|
9921
|
+
# Visible but unrecognizable input is not evidence that the
|
|
9922
|
+
# retained transcript generation should be reset or replaced.
|
|
9923
|
+
continue
|
|
9187
9924
|
try:
|
|
9188
9925
|
st = jp.stat()
|
|
9189
9926
|
except OSError:
|
|
@@ -9219,6 +9956,31 @@ def sync_codex_conversations(
|
|
|
9219
9956
|
initial_conversation = prev[10] if prev else None
|
|
9220
9957
|
initial_turn = prev[11] if prev else None
|
|
9221
9958
|
|
|
9959
|
+
# #347 uses the accounting cache's journal-backed file-range map as
|
|
9960
|
+
# the Codex transcript oracle. It is already attached read-only,
|
|
9961
|
+
# and core ingest runs first, so rebuilds replay the original
|
|
9962
|
+
# decision instead of consulting the currently-active auth.json.
|
|
9963
|
+
try:
|
|
9964
|
+
file_identity = codex_file_identity(discovered)
|
|
9965
|
+
incarnation_row = conn.execute(
|
|
9966
|
+
"SELECT incarnation FROM cache_db.codex_file_incarnations "
|
|
9967
|
+
"WHERE file_identity=?",
|
|
9968
|
+
(file_identity,),
|
|
9969
|
+
).fetchone()
|
|
9970
|
+
incarnation = int(incarnation_row[0]) if incarnation_row else 1
|
|
9971
|
+
account_ranges = [
|
|
9972
|
+
(int(off), key)
|
|
9973
|
+
for off, key in conn.execute(
|
|
9974
|
+
"SELECT from_offset,account_key "
|
|
9975
|
+
"FROM cache_db.codex_file_accounts "
|
|
9976
|
+
"WHERE file_identity=? AND incarnation=? "
|
|
9977
|
+
"ORDER BY from_offset",
|
|
9978
|
+
(file_identity, incarnation),
|
|
9979
|
+
)
|
|
9980
|
+
]
|
|
9981
|
+
except sqlite3.OperationalError:
|
|
9982
|
+
account_ranges = []
|
|
9983
|
+
|
|
9222
9984
|
state = _CodexIterState(
|
|
9223
9985
|
session_id=initial_session_id,
|
|
9224
9986
|
model=initial_model,
|
|
@@ -9241,6 +10003,7 @@ def sync_codex_conversations(
|
|
|
9241
10003
|
)
|
|
9242
10004
|
events = []
|
|
9243
10005
|
event_rows = []
|
|
10006
|
+
account_by_physical: dict[tuple[str, int], str | None] = {}
|
|
9244
10007
|
yielded = 0
|
|
9245
10008
|
try:
|
|
9246
10009
|
with open(jp, "rb") as fh:
|
|
@@ -9256,6 +10019,16 @@ def sync_codex_conversations(
|
|
|
9256
10019
|
):
|
|
9257
10020
|
event = emission.event
|
|
9258
10021
|
events.append(event)
|
|
10022
|
+
covered, account_key = codex_account_for_offset(
|
|
10023
|
+
account_ranges, int(event.line_offset)
|
|
10024
|
+
)
|
|
10025
|
+
if not covered:
|
|
10026
|
+
account_key = rebuild_account_stamps.get(
|
|
10027
|
+
(event.source_path, int(event.line_offset))
|
|
10028
|
+
)
|
|
10029
|
+
account_by_physical[
|
|
10030
|
+
(event.source_path, int(event.line_offset))
|
|
10031
|
+
] = account_key
|
|
9259
10032
|
event_rows.append((
|
|
9260
10033
|
event.source_path,
|
|
9261
10034
|
event.line_offset,
|
|
@@ -9270,6 +10043,7 @@ def sync_codex_conversations(
|
|
|
9270
10043
|
event.turn_id,
|
|
9271
10044
|
event.call_id,
|
|
9272
10045
|
event.payload_json,
|
|
10046
|
+
account_key,
|
|
9273
10047
|
))
|
|
9274
10048
|
if emission.accounting is not None:
|
|
9275
10049
|
yielded += 1
|
|
@@ -9327,12 +10101,13 @@ def sync_codex_conversations(
|
|
|
9327
10101
|
"INSERT OR IGNORE INTO codex_conversation_events "
|
|
9328
10102
|
"(source_path,line_offset,source_root_key,conversation_key,"
|
|
9329
10103
|
"native_thread_id,root_thread_id,parent_thread_id,"
|
|
9330
|
-
"timestamp_utc,record_type,event_type,turn_id,call_id,payload_json
|
|
9331
|
-
"VALUES (
|
|
10104
|
+
"timestamp_utc,record_type,event_type,turn_id,call_id,payload_json,"
|
|
10105
|
+
"account_key) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
9332
10106
|
event_rows,
|
|
9333
10107
|
)
|
|
9334
10108
|
_insert_codex_normalized_rows(
|
|
9335
|
-
conn, normalized.rows, normalized.touches
|
|
10109
|
+
conn, normalized.rows, normalized.touches,
|
|
10110
|
+
account_by_physical,
|
|
9336
10111
|
)
|
|
9337
10112
|
affected_keys.update(
|
|
9338
10113
|
row.conversation_key for row in normalized.rows
|
|
@@ -9345,6 +10120,8 @@ def sync_codex_conversations(
|
|
|
9345
10120
|
_repair_codex_turn_ids_for_source(conn, path_str)
|
|
9346
10121
|
)
|
|
9347
10122
|
_recompute_codex_rollups(conn, affected_keys)
|
|
10123
|
+
import _lib_codex_conversation_query as query
|
|
10124
|
+
query.materialize_codex_find_projection(conn, affected_keys)
|
|
9348
10125
|
terminal = state.thread
|
|
9349
10126
|
conn.execute(
|
|
9350
10127
|
"INSERT INTO codex_conversation_source_files "
|
|
@@ -9392,8 +10169,20 @@ def sync_codex_conversations(
|
|
|
9392
10169
|
stats.files_failed += 1
|
|
9393
10170
|
_report_conversation_progress(progress, "ingest", stats)
|
|
9394
10171
|
|
|
10172
|
+
run_codex_find_projection_backfill(conn)
|
|
9395
10173
|
_report_conversation_progress(progress, "finalize", stats)
|
|
9396
10174
|
if only_paths is None and stats.files_failed == 0:
|
|
10175
|
+
recovery_scope = rebuild_prune_scope or ordinary_prune_scope
|
|
10176
|
+
if (
|
|
10177
|
+
recovery_scope is not None
|
|
10178
|
+
and recovery_scope.recognized_root_keys
|
|
10179
|
+
and not stats.prune_refused
|
|
10180
|
+
and (
|
|
10181
|
+
stats.files_processed + stats.files_skipped_unchanged
|
|
10182
|
+
== stats.files_total
|
|
10183
|
+
)
|
|
10184
|
+
):
|
|
10185
|
+
_clear_codex_prune_refusal(conn)
|
|
9397
10186
|
conn.execute(
|
|
9398
10187
|
"INSERT OR REPLACE INTO cache_meta(key,value) VALUES(?,?)",
|
|
9399
10188
|
(
|