cctally 1.70.0 → 1.72.0

package/bin/_cctally_cache.py

@@ -178,6 +178,11 @@ _should_replace = _lib_jsonl._should_replace
 _lib_conversation = _load_lib("_lib_conversation")
 _iter_message_rows = _lib_conversation.iter_message_rows
 
+# #294 S6: the pure Codex conversation normalization kernel. Pure stdlib leaf
+# (imports only the _lib_conversation / _lib_conversation_query display helpers),
+# so it loads at module-load time alongside _lib_conversation.
+_lib_codex_conversation = _load_lib("_lib_codex_conversation")
+
 # Opt-in backend phase-instrumentation collector (issue #276, Session A). Pure
 # stdlib leaf; near-noop when CCTALLY_PERF_TRACE is unset (phase() returns a
 # shared no-op singleton), so the sync_cache seam wraps below cost nothing on
@@ -320,6 +325,10 @@ _CACHE_MIGRATIONS = _cctally_db_sib._CACHE_MIGRATIONS
 # drop/recreate dance so the per-row delete trigger never fires O(rows) under
 # the held lock on a rebuild / truncation escalation.
 clear_conversation_messages = _cctally_db_sib.clear_conversation_messages
+# #294 S6: storm-free FULL clear of the Codex normalized derived tables (drop
+# triggers -> truncate -> 'delete-all' -> recreate triggers). Used only by the
+# cache-rebuild path + migration 025; partial deletes ride the per-row triggers.
+_codex_conversation_fts_full_clear = _cctally_db_sib._codex_conversation_fts_full_clear
 # cache_meta key/value upsert helper — reused by the resumable reingest cursor
 # writes (#179) so the ON CONFLICT idiom lives in one place. Caller commits.
 _set_cache_meta = _cctally_db_sib._set_cache_meta
@@ -711,6 +720,21 @@ def _delete_codex_file_derived_rows(
         "DELETE FROM codex_conversation_threads WHERE source_path = ?" + root_clause,
         params,
     )
+    # #294 S6: normalized rows + file touches for this file. This is a PARTIAL
+    # delete (one file), so it rides the per-row FTS delete trigger — surviving
+    # conversations keep their postings (§3.4 two-path policy). File touches have
+    # no source_root_key column; a source_path maps to exactly one file/root, so
+    # scoping them by source_path alone is exact. Rollups are repaired by the
+    # caller's _recompute_codex_rollups (§3.2), which needs the affected keys
+    # captured BEFORE this delete.
+    conn.execute(
+        "DELETE FROM codex_conversation_file_touches WHERE source_path = ?",
+        (path_str,),
+    )
+    conn.execute(
+        "DELETE FROM codex_conversation_messages WHERE source_path = ?" + root_clause,
+        params,
+    )
     conn.execute(
         "DELETE FROM codex_session_files WHERE path = ?" + root_clause,
         params,
@@ -725,6 +749,17 @@ def _clear_codex_derived_rows(conn: sqlite3.Connection) -> None:
     conn.execute("DELETE FROM codex_conversation_events")
     conn.execute("DELETE FROM codex_session_files")
     conn.execute("DELETE FROM codex_source_roots")
+    # #294 S6: FULL clear of the normalized derived tables (messages + touches +
+    # rollups) via the storm-free helper — this is a whole-corpus rebuild, so
+    # 'delete-all' resets the FTS shadow tables cleanly (§3.4 full-clear path).
+    _codex_conversation_fts_full_clear(conn)
+    # F3: this clears the Codex physical quota state, so any stored
+    # quota-projection certificate would become stale-valid (its cache
+    # sequence unchanged) and let the reconcile short-circuit skip over
+    # now-deleted data. Invalidate it in the same transaction.
+    conn.execute(
+        "DELETE FROM cache_meta WHERE key='codex_quota_projection_certificate'"
+    )
 
 
 def _bump_codex_physical_mutation_seq(conn: sqlite3.Connection) -> None:
@@ -736,6 +771,181 @@ def _bump_codex_physical_mutation_seq(conn: sqlite3.Connection) -> None:
     )
 
 
+# ── #294 S6: normalized-conversation write helpers (kernel-backed) ────────────
+
+_CODEX_NORM_COLS = (
+    "conversation_key, source_root_key, source_path, line_offset, timestamp_utc, "
+    "turn_id, call_id, kind, event_type, record_family, model, text, "
+    "content_digest, content_len, detail_json, search_tool, search_thinking"
+)
+_CODEX_MSG_INSERT_SQL = (
+    "INSERT OR IGNORE INTO codex_conversation_messages (" + _CODEX_NORM_COLS + ") "
+    "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
+)
+
+
+def _codex_conversation_project_attribution(
+    source_root_key: str | None, cwd: object, git_json: object,
+) -> tuple[str | None, str | None]:
+    """(project_key, project_label) for one conversation's thread facts (§3.2).
+
+    Mirrors the S3 read-time attribution (cwd git-root resolution → basename
+    label, else git identity, else unassigned) so the browse live-recompute
+    fallback matches the stored rollup. Degrades to (None, None) when the S3
+    kernel or a valid source root is unavailable — never guesses.
+    """
+    if not source_root_key:
+        return None, None
+    try:
+        from _cctally_source_analytics import _project_label, _git_resolved_key
+        from _lib_source_analytics import opaque_project_key
+    except Exception:
+        return None, None
+    if isinstance(cwd, str) and cwd:
+        project = _resolve_project_key(cwd, "git-root", {})
+        resolved_key = project.bucket_path
+        cwd_label = _project_label(cwd)
+        project_label = (
+            cwd_label if cwd_label in {"(home)", "(root)"}
+            else _project_label(project.display_key)
+        )
+    elif isinstance(git_json, str) and git_json:
+        git_key = _git_resolved_key(git_json)
+        if git_key is None:
+            resolved_key, project_label = "(unassigned)", "(unassigned)"
+        else:
+            resolved_key, project_label = git_key, "Git project"
+    else:
+        resolved_key, project_label = "(unassigned)", "(unassigned)"
+    try:
+        return opaque_project_key("codex", source_root_key, resolved_key), project_label
+    except ValueError:
+        return None, None
+
+
+def _load_codex_normalized_rows(
+    conn: sqlite3.Connection, conversation_key: str,
+) -> list:
+    """Load a conversation's normalized rows (all files) in physical order as
+    kernel CodexNormalizedRow objects."""
+    return [
+        _lib_codex_conversation.CodexNormalizedRow(*row)
+        for row in conn.execute(
+            "SELECT " + _CODEX_NORM_COLS + " FROM codex_conversation_messages "
+            "WHERE conversation_key = ? "
+            "ORDER BY timestamp_utc, source_path, line_offset",
+            (conversation_key,),
+        )
+    ]
+
+
+def _insert_codex_normalized_rows(conn: sqlite3.Connection, rows: list, touches: list) -> None:
+    """Insert normalized rows (INSERT OR IGNORE on the physical key) + their file
+    touches (message linkage resolved via (source_path, line_offset))."""
+    if rows:
+        conn.executemany(_CODEX_MSG_INSERT_SQL, [
+            (r.conversation_key, r.source_root_key, r.source_path, r.line_offset,
+             r.timestamp_utc, r.turn_id, r.call_id, r.kind, r.event_type,
+             r.record_family, r.model, r.text, r.content_digest, r.content_len,
+             r.detail_json, r.search_tool, r.search_thinking)
+            for r in rows
+        ])
+    for touch in touches:
+        conn.execute(
+            "INSERT OR IGNORE INTO codex_conversation_file_touches "
+            "(message_id, conversation_key, source_path, file_path, tool) "
+            "SELECT m.id, ?, ?, ?, ? FROM codex_conversation_messages m "
+            "WHERE m.source_path = ? AND m.line_offset = ?",
+            (touch.conversation_key, touch.source_path, touch.file_path, touch.tool,
+             touch.source_path, touch.line_offset),
+        )
+
+
+def _recompute_codex_rollups(conn: sqlite3.Connection, conversation_keys) -> None:
+    """Recompute-affected-or-delete the rollup for each conversation (§3.2).
+
+    A rollup is a pure function of surviving codex_conversation_messages (+ thread
+    metadata): aggregate across ALL files of the conversation, delete emptied
+    rollups, stamp item_count (rendered LOGICAL items), title, project attribution,
+    times, and models. Called by every write/delete path so no stale rollup
+    survives.
+    """
+    kern = _lib_codex_conversation
+    for conversation_key in {key for key in conversation_keys if key}:
+        rows = _load_codex_normalized_rows(conn, conversation_key)
+        if not rows:
+            conn.execute(
+                "DELETE FROM codex_conversation_rollups WHERE conversation_key = ?",
+                (conversation_key,),
+            )
+            continue
+        item_count = kern.rollup_item_count(rows)
+        title = kern.derive_title(rows)
+        timestamps = [r.timestamp_utc for r in rows if r.timestamp_utc]
+        started = min(timestamps) if timestamps else None
+        last_activity = max(timestamps) if timestamps else None
+        models = sorted({r.model for r in rows if r.model})
+        models_json = json.dumps(models) if models else None
+        source_root_key = rows[0].source_root_key
+        thread = conn.execute(
+            "SELECT cwd, git_json, parent_thread_id, source_root_key "
+            "FROM codex_conversation_threads WHERE conversation_key = ?",
+            (conversation_key,),
+        ).fetchone()
+        cwd = git_json = parent_thread_id = None
+        if thread is not None:
+            cwd, git_json, parent_thread_id, thread_root = thread
+            if thread_root:
+                source_root_key = thread_root
+        project_key, project_label = _codex_conversation_project_attribution(
+            source_root_key, cwd, git_json)
+        conn.execute(
+            "INSERT INTO codex_conversation_rollups "
+            "(conversation_key, source_root_key, parent_thread_id, item_count, "
+            " started_utc, last_activity_utc, project_key, project_label, "
+            " models_json, title) VALUES (?,?,?,?,?,?,?,?,?,?) "
+            "ON CONFLICT(conversation_key) DO UPDATE SET "
+            " source_root_key=excluded.source_root_key, "
+            " parent_thread_id=excluded.parent_thread_id, "
+            " item_count=excluded.item_count, started_utc=excluded.started_utc, "
+            " last_activity_utc=excluded.last_activity_utc, "
+            " project_key=excluded.project_key, project_label=excluded.project_label, "
+            " models_json=excluded.models_json, title=excluded.title",
+            (conversation_key, source_root_key, parent_thread_id, item_count,
+             started, last_activity, project_key, project_label, models_json, title),
+        )
+
+
+def _replay_codex_normalization(conn: sqlite3.Connection) -> None:
+    """Re-derive ALL normalized rows/touches/rollups from stored
+    codex_conversation_events, per file in (source_path ASC, line_offset ASC)
+    order (migration 025). Runs inside the caller's transaction — the caller
+    full-clears first (§3.4 helper) and owns the commit. Deterministic order +
+    the plain rowid alias make a re-run byte-idempotent."""
+    kern = _lib_codex_conversation
+    events_by_file: dict[str, list] = {}
+    order: list[str] = []
+    for row in conn.execute(
+        "SELECT source_path, line_offset, source_root_key, conversation_key, "
+        "native_thread_id, root_thread_id, parent_thread_id, timestamp_utc, "
+        "record_type, event_type, turn_id, call_id, payload_json "
+        "FROM codex_conversation_events "
+        "ORDER BY source_path ASC, line_offset ASC"
+    ):
+        event = _lib_jsonl.CodexPhysicalEvent(*row)
+        if event.source_path not in events_by_file:
+            events_by_file[event.source_path] = []
+            order.append(event.source_path)
+        events_by_file[event.source_path].append(event)
+    affected: set = set()
+    for source_path in order:
+        result = kern.normalize_codex_events(
+            events_by_file[source_path], initial=kern.CodexStickyState())
+        _insert_codex_normalized_rows(conn, result.rows, result.touches)
+        affected.update(r.conversation_key for r in result.rows)
+    _recompute_codex_rollups(conn, affected)
+
+
 def _collect_inactive_codex_paths_and_roots(
     conn: sqlite3.Connection,
     current_file_identities: set[tuple[str, str]],
@@ -856,16 +1066,27 @@ def _write_codex_file_batch(
     last_root_thread_id: str | None,
     last_parent_thread_id: str | None,
     last_conversation_key: str | None,
+    last_turn_id: str | None,
     reset_file: bool,
     accounting_rows: list[tuple[Any, ...]],
     quota_rows: list[tuple[Any, ...]],
     thread_rows: list[tuple[Any, ...]],
     event_rows: list[tuple[Any, ...]],
+    normalized_rows: list,
+    normalized_touches: list,
     active_root_keys: set[str],
 ) -> int:
     """Write one fully-buffered Codex file atomically and return entry changes."""
     now_iso = dt.datetime.now(dt.timezone.utc).isoformat()
+    # #294 S6: rollups are recomputed-affected-or-deleted (§3.2), so capture the
+    # conversation keys this file's normalized rows touch BEFORE a reset delete
+    # removes them.
+    affected_keys: set = set()
     if reset_file:
+        affected_keys.update(
+            row[0] for row in conn.execute(
+                "SELECT DISTINCT conversation_key FROM codex_conversation_messages "
+                "WHERE source_path = ?", (path_str,)) if row[0])
         _delete_codex_file_derived_rows(conn, path_str)
     conn.execute(
         """INSERT INTO codex_source_roots
@@ -930,18 +1151,27 @@ def _write_codex_file_batch(
                VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""",
             event_rows,
         )
+    # #294 S6: normalized rows + file touches ride the same per-file transaction
+    # as the events themselves. INSERT OR IGNORE on the physical key keeps a delta
+    # re-read idempotent; the per-row FTS triggers index them. Then recompute the
+    # rollup for every affected conversation (aggregating across all its files),
+    # deleting emptied rollups. Threads were inserted above, so project
+    # attribution can read them.
+    _insert_codex_normalized_rows(conn, normalized_rows, normalized_touches)
+    affected_keys.update(r.conversation_key for r in normalized_rows)
+    _recompute_codex_rollups(conn, affected_keys)
     conn.execute(
         """INSERT OR REPLACE INTO codex_session_files
            (path, size_bytes, mtime_ns, last_byte_offset, last_ingested_at,
             last_session_id, last_model, last_total_tokens, source_root_key,
             last_native_thread_id, last_root_thread_id, last_parent_thread_id,
-            last_conversation_key)
-           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""",
+            last_conversation_key, last_turn_id)
+           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
         (
             path_str, size, mtime_ns, final_offset, now_iso, last_session_id,
             last_model, last_total_tokens, discovered.source_root_key,
             last_native_thread_id, last_root_thread_id, last_parent_thread_id,
-            last_conversation_key,
+            last_conversation_key, last_turn_id,
         ),
     )
     _prune_inactive_codex_source_roots(conn, active_root_keys)
@@ -1388,6 +1618,30 @@ def _bump_mutation_seq(conn: sqlite3.Connection) -> int:
     return int(row[0])
 
 
+def _force_retention_prune_after_replay(conn: sqlite3.Connection) -> None:
+    """#313 P3 (F9): run an UNTHROTTLED transcript retention prune after a
+    from-zero replay (a ``--rebuild`` or a truncation/requalification re-ingest,
+    both of which replay from offset 0 and restore >retention-day rows the
+    throttled prune already trimmed). Best-effort — a prune failure must never
+    break a sync. The caller invokes this only after the sync released its
+    provider flock, so the orchestrator can re-acquire it. No-op when retention
+    is disabled (``conversation.retention_days`` 0)."""
+    try:
+        import _lib_conversation_retention as retention
+        from _cctally_config import resolve_retention_days
+        retention_days = resolve_retention_days(_cctally().load_config())
+        if retention_days <= 0:
+            return
+        retention._maybe_prune_conversation_retention(
+            conn,
+            now_utc=dt.datetime.now(dt.timezone.utc),
+            retention_days=retention_days,
+            force=True,
+        )
+    except Exception:
+        pass
+
+
 def sync_cache(
     conn: sqlite3.Connection,
     *,
@@ -2340,13 +2594,20 @@ def sync_cache(
         # and the flock is still held, so the short busy_timeout keeps it from
         # stalling the lock under heavy-reader contention.
         _maybe_truncate_wal(conn, _cctally_core.CACHE_DB_PATH)
-        return stats
+        # #313 P3 (F9): a rebuild or a truncation escalation replays from offset
+        # 0 and restores >retention-day transcript rows. Force an unthrottled
+        # prune AFTER the flock releases below (early lock-contended / deferred
+        # returns above never reach here, so a no-op sync does not prune).
+        did_from_zero_replay = rebuild or stats.files_reset_truncated > 0
     finally:
         try:
             fcntl.flock(lock_fh, fcntl.LOCK_UN)
         except OSError:
             pass
         lock_fh.close()
+    if did_from_zero_replay:
+        _force_retention_prune_after_replay(conn)
+    return stats
 
 
 def backfill_conversation_messages(conn: sqlite3.Connection) -> int:
@@ -3491,6 +3752,17 @@ def sync_codex_cache(
     """
     stats = CodexIngestStats()
     project_after_unlock = False
+    did_from_zero_replay = False
+    # #313 P1 review (F4/F1): when the CACHE certificate is current we cannot
+    # yet decide whether to skip the reconcile — reconcile's own short-circuit
+    # ALSO requires the stats-side quota_projection_state signatures to match
+    # (F1: stats.db can be wiped/recovered while cache.db persists). That
+    # cross-DB read must happen AFTER the Codex flock releases (see the design
+    # comment near the reconcile trigger below), so capture the material for the
+    # deferred stats-side check here. ``None`` means "no deferred check pending"
+    # (the seq-advanced / no-roots / stale-cert branches decide immediately).
+    deferred_cert_roots: "set[str] | None" = None
+    deferred_cert_sigs: "dict[str, str] | None" = None
     c = _cctally()
     _cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
     _cctally_core.CACHE_LOCK_CODEX_PATH.touch()
@@ -3505,6 +3777,18 @@ def sync_codex_cache(
             stats.lock_contended = True
             return stats
 
+        # F4 (#313): the reconcile trigger gate is "did the Codex physical
+        # mutation sequence advance during this sync", NOT rows_changed —
+        # rows_changed counts only inserted accounting rows and misses
+        # quota-only / metadata-only / prune-only batches that bump the
+        # sequence. Capture the baseline before any clear/prune/ingest bump.
+        from _cctally_quota import (
+            codex_physical_mutation_seq,
+            load_codex_quota_projection_certificate,
+            _cache_root_keys,
+        )
+        seq_before = codex_physical_mutation_seq(conn)
+
         if rebuild:
             # Clear INSIDE the lock — see sync_cache() for the full
             # rationale. Done before the existing SELECT so delta
@@ -3554,7 +3838,14 @@ def sync_codex_cache(
             )
             if orphan_sources or orphan_root_keys:
                 before_prune = conn.total_changes
+                # #294 S6: capture the conversation keys the orphan rows belong to
+                # BEFORE deleting, so the rollups can be repaired/deleted after.
+                orphan_keys: set = set()
                 for orphan_path, orphan_root_key in orphan_sources:
+                    orphan_keys.update(
+                        row[0] for row in conn.execute(
+                            "SELECT DISTINCT conversation_key FROM codex_conversation_messages "
+                            "WHERE source_path = ?", (orphan_path,)) if row[0])
                     _delete_codex_file_derived_rows(
                         conn,
                         orphan_path,
@@ -3565,6 +3856,10 @@ def sync_codex_cache(
                     conn, active_root_keys,
                     candidate_root_keys=orphan_root_keys,
                 )
+                # Recompute-affected-or-delete the rollups the prune touched (§3.2):
+                # a conversation with no surviving rows loses its rollup, one that
+                # survives in another file is recomputed from the survivors.
+                _recompute_codex_rollups(conn, orphan_keys)
                 if conn.total_changes != before_prune:
                     _bump_codex_physical_mutation_seq(conn)
                 conn.commit()
@@ -3583,13 +3878,13 @@ def sync_codex_cache(
         existing = {
             row[0]: (
                 row[1], row[2], row[3], row[4], row[5], row[6], row[7],
-                row[8], row[9], row[10], row[11],
+                row[8], row[9], row[10], row[11], row[12],
             )
             for row in conn.execute(
                 "SELECT path, size_bytes, mtime_ns, last_byte_offset, "
                 "last_session_id, last_model, last_total_tokens, source_root_key, "
                 "last_native_thread_id, last_root_thread_id, last_parent_thread_id, "
-                "last_conversation_key "
+                "last_conversation_key, last_turn_id "
                 "FROM codex_session_files"
             )
         }
@@ -3616,6 +3911,8 @@ def sync_codex_cache(
             initial_session_id: str | None = None
             initial_model: str | None = None
             initial_total_tokens = 0
+            # #294 S6: the sticky-turn resume seed (parallel to initial_model).
+            initial_turn_id: str | None = None
             prev_total_tokens: int | None = None
             prev_native_thread_id: str | None = None
             prev_root_thread_id: str | None = None
@@ -3626,7 +3923,7 @@ def sync_codex_cache(
                 (
                     prev_size, _, prev_offset, prev_sid, prev_model, prev_ttot,
                     prev_root_key, prev_native_thread_id, prev_root_thread_id,
-                    prev_parent_thread_id, prev_conversation_key,
+                    prev_parent_thread_id, prev_conversation_key, prev_turn_id,
                 ) = prev
                 prev_total_tokens = (
                     int(prev_ttot) if prev_ttot is not None else None
@@ -3640,18 +3937,23 @@ def sync_codex_cache(
                     initial_session_id = prev_sid
                     initial_model = prev_model
                     initial_total_tokens = prev_total_tokens or 0
+                    initial_turn_id = prev_turn_id
                 else:
                     truncated = True
                     start_offset = 0
                     initial_session_id = None
                     initial_model = None
                     initial_total_tokens = 0
+                    initial_turn_id = None
                     prev_total_tokens = None
 
             accounting_rows: list[tuple[Any, ...]] = []
             quota_rows: list[tuple[Any, ...]] = []
             thread_rows: list[tuple[Any, ...]] = []
             event_rows: list[tuple[Any, ...]] = []
+            # #294 S6: the physical event objects for this file window, fed to the
+            # normalization kernel after the drain (in offset order).
+            events_list: list = []
             final_offset = start_offset
             # Mutable tracker that the iterator updates on every
             # session_meta / turn_context record, regardless of whether a
@@ -3705,6 +4007,7 @@ def sync_codex_cache(
                         state=iter_state,
                     ):
                         event = emission.event
+                        events_list.append(event)
                         event_rows.append((
                             event.source_path, event.line_offset,
                             event.source_root_key, event.conversation_key,
@@ -3811,6 +4114,18 @@ def sync_codex_cache(
                 if terminal_thread is not None else prev_conversation_key
             )
 
+            # #294 S6: normalize this file window's physical events into
+            # normalized rows + file touches, replaying sticky turn/model state
+            # seeded from the persisted resume seed. Buffered before the first DML
+            # (like every other family), so the single retry re-runs the same
+            # in-memory batch. The terminal sticky turn persists as last_turn_id.
+            norm_result = _lib_codex_conversation.normalize_codex_events(
+                events_list,
+                initial=_lib_codex_conversation.CodexStickyState(
+                    turn_id=initial_turn_id, model=initial_model),
+            )
+            new_last_turn_id = norm_result.terminal.turn_id
+
             # Every derived row above was buffered before the first DML. A
             # late database failure therefore rolls the whole file back and
             # retries that same in-memory batch exactly once.
@@ -3831,11 +4146,14 @@ def sync_codex_cache(
                         last_root_thread_id=new_last_root_thread_id,
                         last_parent_thread_id=new_last_parent_thread_id,
                         last_conversation_key=new_last_conversation_key,
+                        last_turn_id=new_last_turn_id,
                         reset_file=truncated or requalified,
                         accounting_rows=accounting_rows,
                         quota_rows=quota_rows,
                         thread_rows=thread_rows,
                         event_rows=event_rows,
+                        normalized_rows=norm_result.rows,
+                        normalized_touches=norm_result.touches,
                         active_root_keys={item.source_root_key for item in files},
                     )
                 except sqlite3.DatabaseError as exc:
@@ -3892,7 +4210,56 @@ def sync_codex_cache(
         # Codex cache flock in ``finally`` below.  cache.db and stats.db are not
         # cross-database atomic: after this committed ingest, a projection
         # interruption is repaired by the next full reconciliation.
-        project_after_unlock = True
+        #
+        # F4 (#313): reconcile when the physical mutation sequence advanced this
+        # sync (a genuine quota/metadata/prune change — NOT rows_changed, which
+        # misses quota-only batches). A pure no-op with an already-coherent
+        # certificate skips even the O(1) reconcile call.
+        #
+        # A no-op sync must STILL reconcile when there are Codex roots but the
+        # projection certificate is missing/stale at the current sequence — a
+        # lost/failed certificate write (best-effort I/O; can fail under a
+        # cache.db lock storm) leaves the dashboard's Codex source "unavailable"
+        # and is recovered by the next unchanged-file sync re-stamping the
+        # certificate. Claude-only users (no Codex roots) always skip.
+        #
+        # F1 (#313 P1 review): even when the CACHE certificate looks current, the
+        # STATS-side quota_projection_state may have been independently
+        # wiped/recovered (this user has a documented stats.db corruption
+        # history). The cache cert alone does NOT prove stats.db still holds the
+        # projection, so the skip decision on that branch is DEFERRED to the
+        # post-flock stats-side signature check below — making the gate's
+        # skip-condition identical to reconcile's own short-circuit-condition.
+        cur_seq = codex_physical_mutation_seq(conn)
+        if cur_seq != seq_before:
+            project_after_unlock = True
+        else:
+            active_roots = _cache_root_keys(conn)
+            if not active_roots:
+                project_after_unlock = False
+            else:
+                certificate = load_codex_quota_projection_certificate(conn)
+                certificate_current = (
+                    certificate is not None
+                    and certificate[0] == cur_seq
+                    and active_roots <= set(certificate[1])
+                )
+                if certificate_current:
+                    # The CACHE certificate is current, but that alone does NOT
+                    # prove stats.db still holds the projection (F1). Defer the
+                    # stats-side signature match until AFTER the Codex flock
+                    # releases below — only then may the reconcile be skipped.
+                    project_after_unlock = False
+                    deferred_cert_roots = set(active_roots)
+                    assert certificate is not None
+                    deferred_cert_sigs = dict(certificate[1])
+                else:
+                    project_after_unlock = True
+        # #313 P3 (F9): a Codex rebuild or a truncation/requalification
+        # re-ingest replays from offset 0 and restores >retention-day
+        # codex_conversation_events. Force an unthrottled prune after the flock
+        # releases (below), so restored old rows don't persist for up to 24h.
+        did_from_zero_replay = rebuild or stats.files_reset_truncated > 0
     finally:
         try:
             fcntl.flock(lock_fh, fcntl.LOCK_UN)
@@ -3900,9 +4267,29 @@ def sync_codex_cache(
             pass
         lock_fh.close()
 
+    if deferred_cert_roots is not None:
+        # F1: the gate's skip-condition must be IDENTICAL to
+        # reconcile_codex_quota_projection's own short-circuit-condition, which
+        # additionally requires the stats-side quota_projection_state signatures
+        # to match the cache certificate for every active root. Read stats.db
+        # HERE — after the Codex flock released in the ``finally`` above — so the
+        # cross-DB read never widens the Codex flock's critical section (the
+        # design invariant documented near the reconcile trigger). A wiped/stale
+        # stats projection (mismatch) forces the reconcile so it self-heals.
+        from _cctally_quota import _stats_projection_signatures_match
+        stats_conn = _cctally_core.open_db()
+        try:
+            if not _stats_projection_signatures_match(
+                stats_conn, deferred_cert_roots, deferred_cert_sigs or {}
+            ):
+                project_after_unlock = True
+        finally:
+            stats_conn.close()
     if project_after_unlock:
         from _cctally_quota import reconcile_codex_quota_projection
         reconcile_codex_quota_projection()
+    if did_from_zero_replay:
+        _force_retention_prune_after_replay(conn)
     return stats
 
 
@@ -4253,6 +4640,41 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
             )
         return 0
 
+    # --prune-conversations: on-demand, UNTHROTTLED transcript retention prune
+    # (#313 P3). Removes >retention-day conversation transcripts (re-derivable
+    # from JSONL). Reclaim the freed disk space with `cctally db vacuum`.
+    if getattr(args, "prune_conversations", False):
+        from _cctally_config import resolve_retention_days
+        import _lib_conversation_retention as retention
+        retention_days = resolve_retention_days(_cctally().load_config())
+        if retention_days <= 0:
+            eprint(
+                "[cache-sync] transcript retention is disabled "
+                "(conversation.retention_days=0); nothing pruned."
+            )
+            return 0
+        result = retention._maybe_prune_conversation_retention(
+            conn,
+            now_utc=dt.datetime.now(dt.timezone.utc),
+            retention_days=retention_days,
+            force=True,
+        )
+        if result is None:
+            eprint(
+                "[cache-sync] prune-conversations skipped: another process "
+                "holds the maintenance or a provider lock; retry shortly."
+            )
+            return 1
+        eprint(
+            f"[cache-sync] pruned transcripts older than {retention_days}d: "
+            f"claude {result.claude_sessions} session(s) / "
+            f"{result.claude_messages} message(s), "
+            f"codex {result.codex_conversations} conversation(s) / "
+            f"{result.codex_events} event(s). "
+            f"Run `cctally db vacuum` to reclaim the freed space."
+        )
+        return 0
+
     # Note: when --rebuild is set we delegate the DELETE to sync_cache /
     # sync_codex_cache, which execute it AFTER acquiring the flock. A
     # pre-sync DELETE here would wipe the cache even when the subsequent