cctally 1.70.0 → 1.72.0

package/bin/_cctally_db.py

@@ -876,7 +876,16 @@ def _run_pending_migrations(
             # absent must still be treated as non-fresh so a flag-only
             # conversation migration's consumer actually runs (e.g. 011's
             # command-args promotion) instead of being stamped without it.
-            "cache.db": ("session_entries", "conversation_messages"),
+            # codex_conversation_events joins them (#294 S6): a Codex-bearing
+            # cache written by S1's fused ingest but missing schema_migrations
+            # would otherwise be classified fresh and stamp 025 WITHOUT replaying
+            # normalization, leaving the read kernels authoritative over an empty
+            # normalized corpus. Probing it forces 025's handler to run and derive
+            # the normalized rows from the retained events.
+            "cache.db": (
+                "session_entries", "conversation_messages",
+                "codex_conversation_events",
+            ),
         }.get(db_label, ())
         for probe_table in probe_tables:
             # _probe_table_nonempty centralizes the "is there data here?"
@@ -2621,6 +2630,81 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
         CREATE INDEX IF NOT EXISTS idx_codex_events_timestamp
             ON codex_conversation_events(timestamp_utc);
 
+        -- #294 S6: normalized Codex conversation storage. Like the S1 tables
+        -- above, these live in the UNCONDITIONAL base-schema script — before the
+        -- legacy-FTS topology checks below — so an existing cache gains them on
+        -- every open. The independent Codex FTS layer
+        -- (_apply_codex_conversation_fts) stands up codex_conversation_fts + its
+        -- own triggers separately, BEFORE the Claude legacy-FTS early-return, so
+        -- a legacy-shape Claude cache still gets the Codex search index.
+        --
+        -- codex_conversation_messages: one normalized row per NORMALIZED physical
+        -- event (§3.1). ``id`` is a PLAIN rowid alias (deliberately NOT
+        -- AUTOINCREMENT): migration 025 replays inserts in deterministic
+        -- (source_path ASC, line_offset ASC) order, so a repeated run is
+        -- byte-idempotent (no sqlite_sequence drift), while fresh ingest inserts
+        -- in discovery order — fresh-vs-migrated equality is asserted
+        -- SEMANTICALLY (row content modulo id).
+        CREATE TABLE IF NOT EXISTS codex_conversation_messages (
+            id               INTEGER PRIMARY KEY,
+            conversation_key TEXT NOT NULL,
+            source_root_key  TEXT NOT NULL,
+            source_path      TEXT NOT NULL,
+            line_offset      INTEGER NOT NULL,
+            timestamp_utc    TEXT,
+            turn_id          TEXT,
+            call_id          TEXT,
+            kind             TEXT NOT NULL,
+            event_type       TEXT,
+            record_family    TEXT NOT NULL,
+            model            TEXT,
+            text             TEXT,
+            content_digest   TEXT NOT NULL,
+            content_len      INTEGER NOT NULL CHECK(content_len >= 0),
+            detail_json      TEXT,
+            search_tool      TEXT,
+            search_thinking  TEXT,
+            UNIQUE(source_path, line_offset)
+        );
+        CREATE INDEX IF NOT EXISTS idx_codex_conv_msgs_conversation
+            ON codex_conversation_messages(conversation_key, timestamp_utc, id);
+        CREATE INDEX IF NOT EXISTS idx_codex_conv_msgs_source
+            ON codex_conversation_messages(source_path);
+
+        -- codex_conversation_rollups: browse-rail materialization (§3.2). A pure
+        -- function of surviving codex_conversation_messages (+ thread metadata),
+        -- recomputed-affected-or-deleted after every write/delete path. item_count
+        -- counts rendered LOGICAL items (mirror-paired), not physical rows.
+        CREATE TABLE IF NOT EXISTS codex_conversation_rollups (
+            conversation_key  TEXT NOT NULL PRIMARY KEY,
+            source_root_key   TEXT NOT NULL,
+            parent_thread_id  TEXT,
+            item_count        INTEGER NOT NULL DEFAULT 0,
+            started_utc       TEXT,
+            last_activity_utc TEXT,
+            project_key       TEXT,
+            project_label     TEXT,
+            models_json       TEXT,
+            title             TEXT
+        );
+        CREATE INDEX IF NOT EXISTS idx_codex_conv_rollups_recent
+            ON codex_conversation_rollups(last_activity_utc DESC, conversation_key DESC);
+
+        -- codex_conversation_file_touches: write-class axis for the `files`
+        -- search kind + outline file stats (§3.3). source_path gives explicit
+        -- lineage so the per-file delete/truncate/prune paths scope deletions
+        -- exactly as they do for the other Codex families.
+        CREATE TABLE IF NOT EXISTS codex_conversation_file_touches (
+            message_id       INTEGER NOT NULL,
+            conversation_key TEXT NOT NULL,
+            source_path      TEXT NOT NULL,
+            file_path        TEXT NOT NULL,
+            tool             TEXT NOT NULL,
+            UNIQUE(message_id, file_path, tool)
+        );
+        CREATE INDEX IF NOT EXISTS idx_codex_conv_touches_source
+            ON codex_conversation_file_touches(source_path);
+
         CREATE TABLE IF NOT EXISTS cache_meta (
             key   TEXT PRIMARY KEY,
             value TEXT
@@ -2662,6 +2746,10 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
     add_column_if_missing(conn, "codex_session_files", "last_root_thread_id", "TEXT")
     add_column_if_missing(conn, "codex_session_files", "last_parent_thread_id", "TEXT")
     add_column_if_missing(conn, "codex_session_files", "last_conversation_key", "TEXT")
+    # #294 S6: the terminal sticky-turn seed for delta resumes, alongside the
+    # existing last_model/thread facts. A batch that ends after a turn_context
+    # and resumes with a response_item stamps the correct effective turn.
+    add_column_if_missing(conn, "codex_session_files", "last_turn_id", "TEXT")
     conn.execute(
         "CREATE INDEX IF NOT EXISTS idx_codex_files_source_root "
         "ON codex_session_files(source_root_key)"
@@ -2761,6 +2849,12 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
         "CREATE INDEX IF NOT EXISTS idx_session_files_session_id "
         "ON session_files(session_id)"
     )
+    # #294 S6: the independent Codex conversation FTS lifecycle. Run BEFORE (and
+    # independent of) the Claude FTS branch below — the Claude branch early-returns
+    # on a legacy conversation_fts(text) shape, so the Codex index must be stood up
+    # here or a legacy-shape cache would never gain Codex search. Never touches any
+    # Claude FTS object (conv_fts_* / conv_title_fts_*).
+    _apply_codex_conversation_fts(conn)
     # FTS5 is optional in the sqlite build. Create the external-content index +
     # sync triggers as separate executes wrapped in one try; on failure create
     # NEITHER the table NOR the triggers (a trigger referencing a missing table
@@ -3188,6 +3282,142 @@ def clear_conversation_messages(conn: sqlite3.Connection) -> None:
     _create_conversation_fts_triggers(conn)
 
 
+# ── #294 S6: independent Codex conversation FTS lifecycle (§3.4) ──────────────
+# The Codex normalized search index owns its OWN external-content FTS5 table +
+# AI/AD/AU trigger trio, entirely separate from the Claude conversation_fts /
+# conversation_title_fts objects. _apply_codex_conversation_fts runs from
+# _apply_cache_schema BEFORE the Claude legacy-FTS early-return, so a legacy-shape
+# Claude cache still gains it. The Codex-scoped `codex_fts_unavailable` cache_meta
+# marker is INDEPENDENT of the Claude `fts5_unavailable` marker. Column names
+# match codex_conversation_messages BY NAME (external-content FTS5 rule).
+_CODEX_CONV_FTS_DDL = (
+    "CREATE VIRTUAL TABLE IF NOT EXISTS codex_conversation_fts USING fts5("
+    "text, search_tool, search_thinking, "
+    "content='codex_conversation_messages', content_rowid='id')"
+)
+# Trigger DDL lives in ONE place so the initial create and the storm-free
+# full-clear (which drops + recreates the trigger set) can never drift. AD/AU use
+# the external-content `'delete'` idiom; AU fires AFTER UPDATE OF the three
+# indexed columns.
+_CODEX_CONV_FTS_TRIGGER_DDL = (
+    "CREATE TRIGGER IF NOT EXISTS codex_conv_fts_ai AFTER INSERT ON codex_conversation_messages "
+    "BEGIN INSERT INTO codex_conversation_fts(rowid, text, search_tool, search_thinking) "
+    "VALUES (new.id, new.text, new.search_tool, new.search_thinking); END",
+    "CREATE TRIGGER IF NOT EXISTS codex_conv_fts_ad AFTER DELETE ON codex_conversation_messages "
+    "BEGIN INSERT INTO codex_conversation_fts(codex_conversation_fts, rowid, text, search_tool, search_thinking) "
+    "VALUES('delete', old.id, old.text, old.search_tool, old.search_thinking); END",
+    "CREATE TRIGGER IF NOT EXISTS codex_conv_fts_au "
+    "AFTER UPDATE OF text, search_tool, search_thinking ON codex_conversation_messages "
+    "BEGIN INSERT INTO codex_conversation_fts(codex_conversation_fts, rowid, text, search_tool, search_thinking) "
+    "VALUES('delete', old.id, old.text, old.search_tool, old.search_thinking); "
+    "INSERT INTO codex_conversation_fts(rowid, text, search_tool, search_thinking) "
+    "VALUES (new.id, new.text, new.search_tool, new.search_thinking); END",
+)
+_CODEX_CONV_FTS_TRIGGER_NAMES = ("codex_conv_fts_au", "codex_conv_fts_ad", "codex_conv_fts_ai")
+
+
+def _create_codex_conversation_fts_triggers(conn: sqlite3.Connection) -> None:
+    """Create the Codex FTS sync trigger set — idempotent (each ``IF NOT EXISTS``).
+    Single source of truth, shared by ``_apply_codex_conversation_fts`` and the
+    storm-free full-clear. The caller must have already created the vtable."""
+    for stmt in _CODEX_CONV_FTS_TRIGGER_DDL:
+        conn.execute(stmt)
+
+
+def _drop_codex_conversation_fts_triggers(conn: sqlite3.Connection) -> None:
+    """Drop the Codex FTS sync trigger set — idempotent (``IF EXISTS``). Swallows
+    ``OperationalError`` per statement so an absent set (FTS5-unavailable build) is
+    tolerated. Dropping these on a no-FTS5 build is what keeps a normalized INSERT
+    from firing a trigger against the missing vtable and rolling back the shared
+    per-file ingest transaction. NEVER touches any Claude FTS trigger."""
+    for name in _CODEX_CONV_FTS_TRIGGER_NAMES:
+        try:
+            conn.execute(f"DROP TRIGGER IF EXISTS {name}")
+        except sqlite3.OperationalError:
+            pass
+
+
+def _apply_codex_conversation_fts(conn: sqlite3.Connection) -> None:
+    """Stand up / recover / degrade the independent Codex conversation FTS (§3.4).
+
+    Four states: create (FTS5 available, no marker) → codex_conversation_fts +
+    triggers; unavailable-at-creation → set ``codex_fts_unavailable`` + skip DDL;
+    capable→unavailable reopen → drop ONLY the Codex triggers + set the marker;
+    recovery (FTS5 available again, marker set) → drop/recreate the vtable + its
+    triggers, ``rebuild`` from the base rows, clear the marker. Never touches any
+    Claude FTS object."""
+    if _fts5_available(conn):
+        recovering = conn.execute(
+            "SELECT 1 FROM cache_meta WHERE key='codex_fts_unavailable'"
+        ).fetchone() is not None
+        try:
+            if recovering:
+                # A prior FTS5-unavailable run ingested normalized rows without a
+                # live trigger (or a capable→unavailable→capable cycle left a stale
+                # vtable). Drop + recreate the index and its triggers, then rebuild
+                # from the base rows.
+                _drop_codex_conversation_fts_triggers(conn)
+                conn.execute("DROP TABLE IF EXISTS codex_conversation_fts")
+            conn.execute(_CODEX_CONV_FTS_DDL)
+            _create_codex_conversation_fts_triggers(conn)
+            if recovering:
+                conn.execute(
+                    "INSERT INTO codex_conversation_fts(codex_conversation_fts) VALUES('rebuild')")
+            conn.execute("DELETE FROM cache_meta WHERE key='codex_fts_unavailable'")
+        except sqlite3.OperationalError:
+            # Partial create cleanup, then mark unavailable so search uses LIKE.
+            _drop_codex_conversation_fts_triggers(conn)
+            try:
+                conn.execute("DROP TABLE IF EXISTS codex_conversation_fts")
+            except sqlite3.OperationalError:
+                pass
+            _set_cache_meta(conn, "codex_fts_unavailable", "1")
+    else:
+        # FTS5 unavailable on THIS build. If a prior capable run left the Codex
+        # triggers, they now reference an unusable vtable and every normalized
+        # INSERT would fail and roll back the shared per-file transaction — so
+        # drop ONLY the Codex triggers (the vtable can't be dropped without fts5,
+        # but with no triggers nothing writes to it) and mark unavailable.
+        _drop_codex_conversation_fts_triggers(conn)
+        _set_cache_meta(conn, "codex_fts_unavailable", "1")
+
+
+def _codex_conversation_fts_full_clear(conn: sqlite3.Connection) -> None:
+    """Storm-free FULL clear of every Codex normalized derived table (§3.4).
+
+    Drops the Codex FTS triggers, truncates codex_conversation_messages (the
+    no-trigger fast path), resets the external-content index via ``'delete-all'``,
+    recreates the triggers, then clears file-touches + rollups. ``'delete-all'`` is
+    valid here precisely because the WHOLE normalized corpus empties; the sequence
+    makes repeated runs byte-idempotent at the FTS shadow-table level (migration
+    025 re-run, cache-rebuild). Partial deletes (per-file truncation, root-set
+    orphan prune) MUST NOT use this — they ride the per-row delete triggers so
+    surviving conversations keep their postings. Falls back to a plain base DELETE
+    when FTS5 is unavailable (no triggers, no usable vtable)."""
+    try:
+        fts_unavailable = conn.execute(
+            "SELECT 1 FROM cache_meta WHERE key='codex_fts_unavailable'"
+        ).fetchone() is not None
+    except sqlite3.OperationalError:
+        fts_unavailable = True
+    if fts_unavailable:
+        conn.execute("DELETE FROM codex_conversation_messages")
+    else:
+        _drop_codex_conversation_fts_triggers(conn)
+        conn.execute("DELETE FROM codex_conversation_messages")
+        conn.execute(
+            "INSERT INTO codex_conversation_fts(codex_conversation_fts) VALUES('delete-all')")
+        _create_codex_conversation_fts_triggers(conn)
+    for stmt in (
+        "DELETE FROM codex_conversation_file_touches",
+        "DELETE FROM codex_conversation_rollups",
+    ):
+        try:
+            conn.execute(stmt)
+        except sqlite3.OperationalError:
+            pass  # table not yet created (pre-schema conn); nothing to clear
+
+
 def _eagerly_apply_cache_migrations() -> None:
     """Open cache.db so its pending migrations (notably
     ``001_dedup_highest_wins``) apply BEFORE stats migration 008's gate
@@ -4278,6 +4508,68 @@ def _024_codex_fused_ingest_rebuild(conn: sqlite3.Connection) -> None:
             conn.execute("DELETE FROM codex_conversation_threads")
             conn.execute("DELETE FROM codex_conversation_events")
             conn.execute("DELETE FROM codex_source_roots")
+            # F3 (#313): clearing Codex quota state without invalidating the
+            # quota-projection certificate would leave a stale-valid cert
+            # (cache sequence unchanged) that lets the reconcile short-circuit
+            # skip over now-deleted data. Delete it in the same transaction.
+            conn.execute(
+                "DELETE FROM cache_meta WHERE key='codex_quota_projection_certificate'"
+            )
+            conn.commit()
+        except Exception:
+            conn.rollback()
+            raise
+    finally:
+        if lock_fh is not None:
+            try:
+                fcntl.flock(lock_fh, fcntl.LOCK_UN)
+            except OSError:
+                pass
+            lock_fh.close()
+
+
+@cache_migration("025_codex_conversation_normalization")
+def _025_codex_conversation_normalization(conn: sqlite3.Connection) -> None:
+    """Derive S6 normalized conversation state for caches whose Codex events were
+    ingested before S6 (#294 S6, §3.5).
+
+    Clears the three derived tables (codex_conversation_messages,
+    codex_conversation_file_touches, codex_conversation_rollups) via the §3.4
+    storm-free full-clear helper, then replays the pure normalization kernel over
+    the stored ``codex_conversation_events`` per file in
+    ``(source_path ASC, line_offset ASC)`` order — rebuilding session_meta /
+    turn_context sticky state as it goes. The physical event log and Codex
+    accounting/thread/quota rows are the replay SOURCE and are never touched.
+
+    Takes ``cache.db.codex.lock`` with the same fcntl-then-``BEGIN IMMEDIATE``
+    order as ``sync_codex_cache``, raising ``MigrationGateNotMet`` on contention
+    (defer, never partially normalize) from both the ordinary and eager cache
+    dispatch paths. Deterministic replay order + the non-AUTOINCREMENT ``id``
+    rowid alias + the storm-free full-clear make a re-run (handler success, crash
+    before the central stamp) byte-idempotent INCLUDING the FTS shadow tables.
+    Never self-stamps; the dispatcher owns ``schema_migrations`` and
+    ``user_version``.
+    """
+    lock_path = _cache_db_codex_lock_path_for_conn(conn)
+    lock_fh = None
+    if lock_path is not None:
+        lock_fh = open(lock_path, "w")
+        try:
+            fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            lock_fh.close()
+            raise MigrationGateNotMet(
+                "cache.db.codex.lock held by a concurrent sync_codex_cache; "
+                "deferring cache 025 conversation normalization (#294 S6)"
+            )
+    try:
+        conn.execute("BEGIN IMMEDIATE")
+        try:
+            # Full clear of only the DERIVED normalized tables (the event log
+            # remains as the replay source), then re-derive from it.
+            _codex_conversation_fts_full_clear(conn)
+            import _cctally_cache
+            _cctally_cache._replay_codex_normalization(conn)
             conn.commit()
         except Exception:
             conn.rollback()
@@ -5802,3 +6094,155 @@ def cmd_db_checkpoint(args: argparse.Namespace) -> int:
         print(f"cctally: {result.db} WAL {mb_b:.1f} MB -> {mb_a:.1f} MB "
               f"({result.frames_checkpointed} frames; {state}).")
     return 0 if result.truncated else 3
+
+
+# VACUUM writes a full fresh copy of the database into a temporary file and then
+# swaps it in, so it transiently needs roughly the DB's own size on top of the
+# existing file, plus room for the drained WAL. A short busy_timeout keeps a
+# contended VACUUM from hanging (F13).
+_VACUUM_BUSY_TIMEOUT_MS = 250
+
+
+def _free_disk_bytes(directory) -> int:
+    """Free bytes on the filesystem holding ``directory`` (mockable in tests)."""
+    import shutil
+    return shutil.disk_usage(str(directory)).free
+
+
+def _vacuum_required_free_bytes(path) -> int:
+    """Conservative free-space floor to VACUUM ``path``: ~2x the DB file plus its
+    current WAL sidecar (F13 — the VACUUM temp copy + a WAL/temp margin)."""
+    try:
+        db_bytes = path.stat().st_size
+    except OSError:
+        db_bytes = 0
+    wal = path.parent / (path.name + "-wal")
+    try:
+        wal_bytes = wal.stat().st_size
+    except OSError:
+        wal_bytes = 0
+    return 2 * db_bytes + wal_bytes
+
+
+def _run_vacuum_exclusive(path, label: str) -> int:
+    """Checkpoint + VACUUM ``path`` under a real SQLite EXCLUSIVE lock (F13).
+
+    ``locking_mode=EXCLUSIVE`` + a short ``busy_timeout`` make a concurrent
+    reader/writer FAIL PROMPTLY (no TOCTOU gap — the exclusion is the DB's own
+    lock, which the advisory flocks do not provide against dashboard readers).
+    Exit 0 on success, 3 when the DB is in use."""
+    conn = sqlite3.connect(f"file:{path}?mode=rw", uri=True)
+    try:
+        conn.execute(f"PRAGMA busy_timeout={_VACUUM_BUSY_TIMEOUT_MS}")
+        page_size = conn.execute("PRAGMA page_size").fetchone()[0]
+        before = conn.execute("PRAGMA page_count").fetchone()[0]
+        try:
+            conn.execute("PRAGMA locking_mode=EXCLUSIVE")
+            conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+            conn.execute("VACUUM")
+        except sqlite3.OperationalError as exc:
+            if "lock" in str(exc).lower() or "busy" in str(exc).lower():
+                eprint(
+                    f"cctally: {label} is in use — VACUUM needs exclusive access. "
+                    f"Stop the dashboard and any other cctally process holding "
+                    f"{label}, then retry."
+                )
+                return 3
+            raise
+        after = conn.execute("PRAGMA page_count").fetchone()[0]
+        freed_mb = max(0, (before - after)) * page_size / (1024 * 1024)
+        print(
+            f"cctally: {label} reclaimed {freed_mb:.1f} MB "
+            f"({before} -> {after} pages)."
+        )
+        return 0
+    finally:
+        conn.close()
+
+
+def _vacuum_one_db(path, label: str, provider_locked: bool) -> int:
+    if not path.exists():
+        print(f"cctally: no {label} database file present; nothing to reclaim.")
+        return 0
+    needed = _vacuum_required_free_bytes(path)
+    free = _free_disk_bytes(path.parent)
+    if free < needed:
+        eprint(
+            f"cctally: not enough free disk to VACUUM {label}: need ~"
+            f"{needed // (1024 * 1024)} MB free, have {free // (1024 * 1024)} MB. "
+            f"Free up space and retry."
+        )
+        return 3
+    core = _cctally_core
+    try:
+        core.APP_DIR.mkdir(parents=True, exist_ok=True)
+    except OSError:
+        pass
+    # Serialize against the retention prune and concurrent vacuums via the
+    # dedicated maintenance flock (F13/F7), then the provider flocks for
+    # cache.db. All non-blocking: fail promptly rather than hang.
+    maint_fh = open(core.CACHE_LOCK_MAINTENANCE_PATH, "w")
+    try:
+        try:
+            fcntl.flock(maint_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except (BlockingIOError, OSError):
+            eprint(
+                f"cctally: {label} VACUUM skipped: another maintenance operation "
+                f"is running. Retry shortly."
+            )
+            return 3
+        held = []
+        try:
+            if provider_locked:
+                for lock_path, lname in (
+                    (core.CACHE_LOCK_PATH, "claude"),
+                    (core.CACHE_LOCK_CODEX_PATH, "codex"),
+                ):
+                    fh = open(lock_path, "w")
+                    try:
+                        fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                    except (BlockingIOError, OSError):
+                        fh.close()
+                        eprint(
+                            f"cctally: {label} VACUUM skipped: a {lname} sync is in "
+                            f"progress. Stop the dashboard / other cctally "
+                            f"processes and retry."
+                        )
+                        return 3
+                    held.append(fh)
+            return _run_vacuum_exclusive(path, label)
+        finally:
+            for fh in held:
+                try:
+                    fcntl.flock(fh, fcntl.LOCK_UN)
+                except OSError:
+                    pass
+                fh.close()
+    finally:
+        try:
+            fcntl.flock(maint_fh, fcntl.LOCK_UN)
+        except OSError:
+            pass
+        maint_fh.close()
+
+
+def cmd_db_vacuum(args: argparse.Namespace) -> int:
+    """Reclaim disk space via VACUUM after a transcript prune (#313 P3, F13).
+
+    NEVER automatic. Holds the maintenance flock + (for cache.db) the provider
+    flocks, then runs a checkpoint + VACUUM under a real SQLite EXCLUSIVE lock so
+    a concurrent dashboard reader fails promptly instead of racing. Refuses when
+    free disk is below ~2x the file + WAL. Exit 0 on success, 3 when a target is
+    in use or disk is short."""
+    which = getattr(args, "db", "cache")
+    targets = []
+    if which in ("cache", "all"):
+        targets.append((_cctally_core.CACHE_DB_PATH, "cache.db", True))
+    if which in ("stats", "all"):
+        targets.append((_cctally_core.DB_PATH, "stats.db", False))
+    overall = 0
+    for path, label, provider_locked in targets:
+        rc = _vacuum_one_db(path, label, provider_locked)
+        if rc != 0:
+            overall = rc
+    return overall

package/bin/_cctally_parser.py

@@ -2134,6 +2134,14 @@ def _build_cache_sync_parser(subparsers, name, *, help_text, xref=None):
         help="Prune cache rows for source files removed from disk "
              "(e.g. a deleted git worktree) without a full rebuild.",
     )
+    p_cache_sync.add_argument(
+        "--prune-conversations",
+        action="store_true",
+        help="Prune conversation transcripts older than "
+             "conversation.retention_days (default 180) now, without a full "
+             "rebuild. Re-derivable from JSONL; run `cctally db vacuum` to "
+             "reclaim the freed disk space.",
+    )
     p_cache_sync.set_defaults(func=c.cmd_cache_sync)
 
 def _build_project_parser(subparsers, name, *, help_text, xref=None, fixed_source=None):
@@ -2802,6 +2810,18 @@ def _build_db_parser(subparsers, name, *, help_text, xref=None):
         help=argparse.SUPPRESS,
     )
     db_checkpoint.set_defaults(func=c.cmd_db_checkpoint)
+    db_vacuum = db_sub.add_parser(
+        "vacuum",
+        help="Reclaim disk space after a transcript prune (VACUUM) — "
+             "exclusive, never automatic",
+    )
+    db_vacuum.add_argument(
+        "--db",
+        choices=("cache", "stats", "all"),
+        default="cache",
+        help="Which DB to VACUUM (default: cache)",
+    )
+    db_vacuum.set_defaults(func=c.cmd_db_vacuum)
 
 def _build_doctor_parser(subparsers, name, *, help_text, xref=None):
     """Build the `doctor` parser (registered via _REGISTRATION; #279 S6 W3).

package/bin/_cctally_quota.py

@@ -183,6 +183,32 @@ def _store_codex_quota_projection_certificate(
         return
 
 
+def _stats_projection_signatures_match(
+    stats_conn: sqlite3.Connection,
+    active_roots: set[str],
+    cert_sigs: Mapping[str, str],
+) -> bool:
+    """True iff stats.db's projection signature matches the certificate for every root.
+
+    The cache certificate alone does not prove stats.db still holds the
+    projection: stats.db can be independently wiped/recovered while cache.db
+    persists (F1).  Require an exact ``quota_projection_state.physical_signature``
+    match for every active root before the reconcile is allowed to short-circuit.
+    A missing row, a mismatch, or any ``sqlite3.Error`` degrades to False, which
+    forces the full reconcile (fail-safe).
+    """
+    try:
+        rows = stats_conn.execute(
+            "SELECT source_root_key, physical_signature FROM quota_projection_state"
+        ).fetchall()
+    except sqlite3.Error:
+        return False
+    projection = {str(row[0]): str(row[1]) for row in rows}
+    return all(
+        projection.get(root) == cert_sigs.get(root) for root in active_roots
+    )
+
+
 def load_codex_quota_observations(
     *,
     source_root_keys: Iterable[str] | None = None,
@@ -754,22 +780,48 @@ def reconcile_codex_quota_projection(
         raise ValueError("now must be timezone-aware")
     now_iso = _utc_iso(now)
 
+    alert_eligible_roots = {str(key) for key in alert_eligible_root_keys}
+
     try:
         cache = _cache_connection()
     except (FileNotFoundError, sqlite3.Error):
         return QuotaProjectionResult(None, 0, 0, 0, 0, 0, 0)
     try:
-        active_roots = (
-            _cache_root_keys(cache)
-            if source_root_keys is None else {str(key) for key in source_root_keys}
-        )
-        physical_sequence = codex_physical_mutation_seq(cache)
+        # F2: read active_roots, the physical sequence, and the certificate
+        # inside ONE WAL read snapshot so a concurrent commit cannot interleave
+        # a stale sequence with a fresh certificate.
+        cache.execute("BEGIN")
+        try:
+            active_roots = (
+                _cache_root_keys(cache)
+                if source_root_keys is None else {str(key) for key in source_root_keys}
+            )
+            physical_sequence = codex_physical_mutation_seq(cache)
+            certificate = load_codex_quota_projection_certificate(cache)
+        finally:
+            cache.commit()
+        # Short-circuit: when nothing is alert-eligible and the certificate
+        # proves the cache physical state is current AND the stats-side
+        # projection still matches it (F1), the ~2.9 s observation load and the
+        # whole reconcile are provably a no-op. Any missed concurrent write
+        # leaves cur_seq != cert_seq (or a stats-signature mismatch) on the next
+        # call, so the scheme is self-healing.
+        if not alert_eligible_roots and certificate is not None:
+            cert_seq, cert_sigs = certificate
+            if physical_sequence == cert_seq and active_roots <= set(cert_sigs):
+                stats_conn = _cctally_core.open_db()
+                try:
+                    if _stats_projection_signatures_match(
+                        stats_conn, active_roots, cert_sigs
+                    ):
+                        return QuotaProjectionResult(None, 0, 0, 0, 0, 0, 0)
+                finally:
+                    stats_conn.close()
         observations = load_codex_quota_observations(
             source_root_keys=active_roots, cache_conn=cache,
         )
     finally:
         cache.close()
-    alert_eligible_roots = {str(key) for key in alert_eligible_root_keys}
 
     # No configured roots and no existing interpreted history means there is no
     # stats work.  This preserves the existing empty-Codex sync fast path.