cctally 1.28.0 → 1.30.0

package/bin/_cctally_db.py

@@ -569,7 +569,14 @@ def _recover_version_ahead(
                         ("schema_migrations_skipped", skipped)):
         try:
             for row in conn.execute(f"SELECT name FROM {table}").fetchall():
-                dest.add(row[0])
+                # Normalize legacy unprefixed markers to their canonical NNN_
+                # name (issue #148). The alias union above keeps such a row from
+                # being trimmed; without this normalization the membership test
+                # below compares canonical m.name against the legacy alias and
+                # falsely concludes the migration is missing, resetting
+                # user_version to 0 and forcing a needless full re-walk. Mirrors
+                # the alias-aware read in cmd_db_status.
+                dest.add(aliases.get(row[0], row[0]))
         except sqlite3.OperationalError:
             pass
 
@@ -2401,18 +2408,11 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
             conn.execute(
                 "CREATE VIRTUAL TABLE IF NOT EXISTS conversation_fts "
                 "USING fts5(text, content='conversation_messages', content_rowid='id')")
-            conn.execute(
-                "CREATE TRIGGER IF NOT EXISTS conv_fts_ai AFTER INSERT ON conversation_messages "
-                "BEGIN INSERT INTO conversation_fts(rowid, text) VALUES (new.id, new.text); END")
-            conn.execute(
-                "CREATE TRIGGER IF NOT EXISTS conv_fts_ad AFTER DELETE ON conversation_messages "
-                "BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text) "
-                "VALUES('delete', old.id, old.text); END")
-            conn.execute(
-                "CREATE TRIGGER IF NOT EXISTS conv_fts_au AFTER UPDATE OF text ON conversation_messages "
-                "BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text) "
-                "VALUES('delete', old.id, old.text); "
-                "INSERT INTO conversation_fts(rowid, text) VALUES (new.id, new.text); END")
+            # Trigger DDL lives in ONE place (_CONV_FTS_TRIGGER_DDL) so this
+            # initial create and the #138 storm-free full-clear
+            # (clear_conversation_messages, which drops + recreates the
+            # triggers) can never drift.
+            _create_conversation_fts_triggers(conn)
             if recovering:
                 # Repopulate the freshly-(re)created index from the base table
                 # so pre-recovery history is searchable. Cheap no-op when
@@ -2422,14 +2422,11 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
             conn.execute("DELETE FROM cache_meta WHERE key='fts5_unavailable'")
         except sqlite3.OperationalError:
             # partial create cleanup, then mark unavailable
-            for stmt in ("DROP TRIGGER IF EXISTS conv_fts_au",
-                         "DROP TRIGGER IF EXISTS conv_fts_ad",
-                         "DROP TRIGGER IF EXISTS conv_fts_ai",
-                         "DROP TABLE IF EXISTS conversation_fts"):
-                try:
-                    conn.execute(stmt)
-                except sqlite3.OperationalError:
-                    pass
+            _drop_conversation_fts_triggers(conn)
+            try:
+                conn.execute("DROP TABLE IF EXISTS conversation_fts")
+            except sqlite3.OperationalError:
+                pass
             _set_cache_meta(conn, "fts5_unavailable", "1")
     else:
         # FTS5 is unavailable on THIS sqlite build. If a prior (FTS-capable)
@@ -2441,13 +2438,7 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
         # writes succeed under the LIKE fallback. (The conversation_fts vtable
         # itself can't be DROPped without the fts5 module, but with no triggers
         # nothing writes to it.)
-        for stmt in ("DROP TRIGGER IF EXISTS conv_fts_au",
-                     "DROP TRIGGER IF EXISTS conv_fts_ad",
-                     "DROP TRIGGER IF EXISTS conv_fts_ai"):
-            try:
-                conn.execute(stmt)
-            except sqlite3.OperationalError:
-                pass
+        _drop_conversation_fts_triggers(conn)
         _set_cache_meta(conn, "fts5_unavailable", "1")
     # The FTS branch above issues DML (DELETE/INSERT on cache_meta) which opens
     # an implicit transaction under sqlite3's legacy autocommit mode. Close it
@@ -2475,6 +2466,93 @@ def _set_cache_meta(conn: sqlite3.Connection, key: str, value: str) -> None:
                  "ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, value))
 
 
+# Conversation FTS sync triggers (external-content FTS5). Defined ONCE here so
+# the initial create in _apply_cache_schema and the #138 storm-free full-clear
+# in clear_conversation_messages (which drops + recreates them) can never drift.
+# conv_fts_ad / conv_fts_au use the external-content `'delete'` idiom.
+_CONV_FTS_TRIGGER_DDL = (
+    "CREATE TRIGGER IF NOT EXISTS conv_fts_ai AFTER INSERT ON conversation_messages "
+    "BEGIN INSERT INTO conversation_fts(rowid, text) VALUES (new.id, new.text); END",
+    "CREATE TRIGGER IF NOT EXISTS conv_fts_ad AFTER DELETE ON conversation_messages "
+    "BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text) "
+    "VALUES('delete', old.id, old.text); END",
+    "CREATE TRIGGER IF NOT EXISTS conv_fts_au AFTER UPDATE OF text ON conversation_messages "
+    "BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text) "
+    "VALUES('delete', old.id, old.text); "
+    "INSERT INTO conversation_fts(rowid, text) VALUES (new.id, new.text); END",
+)
+# Drop by name (the body is irrelevant to DROP TRIGGER); reverse order is
+# cosmetic — order doesn't matter for independent triggers.
+_CONV_FTS_TRIGGER_NAMES = ("conv_fts_au", "conv_fts_ad", "conv_fts_ai")
+
+
+def _create_conversation_fts_triggers(conn: sqlite3.Connection) -> None:
+    """Create the three external-content FTS5 sync triggers (idempotent —
+    each is ``IF NOT EXISTS``). Single source of truth for the trigger DDL,
+    shared by ``_apply_cache_schema`` and ``clear_conversation_messages``
+    (#138). The caller must have already created ``conversation_fts``."""
+    for stmt in _CONV_FTS_TRIGGER_DDL:
+        conn.execute(stmt)
+
+
+def _drop_conversation_fts_triggers(conn: sqlite3.Connection) -> None:
+    """Drop the three FTS5 sync triggers (idempotent — ``IF EXISTS``). Swallows
+    ``OperationalError`` per statement so a partial/absent trigger set (e.g. an
+    FTS-unavailable build) is tolerated."""
+    for name in _CONV_FTS_TRIGGER_NAMES:
+        try:
+            conn.execute(f"DROP TRIGGER IF EXISTS {name}")
+        except sqlite3.OperationalError:
+            pass
+
+
+def clear_conversation_messages(conn: sqlite3.Connection) -> None:
+    """Full-clear ``conversation_messages`` + its FTS index WITHOUT firing the
+    per-row delete trigger O(rows) (#138).
+
+    A bulk ``DELETE FROM conversation_messages`` fires ``conv_fts_ad`` once per
+    row — each an FTS5 ``'delete'`` shadow-write — AND forfeits SQLite's
+    no-trigger truncate fast-path, stalling the held ``cache.db.lock`` far
+    longer than the ``session_entries`` clear alone. We suppress the triggers:
+
+        drop all 3 conv_fts triggers
+          → DELETE FROM conversation_messages   (true truncate fast-path now)
+          → INSERT INTO conversation_fts(conversation_fts) VALUES('delete-all')
+                                                (resets the external-content index)
+          → recreate all 3 triggers
+
+    Ordering is load-bearing: clearing the FTS index while the per-row delete
+    trigger is still live makes the base ``DELETE`` write ``'delete'`` postings
+    against already-gone rows and CORRUPTS the index (``database disk image is
+    malformed``; verified on SQLite 3.53.1). Dropping the triggers first makes
+    the base ``DELETE`` not touch the index at all; the explicit ``'delete-all'``
+    then resets it cleanly and ``integrity-check`` still passes.
+
+    Runs inside the caller's open transaction (the held ``cache.db.lock``); the
+    caller owns the commit. When FTS5 is unavailable
+    (``cache_meta.fts5_unavailable`` set → no triggers, no usable vtable),
+    falls back to a plain base ``DELETE`` — there are no triggers to storm and a
+    ``'delete-all'`` would error on the absent vtable."""
+    try:
+        fts_unavailable = conn.execute(
+            "SELECT 1 FROM cache_meta WHERE key='fts5_unavailable'"
+        ).fetchone() is not None
+    except sqlite3.OperationalError:
+        # No cache_meta yet — only possible before the schema is applied, in
+        # which case there is no FTS vtable/triggers either. Bias to the plain
+        # DELETE: it can't storm what doesn't exist and won't touch a vtable.
+        fts_unavailable = True
+
+    if fts_unavailable:
+        conn.execute("DELETE FROM conversation_messages")
+        return
+
+    _drop_conversation_fts_triggers(conn)
+    conn.execute("DELETE FROM conversation_messages")
+    conn.execute("INSERT INTO conversation_fts(conversation_fts) VALUES('delete-all')")
+    _create_conversation_fts_triggers(conn)
+
+
 def _eagerly_apply_cache_migrations() -> None:
     """Open cache.db so its pending migrations (notably
     ``001_dedup_highest_wins``) apply BEFORE stats migration 008's gate
@@ -2913,6 +2991,24 @@ def _002_conversation_messages_backfill(conn: sqlite3.Connection) -> None:
     conn.commit()
 
 
+@cache_migration("003_conversation_reingest_tool_ids")
+def _003_conversation_reingest_tool_ids(conn: sqlite3.Connection) -> None:
+    """Flag-only re-ingest of conversation_messages so tool_use.id /
+    tool_result.tool_use_id / preview land on existing history (#164).
+
+    The destructive clear + offset-0 backfill run in sync_cache UNDER the
+    cache.db.lock flock — NOT here. Clearing in the handler would violate the
+    lock discipline cache-001 follows and would empty the reader on
+    stats-only / eager-migration opens or ``dashboard --no-sync``. A distinct
+    flag from 002's conversation_backfill_pending: 002 = backfill-without-clear;
+    003 = clear-then-backfill. The dispatcher stamps this migration centrally
+    on the existing-install path (issue #140); a fresh install stamps it
+    without running (empty table), and the flag — if ever set — is a harmless
+    no-op there."""
+    _set_cache_meta(conn, "conversation_reingest_pending", "1")
+    conn.commit()
+
+
 # === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
 
 @stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
@@ -3775,9 +3871,16 @@ def _migration_budget_milestone_period_keys(conn: sqlite3.Connection) -> None:
         }
         return "period" in cols
 
-    # Compute needs-rebuild BEFORE any transaction (no deferred-BEGIN-then-read
-    # on stats.db — SQLITE_BUSY_SNAPSHOT, migrations-gotchas.md).
-    pending = [s for s in specs if not _has_period(s[0])]
+    def _table_exists(table: str) -> bool:
+        return conn.execute(
+            "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
+        ).fetchone() is not None
+
+    # Compute needs-rebuild BEFORE any transaction (no deferred-BEGIN-then-read on
+    # stats.db — SQLITE_BUSY_SNAPSHOT, migrations-gotchas.md). A spec table that
+    # does not exist (e.g. codex_budget_milestones on a DB predating that feature,
+    # now that v012 no longer live-creates it — #143) needs no period column.
+    pending = [s for s in specs if _table_exists(s[0]) and not _has_period(s[0])]
 
     if not pending:
         # Fresh install (live CREATE already made the new shape) or prior run.
@@ -3800,6 +3903,83 @@ def _migration_budget_milestone_period_keys(conn: sqlite3.Connection) -> None:
         raise
 
 
+@stats_migration("012_unify_budget_milestones_vendor")
+def _migration_unify_budget_milestones_vendor(conn: sqlite3.Connection) -> None:
+    """Merge ``codex_budget_milestones`` into a vendor-tagged ``budget_milestones``
+    (issue #143).
+
+    ``budget_milestones`` (Claude, keyed ``week_start_at``) and
+    ``codex_budget_milestones`` (Codex, keyed ``period_start_at``) are
+    structurally identical modulo vendor + key-column name. This migration
+    rebuilds ``budget_milestones`` with a ``vendor`` column and the renamed
+    ``period_start_at`` key, copies Claude rows (``week_start_at``->``period_start_at``,
+    ``vendor='claude'``) and Codex rows (``vendor='codex'``), and drops the Codex
+    table. History + ``alerted_at`` + ``period`` are preserved verbatim; the
+    write-once ``period`` NULL sentinel is carried as-is. ``id`` is NOT copied
+    (AUTOINCREMENT reassigns — the envelope/dispatch ids are composite strings,
+    never the row PK).
+
+    State machine (idempotent / partial-state safe): the Claude rebuild and the
+    Codex absorb are independently guarded, so a retry after a crash-before-stamp
+    (table already unified, Codex maybe gone) is a clean no-op or a Codex-only
+    absorb. Reads happen BEFORE BEGIN IMMEDIATE (SQLITE_BUSY_SNAPSHOT).
+    """
+    def _cols(table: str) -> set:
+        return {str(r[1]) for r in conn.execute(f"PRAGMA table_info({table})").fetchall()}
+
+    def _table_exists(table: str) -> bool:
+        return conn.execute(
+            "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
+        ).fetchone() is not None
+
+    claude_needs_rebuild = "vendor" not in _cols("budget_milestones")
+    codex_present = _table_exists("codex_budget_milestones")
+    if not claude_needs_rebuild and not codex_present:
+        return  # already unified, no Codex leftover -> dispatcher fast-stamps
+
+    new_table = """
+        CREATE TABLE budget_milestones (
+            id              INTEGER PRIMARY KEY AUTOINCREMENT,
+            vendor          TEXT    NOT NULL,
+            period_start_at TEXT    NOT NULL,
+            period          TEXT,
+            threshold       INTEGER NOT NULL,
+            budget_usd      REAL    NOT NULL,
+            spent_usd       REAL    NOT NULL,
+            consumption_pct REAL    NOT NULL,
+            crossed_at_utc  TEXT    NOT NULL,
+            alerted_at      TEXT,
+            UNIQUE(vendor, period_start_at, period, threshold)
+        )
+    """
+    cols = ("vendor, period_start_at, period, threshold, budget_usd, spent_usd, "
+            "consumption_pct, crossed_at_utc, alerted_at")
+    conn.execute("BEGIN IMMEDIATE")
+    try:
+        if claude_needs_rebuild:
+            conn.execute("ALTER TABLE budget_milestones RENAME TO budget_milestones_old_012")
+            conn.execute(new_table)
+            conn.execute(
+                f"INSERT INTO budget_milestones ({cols}) "
+                "SELECT 'claude', week_start_at, period, threshold, budget_usd, "
+                "spent_usd, consumption_pct, crossed_at_utc, alerted_at "
+                "FROM budget_milestones_old_012"
+            )
+            conn.execute("DROP TABLE budget_milestones_old_012")
+        if codex_present:
+            conn.execute(
+                f"INSERT INTO budget_milestones ({cols}) "
+                "SELECT 'codex', period_start_at, period, threshold, budget_usd, "
+                "spent_usd, consumption_pct, crossed_at_utc, alerted_at "
+                "FROM codex_budget_milestones"
+            )
+            conn.execute("DROP TABLE codex_budget_milestones")
+        conn.commit()
+    except Exception:
+        conn.rollback()
+        raise
+
+
 # === Region 8: Test-only migration registration (was bin/cctally:12086-12140) ===
 
 # ──────────────────────────────────────────────────────────────────────