cctally 1.65.0 → 1.67.0

package/bin/_cctally_db.py

@@ -91,6 +91,11 @@ from _cctally_core import (
     parse_iso_datetime,
 )
 
+# #279 S2 F2: demonstration adopter of the stdlib-logging chokepoint —
+# the migration gate-defer diagnostic routes through _lib_log so
+# CCTALLY_DEBUG verbosity is decided in one place.
+import _lib_log
+
 # Stats migration 008 needs the same per-entry cost computation used by
 # the live cost-report path. Direct import keeps the kernel single-sourced
 # (no shim drift); _lib_pricing is a stdlib-only leaf module so no cycle
@@ -237,6 +242,19 @@ class ProdMigrationRefused(Exception):
         )
 
 
+class StatsDbCorruptError(sqlite3.DatabaseError):
+    """stats.db failed the open-time probe (#279 S1 F4).
+
+    Subclasses ``sqlite3.DatabaseError`` so graceful-degrade sites (doctor,
+    dashboard/TUI background threads, the 5h-anchor fallback) keep treating it
+    as a DB failure exactly as before — but command-level handlers that map DB
+    errors to OTHER exit codes must re-raise it so the global exit-2 diagnosis
+    wins (``cmd_record_credit`` does; its documented DB-error exit is 3). NOT
+    auto-recreated: stats.db is the non-re-derivable DB (recorded usage
+    history), unlike cache.db.
+    """
+
+
 class MigrationGateNotMet(Exception):
     """Migration cannot run yet because a cross-DB prerequisite is unsatisfied.
 
@@ -383,12 +401,44 @@ def _make_migration_decorator(registry: list[Migration], db_label: str, name: st
 
 
 def stats_migration(name: str):
-    """Register a stats.db migration. Use as @stats_migration("NNN_descriptive_name")."""
+    """Register a stats.db migration. Use as @stats_migration("NNN_descriptive_name").
+
+    HANDLER CONTRACT — idempotency is MANDATORY, not best-effort. The
+    dispatcher applies a handler and stamps its ``schema_migrations`` marker in
+    a TWO-TRANSACTION window (``m.handler(conn)`` commits the data change, THEN
+    ``_stamp_applied`` writes the marker in a separate transaction — see
+    ``_run_pending_migrations``). A crash BETWEEN those two commits leaves the
+    data change applied but unmarked, so the NEXT ``open_db()`` re-invokes the
+    handler. A handler that is not idempotent (re-derives from already-migrated
+    state, double-applies, or raises on a second run) corrupts or bricks the DB
+    on that retry. Every handler MUST therefore be safe to run again on its own
+    output: probe-then-return on the already-applied shape (a fast-path guard
+    like ``if 'col' in cols(...): return``), or use ``INSERT OR IGNORE`` /
+    ``UPDATE … WHERE not-yet-migrated`` / upsert idioms. The same contract
+    covers a ``MigrationGateNotMet`` deferral: the dispatcher leaves the
+    migration pending and re-runs it on a later open, which is just another
+    handler retry. Carve-out: cache ``001_dedup_highest_wins`` stamps its marker
+    ATOMICALLY inside its own ``BEGIN IMMEDIATE`` (the destructive wipe and the
+    stamp must commit together), so it has no two-transaction window — but it is
+    still idempotent (its in-txn ``already_applied`` re-check). Per-migration
+    goldens (``tests/test_*_per_migration_goldens.py``) pin the second-invocation
+    no-op; the registry-completeness guard enforces one per migration.
+    """
     return _make_migration_decorator(_STATS_MIGRATIONS, "stats.db", name)
 
 
 def cache_migration(name: str):
-    """Register a cache.db migration. Use as @cache_migration("NNN_descriptive_name")."""
+    """Register a cache.db migration. Use as @cache_migration("NNN_descriptive_name").
+
+    HANDLER CONTRACT — idempotency is MANDATORY. See ``stats_migration`` above
+    for the full statement: the dispatcher's handler-commit → marker-stamp
+    two-transaction window means a crash (or a ``MigrationGateNotMet`` deferral)
+    re-invokes the handler on its own output, so every handler must be safe to
+    re-run (probe-then-return / ``INSERT OR IGNORE`` / not-yet-migrated guards).
+    The flag-only re-ingest cache migrations (003-007, 009, 014, 017) satisfy it
+    by ``_set_cache_meta`` upsert; ``001_dedup_highest_wins`` is the lone
+    atomic-stamp carve-out.
+    """
     return _make_migration_decorator(_CACHE_MIGRATIONS, "cache.db", name)
 
 
@@ -480,7 +530,7 @@ def _would_block_prod_migration(conn: sqlite3.Connection) -> bool:
     never trip it; HOME-faking-immune via _real_prod_data_dir (password DB, not
     $HOME); suppressor-INDEPENDENT raw .git check so it still fires under the
     test-suite's CCTALLY_DISABLE_DEV_AUTODETECT. Escape: CCTALLY_ALLOW_PROD_MIGRATION."""
-    if os.environ.get("CCTALLY_ALLOW_PROD_MIGRATION"):
+    if _cctally_core._truthy_env("CCTALLY_ALLOW_PROD_MIGRATION"):
         return False
     if not (_cctally_core._repo_root() / ".git").exists():
         return False
@@ -875,8 +925,8 @@ def _run_pending_migrations(
             # outcome (apply OR gate-defer) clears any prior failure log
             # for this migration's qualified name.
             _clear_migration_error_log_entries(qualified_name)
-            if os.environ.get("CCTALLY_DEBUG"):
-                eprint(
+            if _lib_log.debug_enabled():
+                _lib_log.get_logger("db").debug(
                     f"[migration {qualified_name}] deferred: {gate_exc}"
                 )
             # D2 — ``continue``, NOT ``break``. A gate-defer leaves the DB
@@ -2350,6 +2400,16 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
             ON conversation_messages(source_path);
         CREATE INDEX IF NOT EXISTS idx_conv_turnkey
             ON conversation_messages(msg_id, req_id);
+        -- #289: partial covering index on cwd. build_anon_plan_for_db sources
+        -- anonymization tokens with `SELECT DISTINCT cwd FROM conversation_messages
+        -- WHERE cwd IS NOT NULL AND cwd != ''`; without this index that DISTINCT is
+        -- a full scan of the whole (prose-heavy) table (~20s cold on a 5.5 GB
+        -- cache). The partial `(cwd)` index matches the WHERE exactly so the
+        -- DISTINCT is answered by an ordered index-only walk. MIRRORS migration 021
+        -- (base schema here for fresh/rebuilt caches, migration for existing ones).
+        CREATE INDEX IF NOT EXISTS idx_conversation_messages_cwd
+            ON conversation_messages(cwd)
+            WHERE cwd IS NOT NULL AND cwd != '';
 
         -- #193: per-session AI-generated title, isolated from the six places
         -- that iterate conversation_messages. The explicit NOT NULL on the
@@ -2500,6 +2560,24 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
     conn.execute(
         "CREATE INDEX IF NOT EXISTS idx_entries_mutation_seq "
         "ON session_entries(mutation_seq, mutation_min_ts)")
+    # #279 S3 F3: DB-level idempotency backstop mirroring
+    # codex_session_entries' UNIQUE(source_path, line_offset). Guarded and
+    # OUTSIDE the top executescript: this function runs on EVERY open BEFORE the
+    # migration dispatcher, so a legacy cache.db holding historical
+    # physical-key duplicates must tolerate the index being ABSENT until cache
+    # migration 020 dedups it — an unguarded CREATE UNIQUE INDEX here would
+    # brick every open of such a DB before 020 could ever run. Fresh and clean
+    # DBs get the index immediately; a dirty legacy one gets it from 020 (or the
+    # first open after 020 dedups the duplicates). Kept after the
+    # add_column_if_missing calls (like idx_entries_mutation_seq) so the columns
+    # exist, and BEFORE the legacy-FTS early-return so an old-shape DB still
+    # receives it.
+    try:
+        conn.execute(
+            "CREATE UNIQUE INDEX IF NOT EXISTS idx_entries_physical "
+            "ON session_entries(source_path, line_offset)")
+    except sqlite3.IntegrityError:
+        pass
     # Existing-DB guard for the skill-content fold link (cctally-dev
     # skill-content-nesting): the message-level sourceToolUseID. Idempotent
     # column-add (no marker, no version); cache migration 006 then re-ingests
@@ -3865,6 +3943,81 @@ def _019_create_conversation_file_touches(conn: sqlite3.Connection) -> None:
     conn.commit()
 
 
+@cache_migration("020_session_entries_physical_unique")
+def _020_session_entries_physical_unique(conn: sqlite3.Connection) -> None:
+    """#279 S3 F3: dedup historical (source_path, line_offset) duplicates
+    (keep-first-id) and ensure the physical-key UNIQUE backstop index exists.
+
+    Duplicated physical keys are strictly ingest-bug artifacts (spec §4: correct
+    offset bookkeeping cannot produce them) with no principled winner; MIN(id)
+    keeps the first-ingested row, matching the pre-bug state. cache.db is
+    re-derivable — `cache-sync --rebuild` remains the escape hatch. Mirrors
+    cache 001's flock-then-BEGIN-IMMEDIATE pattern (#105): mutual exclusion with
+    a mid-walk sync_cache, deferring via MigrationGateNotMet on contention.
+
+    Fresh installs never run this handler: the dispatcher stamps it WITHOUT
+    running (the fresh-install branch), and `_apply_cache_schema` already created
+    the index on the fresh DB. NO self-stamp — the dispatcher central-stamps on a
+    clean return (#140). Run-twice safe: the DELETE is idempotent (no dupes left
+    on a second run) and the index create is `IF NOT EXISTS`; safe on an empty
+    table.
+    """
+    lock_path = _cache_db_lock_path_for_conn(conn)
+    lock_fh = None
+    if lock_path is not None:
+        lock_fh = open(lock_path, "w")
+        try:
+            fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            lock_fh.close()
+            raise MigrationGateNotMet(
+                "cache.db.lock held by a concurrent sync_cache; deferring "
+                "cache 020 physical-unique dedup (#279 S3)"
+            )
+    try:
+        conn.execute("BEGIN IMMEDIATE")
+        try:
+            conn.execute(
+                "DELETE FROM session_entries WHERE id NOT IN ("
+                "SELECT MIN(id) FROM session_entries "
+                "GROUP BY source_path, line_offset)")
+            conn.execute(
+                "CREATE UNIQUE INDEX IF NOT EXISTS idx_entries_physical "
+                "ON session_entries(source_path, line_offset)")
+            conn.commit()
+        except Exception:
+            conn.rollback()
+            raise
+    finally:
+        if lock_fh is not None:
+            try:
+                fcntl.flock(lock_fh, fcntl.LOCK_UN)
+            except OSError:
+                pass
+            lock_fh.close()
+
+
+@cache_migration("021_index_conversation_messages_cwd")
+def _021_index_conversation_messages_cwd(conn: sqlite3.Connection) -> None:
+    """#289: partial covering index on conversation_messages(cwd) to collapse the
+    full-table `SELECT DISTINCT cwd` scan in build_anon_plan_for_db (~20s cold on
+    a 5.5 GB cache) into an index-only search.
+
+    Fresh installs never run this handler: the dispatcher stamps it WITHOUT
+    running (the fresh-install branch), and `_apply_cache_schema` already created
+    the index on the fresh DB. NO self-stamp — the dispatcher central-stamps on a
+    clean return (#140). Run-twice safe: CREATE INDEX is IF NOT EXISTS. cache.db
+    is re-derivable — `cache-sync --rebuild` is the escape hatch.
+
+    No flock/BEGIN IMMEDIATE (unlike 020): a single CREATE INDEX IF NOT EXISTS is
+    atomic and touches no rows, so it needs no mutual-exclusion scaffolding.
+    """
+    conn.execute(
+        "CREATE INDEX IF NOT EXISTS idx_conversation_messages_cwd "
+        "ON conversation_messages(cwd) WHERE cwd IS NOT NULL AND cwd != ''"
+    )
+
+
 # === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
 
 @stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")

package/bin/_cctally_doctor.py

@@ -25,6 +25,7 @@ from __future__ import annotations
 
 import argparse
 import datetime as dt
+import fcntl
 import json
 import pathlib
 import shutil
@@ -45,12 +46,18 @@ def doctor_gather_state(
     *,
     now_utc: "dt.datetime | None" = None,
     runtime_bind: "str | None" = None,
+    deep: bool = False,
 ):
     """I/O chokepoint for `cctally doctor` (spec §7.2).
 
     H1 invariant: config.json is read RAW (NOT via load_config), since
     load_config auto-creates the file on first run — a read-only
     diagnostic command must never mutate user state.
+
+    `deep=True` (CLI cmd_doctor only) additionally runs `PRAGMA
+    quick_check(1)` on each DB (#279 S2 F5b); the dashboard/TUI callers
+    stay `deep=False` — the rebuild loop calls the gather every rebuild
+    and quick_check on a large cache.db costs seconds.
     """
     import _lib_doctor
 
@@ -59,8 +66,17 @@ def doctor_gather_state(
         now_utc = _now_utc()
 
     # ── Install ──────────────────────────────────────────────────────
-    repo_root = c._setup_resolve_repo_root()
-    dst_dir = c._setup_local_bin_dir()
+    # #279 S2 F5d: guard the only two unguarded statements in the
+    # otherwise fail-soft gather — an exception here would kill the whole
+    # report. Downstream consumers already degrade on None.
+    try:
+        repo_root = c._setup_resolve_repo_root()
+    except Exception:
+        repo_root = None
+    try:
+        dst_dir = c._setup_local_bin_dir()
+    except Exception:
+        dst_dir = None
     try:
         symlink_state = c._setup_compute_symlink_state(repo_root, dst_dir)
     except Exception:
@@ -382,6 +398,81 @@ def doctor_gather_state(
     except Exception:
         pass
 
+    # ── Parse health (#279 S2 F5a) ───────────────────────────────────
+    parse_health_claude = parse_health_codex = None
+    try:
+        if _cctally_core.CACHE_DB_PATH.exists():
+            conn = sqlite3.connect(str(_cctally_core.CACHE_DB_PATH))
+            try:
+                for _key in ("parse_health_claude", "parse_health_codex"):
+                    try:
+                        row = conn.execute(
+                            "SELECT value FROM cache_meta WHERE key = ?",
+                            (_key,),
+                        ).fetchone()
+                        if row and row[0]:
+                            _parsed = json.loads(row[0])
+                            if isinstance(_parsed, dict):
+                                if _key == "parse_health_claude":
+                                    parse_health_claude = _parsed
+                                else:
+                                    parse_health_codex = _parsed
+                    except (sqlite3.OperationalError, ValueError):
+                        pass
+            finally:
+                conn.close()
+    except Exception:
+        pass
+
+    # ── Integrity (deep only — #279 S2 F5b) ──────────────────────────
+    stats_db_quick_check = cache_db_quick_check = None
+    if deep:
+        for _label, _path in (("stats", _cctally_core.DB_PATH),
+                              ("cache", _cctally_core.CACHE_DB_PATH)):
+            _result = None
+            try:
+                if _path.exists():
+                    _conn = sqlite3.connect(str(_path))
+                    try:
+                        _row = _conn.execute(
+                            "PRAGMA quick_check(1)").fetchone()
+                        _result = (str(_row[0])
+                                   if _row and _row[0] is not None else None)
+                    finally:
+                        _conn.close()
+            except sqlite3.DatabaseError as exc:
+                _result = f"open failed: {exc}"
+            except Exception:
+                _result = None
+            if _label == "stats":
+                stats_db_quick_check = _result
+            else:
+                cache_db_quick_check = _result
+
+    # ── Lock state (#279 S2 F5c) — read-only: never create files ─────
+    locks_held: "dict | None" = None
+    try:
+        locks_held = {}
+        for _name, _lp in (
+            ("cache.db.lock", _cctally_core.CACHE_LOCK_PATH),
+            ("cache.db.codex.lock", _cctally_core.CACHE_LOCK_CODEX_PATH),
+        ):
+            if not _lp.exists():
+                locks_held[_name] = False
+                continue
+            try:
+                with open(_lp, "r") as _lf:
+                    try:
+                        fcntl.flock(_lf, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                        fcntl.flock(_lf, fcntl.LOCK_UN)
+                        locks_held[_name] = False
+                    except OSError:
+                        locks_held[_name] = True
+            except OSError:
+                locks_held[_name] = None
+    except Exception:
+        locks_held = None
+
     # ── Safety ───────────────────────────────────────────────────────
     # `dashboard.bind` is read via the same chokepoint that powers
     # `cctally config get dashboard.bind` — `_config_known_value`
@@ -548,6 +639,13 @@ def doctor_gather_state(
         conv_sessions_rollup_count=conv_sessions_rollup_count,
         conv_messages_distinct_sessions=conv_messages_distinct_sessions,
         conv_rollup_sync_in_progress=conv_rollup_sync_in_progress,
+        # #279 S2 F5: parse-health records, deep quick_check results, and
+        # non-blocking lock-file probes (appended after the defaulted tail).
+        parse_health_claude=parse_health_claude,
+        parse_health_codex=parse_health_codex,
+        stats_db_quick_check=stats_db_quick_check,
+        cache_db_quick_check=cache_db_quick_check,
+        locks_held=locks_held,
     )
 
 
@@ -572,7 +670,10 @@ def cmd_doctor(args: argparse.Namespace) -> int:
     if quiet and verbose:
         eprint("doctor: --quiet and --verbose are mutually exclusive")
         return 2
-    state = c.doctor_gather_state()
+    # #279 S2 F5b: deep=True runs PRAGMA quick_check(1) — CLI-only (the
+    # dashboard/TUI gather callers stay deep=False so their per-rebuild
+    # gather never pays the multi-second cost on a large cache.db).
+    state = c.doctor_gather_state(deep=True)
     report = _lib_doctor.run_checks(state)
     if getattr(args, "json", False):
         print(json.dumps(

package/bin/_cctally_five_hour.py

@@ -708,7 +708,8 @@ def cmd_blocks(args: argparse.Namespace) -> int:
         blocks = [b for b in blocks if b.is_active and not b.is_gap]
         if not blocks:
             if args.json:
-                print('{\n  "blocks": [],\n  "message": "No active block"\n}')
+                print(json.dumps(_c.stamp_schema_version(
+                    {"blocks": [], "message": "No active block"}), indent=2))
             else:
                 print("No active session block found.")
             return 0
@@ -846,9 +847,17 @@ def _format_block_start(iso: str, tz: "ZoneInfo | None") -> str:
     Renders as ``YYYY-MM-DD HH:MM <SUFFIX>`` where the suffix is the
     zone label per ``display_tz_label``. Naive inputs are treated as
     UTC; ``tz=None`` means "host-local via bare astimezone()".
+
+    The displayed time is rounded to the nearest 10-minute boundary to
+    normalize Anthropic reset-capture jitter (e.g. a :39 recorded reset
+    renders as :40). The stored ``block_start_at`` is unchanged.
     """
     _c = _cctally()
-    return _c.format_display_dt(iso, tz, fmt="%Y-%m-%d %H:%M", suffix=True)
+    parsed = dt.datetime.fromisoformat(iso.replace("Z", "+00:00"))
+    return _c.format_display_dt(
+        _c._round_to_ten_minutes(parsed), tz,
+        fmt="%Y-%m-%d %H:%M", suffix=True,
+    )
 
 
 def _format_hhmm_in_tz(iso: str, tz: "ZoneInfo | None") -> str:
@@ -857,9 +866,16 @@ def _format_hhmm_in_tz(iso: str, tz: "ZoneInfo | None") -> str:
     Mirrors ``_format_block_start``'s tz resolution so paired start/end
     cells in the same row stay in the same zone. Naive inputs are
     treated as UTC; ``tz=None`` means host-local. No suffix.
+
+    Rounded to the nearest 10-minute boundary for the same reset-jitter
+    normalization as ``_format_block_start`` (the paired start/end cells
+    must round together or a 5h window renders as 4h59m/5h01m).
     """
     _c = _cctally()
-    return _c.format_display_dt(iso, tz, fmt="%H:%M", suffix=False)
+    parsed = dt.datetime.fromisoformat(iso.replace("Z", "+00:00"))
+    return _c.format_display_dt(
+        _c._round_to_ten_minutes(parsed), tz, fmt="%H:%M", suffix=False,
+    )
 
 
 def _block_is_active(