cctally 1.83.1 → 1.84.0

package/bin/_cctally_db.py

@@ -754,6 +754,7 @@ def _corruption_trigger_record(
 def write_corruption_forensics(
     db_path,
     *,
+    probe_db_path: "pathlib.Path | None" = None,
     db_label: str = "stats",
     trigger_origin: "str | None" = None,
     trigger_exception: "BaseException | None" = None,
@@ -769,13 +770,21 @@ def write_corruption_forensics(
     heal/rebuild callers. Cache recovery opts into the typed result: only a
     persisted bundle plus either a non-``ok`` integrity row or a positively
     classified corruption failure during the probe confirms destructive
-    quarantine. Probe/write unavailability is always unconfirmed.
+    quarantine. ``probe_db_path`` lets a caller run integrity against an
+    isolated family snapshot while the bundle still identifies the live
+    ``db_path``. Probe/write unavailability is always unconfirmed. Confirmed
+    bundles are atomically replaced and fsynced before this function returns.
     """
     if (trigger_origin is None) != (trigger_exception is None):
         raise ValueError(
             "trigger_origin and trigger_exception must be supplied together"
         )
     db_path = pathlib.Path(db_path)
+    probe_db_path = (
+        pathlib.Path(probe_db_path)
+        if probe_db_path is not None
+        else db_path
+    )
     ts = _db_backup_timestamp()
     try:
         _cctally_core.LOG_DIR.mkdir(parents=True, exist_ok=True)
@@ -808,7 +817,7 @@ def write_corruption_forensics(
         except OSError:
             bundle["family"][p.name] = None
     try:
-        c = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
+        c = sqlite3.connect(f"file:{probe_db_path}?mode=ro", uri=True)
         try:
             c.execute("PRAGMA busy_timeout=2000")
             bundle["integrityCheck"] = [
@@ -860,11 +869,7 @@ def write_corruption_forensics(
     except Exception as exc:
         bundle["holders"] = [f"error: {exc}"]
     try:
-        out.write_text(json.dumps(bundle, indent=2, sort_keys=True))
-        try:
-            os.chmod(out, 0o600)
-        except OSError:
-            pass
+        _atomic_write_private_json(out, bundle)
     except OSError:
         result = CorruptionForensicsResult(
             path=None,
@@ -3328,6 +3333,43 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
             last_seen_utc        TEXT NOT NULL
         );
 
+        -- #416 spec §3.2/§3.3: the durable Codex attribution map. Two tables,
+        -- one concept. `codex_file_incarnations` is the per-file incarnation
+        -- counter (bumped when a truncation resets the file to offset zero, so
+        -- reused offsets can never be covered by the previous incarnation's
+        -- intervals); `codex_file_accounts` holds the decisions themselves, one
+        -- row per (identity, incarnation, byte range).
+        --
+        -- `account_key` is NULLABLE on purpose: NULL is the stably-absent
+        -- SENTINEL DECISION (no auth / api-key mode), matching the
+        -- `NULL ≡ unattributed` cache-read rule used by every other family. The
+        -- literal string "unattributed" is never stored. A torn auth read
+        -- records NO ROW AT ALL — "undecided" and "decided: no account" are
+        -- distinct states and readers must not collapse them.
+        --
+        -- Both live in the UNCONDITIONAL executescript, BEFORE the FTS5
+        -- `legacy_present` early-return below, because they are plain tables
+        -- with no FTS-shape dependency and the ingest path needs them on every
+        -- open (the `_apply_cache_schema_legacy_early_return_before_new_table`
+        -- class). Cache migration 031 exists only to bump the registry head so
+        -- an existing install re-runs this schema apply.
+        CREATE TABLE IF NOT EXISTS codex_file_incarnations (
+            file_identity  TEXT    NOT NULL PRIMARY KEY,
+            incarnation    INTEGER NOT NULL,
+            updated_at_utc TEXT
+        );
+        CREATE TABLE IF NOT EXISTS codex_file_accounts (
+            file_identity  TEXT    NOT NULL,
+            incarnation    INTEGER NOT NULL,
+            from_offset    INTEGER NOT NULL,
+            root_scope     TEXT    NOT NULL,
+            account_key    TEXT,
+            decided_at_utc TEXT    NOT NULL,
+            PRIMARY KEY (file_identity, incarnation, from_offset)
+        );
+        CREATE INDEX IF NOT EXISTS idx_codex_file_accounts_root
+            ON codex_file_accounts(root_scope);
+
         CREATE TABLE IF NOT EXISTS codex_conversation_threads (
             conversation_key     TEXT NOT NULL PRIMARY KEY,
             source_root_key      TEXT NOT NULL,
@@ -3641,6 +3683,15 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
     # backfills it from the still-present legacy event corpus before dropping
     # that corpus.
     add_column_if_missing(conn, "quota_window_snapshots", "observed_model", "TEXT")
+    # #416 spec §4.1/§4.2: the tolerance-anchored canonical reset, resolved at
+    # INGEST over the complete population and stored beside the raw value (which
+    # is retained unchanged as evidence). NULL means "not yet resolved" and every
+    # reader falls back to the raw reset, i.e. exactly today's behaviour — so an
+    # old binary reading a new cache, or a row written before migration 032's
+    # backfill, degrades rather than breaking. Migration 032 arms the one-time
+    # backfill over existing history.
+    add_column_if_missing(
+        conn, "quota_window_snapshots", "canonical_resets_at_utc", "TEXT")
     conn.execute(
         "CREATE INDEX IF NOT EXISTS idx_session_files_session_id "
         "ON session_files(session_id)"
@@ -3795,6 +3846,11 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
         DROP TABLE IF EXISTS quota_window_snapshots;
         DROP TABLE IF EXISTS codex_conversation_threads;
         DROP TABLE IF EXISTS codex_source_roots;
+        -- #416: the Codex attribution map is a cache.db accounting concern; it
+        -- rides in via _apply_cache_schema above and is dropped here with the
+        -- rest of the accounting families so conversations.db stays transcripts-only.
+        DROP TABLE IF EXISTS codex_file_accounts;
+        DROP TABLE IF EXISTS codex_file_incarnations;
 
         CREATE TABLE IF NOT EXISTS conversation_source_files (
             path             TEXT PRIMARY KEY,
@@ -5853,6 +5909,116 @@ def _030_session_entries_cache_creation_split(conn: sqlite3.Connection) -> None:
     conn.commit()
 
 
+@cache_migration("031_codex_file_account_map")
+def _031_codex_file_account_map(conn: sqlite3.Connection) -> None:
+    """#416 spec §3.2/§3.3: land the durable Codex attribution map on an
+    existing install.
+
+    The two tables (``codex_file_incarnations``, ``codex_file_accounts``) are
+    created by ``_apply_cache_schema`` in its UNCONDITIONAL executescript — the
+    repo's table-addition rule, and specifically BEFORE the FTS5
+    ``legacy_present`` early-return so a legacy-shape cache still receives them.
+    This migration exists because that schema apply is VERSION-GATED: a
+    steady-state open compares ``PRAGMA user_version`` against
+    ``len(_CACHE_MIGRATIONS)`` and skips the whole DDL pass when they match
+    (``_cctally_store.schema_current``). Registering here bumps the head, so an
+    already-current install re-runs the schema apply and gains the tables. Same
+    mechanism migration 029 relies on, stated there in the same words.
+
+    There is deliberately NO BACKFILL (spec D1: history that was never durably
+    stamped becomes ``unattributed``; nothing is inferred — heuristic backfill
+    cannot distinguish two accounts that share an email). The map starts empty
+    and fills forward from ingest decisions plus journal replay.
+
+    The idempotent DDL below is a defensive re-assert, not the primary creation
+    path: it keeps the handler self-contained if ``_apply_cache_schema``'s
+    ordering ever drifts, and it is what the per-migration golden exercises
+    (the golden's ``pre.sqlite`` is a genuine 030-head install that predates
+    both tables). Re-running is a no-op. NO self-stamp — the dispatcher
+    central-stamps on a clean return (#140).
+    """
+    conn.executescript(
+        """
+        CREATE TABLE IF NOT EXISTS codex_file_incarnations (
+            file_identity  TEXT    NOT NULL PRIMARY KEY,
+            incarnation    INTEGER NOT NULL,
+            updated_at_utc TEXT
+        );
+        CREATE TABLE IF NOT EXISTS codex_file_accounts (
+            file_identity  TEXT    NOT NULL,
+            incarnation    INTEGER NOT NULL,
+            from_offset    INTEGER NOT NULL,
+            root_scope     TEXT    NOT NULL,
+            account_key    TEXT,
+            decided_at_utc TEXT    NOT NULL,
+            PRIMARY KEY (file_identity, incarnation, from_offset)
+        );
+        CREATE INDEX IF NOT EXISTS idx_codex_file_accounts_root
+            ON codex_file_accounts(root_scope);
+        """
+    )
+    conn.commit()
+
+
+@cache_migration("032_codex_canonical_reset_anchor")
+def _032_codex_canonical_reset_anchor(conn: sqlite3.Connection) -> None:
+    """#416 spec §4.1/§4.2: backfill the tolerance-anchored canonical reset onto
+    existing Codex quota history.
+
+    ``canonical_resets_at_utc`` itself is a plain column addition, so it lands
+    through ``add_column_if_missing`` in ``_apply_cache_schema`` (the idempotent
+    guard pattern — no marker, no version). This migration exists for the
+    BACKFILL, which is a data-shape change and therefore does go through the
+    framework, and it bumps the registry head so a steady-state install re-runs
+    the version-gated schema apply and actually gains the column (the same
+    mechanism 029 and 031 rely on).
+
+    Ordering is ``(source_path, line_offset)`` — the order the rollout walk
+    itself visits bytes in — so the anchors this backfill establishes are the
+    same ones a later ``cache-sync --rebuild`` re-derives, rather than a
+    different-but-also-deterministic set.
+
+    Idempotent by construction: only rows whose anchor is still NULL are
+    considered, and rows already carrying an anchor SEED the resolver, so a
+    re-run cannot move an established anchor.
+    """
+    import _cctally_cache as cache_mod
+
+    cols = {
+        str(row[1]) for row in conn.execute(
+            "PRAGMA table_info(quota_window_snapshots)")
+    }
+    if "canonical_resets_at_utc" not in cols:
+        # An older schema apply did not reach the column add (a legacy-shape
+        # cache whose FTS early-return fires first). Nothing to backfill; the
+        # readers' raw-reset fallback keeps behaviour identical.
+        return
+    resolver = cache_mod.CodexResetAnchorResolver(conn)
+    updates: list[tuple[str, int]] = []
+    for row in conn.execute(
+        "SELECT id, source_root_key, observed_slot, logical_limit_key, "
+        "       window_minutes, resets_at_utc "
+        "  FROM quota_window_snapshots "
+        " WHERE source = 'codex' AND canonical_resets_at_utc IS NULL "
+        "   AND source_root_key IS NOT NULL AND observed_slot IS NOT NULL "
+        " ORDER BY source_path, line_offset, id"
+    ).fetchall():
+        anchor = resolver.resolve(
+            source_root_key=row[1], observed_slot=row[2],
+            logical_limit_key=row[3], window_minutes=row[4],
+            resets_at_utc=row[5],
+        )
+        if anchor is not None:
+            updates.append((anchor, int(row[0])))
+    if updates:
+        conn.executemany(
+            "UPDATE quota_window_snapshots SET canonical_resets_at_utc = ? "
+            "WHERE id = ?",
+            updates,
+        )
+    conn.commit()
+
+
 # === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
 
 @stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")

package/bin/_cctally_doctor.py

@@ -24,6 +24,7 @@ Spec: docs/superpowers/specs/2026-05-30-extract-diagnostics-cmd-design.md
 from __future__ import annotations
 
 import argparse
+import contextlib
 import datetime as dt
 import fcntl
 import json
@@ -78,6 +79,53 @@ def _stats_ro_guarded():
     )
 
 
+@contextlib.contextmanager
+def _conversation_ro_guarded(*, timeout: float):
+    """Bounded read-only transcript handle participating in #415 recovery."""
+    path = pathlib.Path(_cctally_core.CONVERSATIONS_DB_PATH)
+    if not path.exists():
+        yield None
+        return
+    marker = path.with_name(f"{path.name}.repairing")
+    pending = path.with_name(f"{path.name}.quarantine-pending.json")
+    recovery = path.with_name(f"{path.name}.recovery.json")
+    maintenance = pathlib.Path(
+        _cctally_core.CONVERSATIONS_LOCK_MAINTENANCE_PATH
+    )
+    lock_fh = conn = None
+    try:
+        maintenance.parent.mkdir(parents=True, exist_ok=True)
+        lock_fh = open(maintenance, "a+")
+        try:
+            fcntl.flock(lock_fh, fcntl.LOCK_SH | fcntl.LOCK_NB)
+        except BlockingIOError:
+            yield None
+            return
+        if marker.exists() or pending.exists() or recovery.exists():
+            yield None
+            return
+        conn = sqlite3.connect(
+            path.resolve().as_uri() + "?mode=ro",
+            uri=True,
+            timeout=max(0.0, timeout),
+        )
+        if marker.exists() or pending.exists() or recovery.exists():
+            conn.close()
+            conn = None
+            yield None
+            return
+        yield conn
+    finally:
+        if conn is not None:
+            conn.close()
+        if lock_fh is not None:
+            try:
+                fcntl.flock(lock_fh, fcntl.LOCK_UN)
+            except OSError:
+                pass
+            lock_fh.close()
+
+
 def _journal_heal_incident(kind: str, name: str, now_utc: dt.datetime) -> dict:
     """One auto-heal artifact record for the doctor journal leg (§9). Parses the
     legacy ``%Y%m%dT%H%M%SZ`` or collision-safe rebuild
@@ -533,7 +581,7 @@ def _doctor_gather_state_impl(
     diagnostic command must never mutate user state.
 
     `deep=True` (CLI cmd_doctor only) additionally runs `PRAGMA
-    quick_check(1)` on each DB (#279 S2 F5b); the dashboard/TUI callers
+    quick_check(1)` on each DB (#279 S2 F5b, #415); the dashboard/TUI callers
     stay `deep=False` — the rebuild loop calls the gather every rebuild
     and quick_check on a large cache.db costs seconds.
     """
@@ -867,15 +915,15 @@ def _doctor_gather_state_impl(
     try:
         if _cctally_core.CONVERSATIONS_DB_PATH.exists():
             # This gather also runs inside dashboard snapshot precompute. A
-            # transcript writer may hold an exclusive SQLite lock, so use a
-            # read-only zero-timeout probe: conversation health can degrade,
-            # but it must never delay core snapshot freshness (#320).
-            conv_uri = (
-                _cctally_core.CONVERSATIONS_DB_PATH.resolve().as_uri()
-                + "?mode=ro"
-            )
-            conn = sqlite3.connect(conv_uri, uri=True, timeout=0.0)
-            try:
+            # transcript writer or recovery may hold an exclusive lock, so use
+            # the recovery-aware read-only zero-timeout probe: conversation
+            # health can degrade, but it must never delay core snapshot
+            # freshness (#320, #415).
+            with _conversation_ro_guarded(timeout=0.0) as conn:
+                if conn is None:
+                    raise sqlite3.OperationalError(
+                        "conversation store maintenance in progress"
+                    )
                 try:
                     row = conn.execute("PRAGMA page_count").fetchone()
                     if row and row[0] is not None:
@@ -916,8 +964,6 @@ def _doctor_gather_state_impl(
                         conv_rollup_sync_in_progress = True
                 except Exception:
                     pass
-            finally:
-                conn.close()
         # Non-blocking flock probe: if a transcript writer/reingest holds the
         # conversations.db lock, the rollup may be mid-recompute → in progress. We
         # acquire LOCK_EX|LOCK_NB and immediately release; failure (held) is the
@@ -1063,11 +1109,15 @@ def _doctor_gather_state_impl(
 
     # ── Parse health (#279 S2 F5a) ───────────────────────────────────
     parse_health_claude = parse_health_codex = None
+    # #416 review B4: the durable record that a torn Codex `auth.json` halted
+    # ingest. Same cache_meta read, same degrade-to-None-on-anything contract.
+    codex_torn_deferred = None
     try:
         if _cache_probe_allowed and _cctally_core.CACHE_DB_PATH.exists():
             conn = sqlite3.connect(str(_cctally_core.CACHE_DB_PATH))
             try:
-                for _key in ("parse_health_claude", "parse_health_codex"):
+                for _key in ("parse_health_claude", "parse_health_codex",
+                             "codex_torn_auth_deferred"):
                     try:
                         row = conn.execute(
                             "SELECT value FROM cache_meta WHERE key = ?",
@@ -1078,8 +1128,10 @@ def _doctor_gather_state_impl(
                             if isinstance(_parsed, dict):
                                 if _key == "parse_health_claude":
                                     parse_health_claude = _parsed
-                                else:
+                                elif _key == "parse_health_codex":
                                     parse_health_codex = _parsed
+                                else:
+                                    codex_torn_deferred = _parsed
                     except (sqlite3.OperationalError, ValueError):
                         pass
             finally:
@@ -1089,13 +1141,24 @@ def _doctor_gather_state_impl(
 
     # ── Integrity (deep only — #279 S2 F5b) ──────────────────────────
     stats_db_quick_check = cache_db_quick_check = None
+    conversations_db_quick_check = None
     if deep:
         for _label, _path in (("stats", _cctally_core.DB_PATH),
-                              ("cache", _cctally_core.CACHE_DB_PATH)):
+                              ("cache", _cctally_core.CACHE_DB_PATH),
+                              ("conversations",
+                               _cctally_core.CONVERSATIONS_DB_PATH)):
             _result = None
             try:
                 if (
-                    _path.exists()
+                    (
+                        _path.exists()
+                        or (
+                            _label == "conversations"
+                            and _path.with_name(
+                                f"{_path.name}.recovery.json"
+                            ).exists()
+                        )
+                    )
                     and (_label != "cache" or _cache_probe_allowed)
                 ):
                     # #386: the stats leg holds a read-write handle for the whole
@@ -1104,24 +1167,40 @@ def _doctor_gather_state_impl(
                     # protocol. The cache leg keeps its own opener.
                     if _label == "stats":
                         import _cctally_store as _store_mod
-                        _conn = _store_mod.stats_open_guarded(_path)
+                        _conn_ctx = contextlib.closing(
+                            _store_mod.stats_open_guarded(_path)
+                        )
+                    elif _label == "cache":
+                        _conn_ctx = contextlib.closing(
+                            sqlite3.connect(str(_path))
+                        )
                     else:
-                        _conn = sqlite3.connect(str(_path))
-                    try:
-                        _row = _conn.execute(
-                            "PRAGMA quick_check(1)").fetchone()
-                        _result = (str(_row[0])
-                                   if _row and _row[0] is not None else None)
-                    finally:
-                        _conn.close()
+                        _conn_ctx = _conversation_ro_guarded(timeout=2.0)
+                    with _conn_ctx as _conn:
+                        if _conn is not None:
+                            _row = _conn.execute(
+                                "PRAGMA quick_check(1)").fetchone()
+                            _result = (
+                                str(_row[0])
+                                if _row and _row[0] is not None else None
+                            )
+                        elif (
+                            _label == "conversations"
+                            and _path.with_name(
+                                f"{_path.name}.recovery.json"
+                            ).exists()
+                        ):
+                            _result = "recovery in progress"
             except sqlite3.DatabaseError as exc:
                 _result = f"open failed: {exc}"
             except Exception:
                 _result = None
             if _label == "stats":
                 stats_db_quick_check = _result
-            else:
+            elif _label == "cache":
                 cache_db_quick_check = _result
+            else:
+                conversations_db_quick_check = _result
 
     # ── Lock state (#279 S2 F5c) — read-only: never create files ─────
     locks_held: "dict | None" = None
@@ -1627,8 +1706,10 @@ def _doctor_gather_state_impl(
         # non-blocking lock-file probes (appended after the defaulted tail).
         parse_health_claude=parse_health_claude,
         parse_health_codex=parse_health_codex,
+        codex_torn_deferred=codex_torn_deferred,
         stats_db_quick_check=stats_db_quick_check,
         cache_db_quick_check=cache_db_quick_check,
+        conversations_db_quick_check=conversations_db_quick_check,
         locks_held=locks_held,
         # #297: cache.db WAL size backstop (gathered outside the deep branch).
         cache_db_wal_bytes=cache_db_wal_bytes,