cctally 1.74.0 → 1.75.1

package/bin/_lib_conversation_retention.py

@@ -1,4 +1,4 @@
-"""#313 P3: 180-day conversation-transcript retention prune kernel.
+"""#313 P3: conversation-transcript retention prune kernel.
 
 Prunes ONLY the re-derivable transcript rows:
   * Claude: ``conversation_messages`` + their ``conversation_file_touches`` /
@@ -63,7 +63,7 @@ def _cutoff_iso(cutoff_utc: dt.datetime) -> str:
     ``...Z`` timestamps.
 
     Second-granular: the sub-second mixed-precision edge at the exact cutoff
-    second is immaterial for a 180-day window — any message wrongly classified
+    second is immaterial for a multi-day retention window — any message wrongly classified
     there is re-derivable from JSONL and the boundary self-corrects on the next
     (daily) prune.
     """
@@ -197,6 +197,27 @@ def _stamp_retention_prune(conn: sqlite3.Connection, now_utc: dt.datetime) -> No
     )
 
 
+def _reclaim_incremental_vacuum(conn: sqlite3.Connection) -> None:
+    """Drive zero-column incremental-vacuum rows to completion portably."""
+    remaining = int(conn.execute("PRAGMA freelist_count").fetchone()[0])
+    if remaining <= 0:
+        return
+    chunk_pages = 4096
+    max_passes = (remaining + chunk_pages - 1) // chunk_pages + 1
+    for _ in range(max_passes):
+        requested = min(remaining, chunk_pages)
+        # executescript() routes through sqlite3_exec(), which steps zero-column
+        # pragma rows through SQLITE_DONE on Python/SQLite combinations where a
+        # Cursor.fetchall() can stop after the first row (public Linux 3.11).
+        conn.executescript(f"PRAGMA incremental_vacuum({requested});")
+        after = int(conn.execute("PRAGMA freelist_count").fetchone()[0])
+        if after <= 0:
+            return
+        if after >= remaining:
+            return
+        remaining = after
+
+
 def _maybe_prune_conversation_retention(
     conn: sqlite3.Connection,
     *,
@@ -259,6 +280,25 @@ def _maybe_prune_conversation_retention(
                 except Exception:
                     conn.rollback()
                     raise
+                # Return the freed pages to the OS. On an INCREMENTAL auto-vacuum
+                # cache.db (the default for freshly-created DBs, #313 P3) this
+                # shrinks the file on disk instead of leaving a growing freelist,
+                # so the transcript prune reclaims space automatically without a
+                # manual `cctally db vacuum`; on a legacy auto_vacuum=NONE cache it
+                # is a harmless no-op (those still reclaim via `db vacuum` or a
+                # `cache-sync --rebuild`). Runs OUTSIDE the committed transaction,
+                # still under the maintenance + provider flocks (no concurrent
+                # writer), and best-effort — a reclaim error must never fail the
+                # already-durable prune.
+                if stats.total_rows > 0:
+                    try:
+                        # Use the sqlite3_exec path and verify progress between
+                        # bounded chunks. This clears the freelist and drops
+                        # page_count; the physical file shrinks on the next
+                        # `wal_checkpoint(TRUNCATE)` the sync loop forces (#297).
+                        _reclaim_incremental_vacuum(conn)
+                    except sqlite3.Error:
+                        pass
                 return stats
             finally:
                 try:

package/bin/_lib_doctor.py

@@ -1650,10 +1650,10 @@ def _check_db_integrity(s: DoctorState) -> CheckResult:
             summary=f"stats.db quick_check: {s.stats_db_quick_check}",
             remediation=(
                 "stats.db (the non-re-derivable DB) reports corruption. "
-                "Back up the file FIRST, then try "
-                "`sqlite3 <path> \".recover\"` into a fresh file; "
-                "`cctally db recover` does not repair corruption. Please "
-                "file a bug. Do not delete the file."),
+                "Stop the dashboard and other cctally processes, then run "
+                "`cctally db repair --db stats --yes`. The command preserves "
+                "a backup of the corrupt original before replacing anything. "
+                "Do not copy, restore, move, or delete the live DB by hand."),
             details=details,
         )
     if s.cache_db_quick_check is not None and s.cache_db_quick_check != "ok":

package/bin/cctally

@@ -933,6 +933,9 @@ Migration = _cctally_db.Migration
 DowngradeDetected = _cctally_db.DowngradeDetected
 ProdMigrationRefused = _cctally_db.ProdMigrationRefused
 StatsDbCorruptError = _cctally_db.StatsDbCorruptError
+StatsDbMaintenanceError = _cctally_db.StatsDbMaintenanceError
+_is_sqlite_corruption_error = _cctally_db._is_sqlite_corruption_error
+_stats_corruption_guidance = _cctally_db._stats_corruption_guidance
 _STATS_MIGRATIONS = _cctally_db._STATS_MIGRATIONS
 _CACHE_MIGRATIONS = _cctally_db._CACHE_MIGRATIONS
 _make_migration_decorator = _cctally_db._make_migration_decorator
@@ -957,6 +960,8 @@ _db_path_for_label = _cctally_db._db_path_for_label
 cmd_db_skip = _cctally_db.cmd_db_skip
 cmd_db_unskip = _cctally_db.cmd_db_unskip
 cmd_db_recover = _cctally_db.cmd_db_recover
+cmd_db_repair = _cctally_db.cmd_db_repair
+cmd_db_backup = _cctally_db.cmd_db_backup
 cmd_db_checkpoint = _cctally_db.cmd_db_checkpoint
 cmd_db_vacuum = _cctally_db.cmd_db_vacuum
 
@@ -1384,6 +1389,7 @@ cmd_codex_quota_statusline = _cctally_quota.cmd_codex_quota_statusline
 cmd_codex_quota_forecast = _cctally_quota.cmd_codex_quota_forecast
 cmd_codex_quota_blocks = _cctally_quota.cmd_codex_quota_blocks
 cmd_codex_quota_breakdown = _cctally_quota.cmd_codex_quota_breakdown
+cmd_codex_percent_breakdown = _cctally_quota.cmd_codex_percent_breakdown
 
 # S3's qualified Codex accounting adapter is intentionally cache-only: callers
 # that need project identity must not downgrade to an unqualified rollout parse.
@@ -2926,6 +2932,7 @@ _is_reset_drop                      = _cctally_weekrefs._is_reset_drop
 # handler (C8, #125 Batch B). Parser dispatch reaches c.cmd_percent_breakdown.
 _cctally_percent_breakdown = _load_sibling("_cctally_percent_breakdown")
 cmd_percent_breakdown = _cctally_percent_breakdown.cmd_percent_breakdown
+_render_percent_breakdown_terminal = _cctally_percent_breakdown._render_percent_breakdown_terminal
 
 
 # _is_cctally_hook_command, SetupError, _load_claude_settings,
@@ -3207,7 +3214,26 @@ def main(argv: list[str] | None = None) -> int:
         # + recovery guidance, exit 2. Caught before the generic handler so it
         # never renders as a raw traceback.
         eprint(f"cctally: {exc}")
-        return 2
+        return 3
+    except StatsDbMaintenanceError as exc:
+        eprint(f"cctally: {exc}")
+        return 3
+    except sqlite3.DatabaseError as exc:
+        if _is_sqlite_corruption_error(exc):
+            eprint(
+                "cctally: a SQLite database is corrupt — run `cctally doctor` "
+                "to identify it. For stats.db run `cctally db repair --db "
+                "stats --yes`; cache.db is re-derivable via `cctally "
+                "cache-sync --rebuild`."
+            )
+            return 3
+        if _lib_log.debug_enabled():
+            _lib_log.get_logger("cli").debug(
+                "unhandled exception in %r",
+                getattr(args, "command", None), exc_info=True,
+            )
+        eprint(f"Error: {exc}")
+        rc = 1
     except Exception as exc:  # pragma: no cover
         # #279 S2 F2: CCTALLY_DEBUG=1 surfaces the full traceback on stderr
         # via the _lib_log chokepoint; default mode stays byte-identical.