cctally 1.67.1 → 1.68.0

package/CHANGELOG.md

@@ -5,6 +5,18 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Unreleased]
 
+## [1.68.0] - 2026-07-13
+
+### Added
+- `cctally db checkpoint [--db {cache,stats}] [--json]` — fast, non-destructive WAL drain (TRUNCATE checkpoint) for cache.db/stats.db; the manual escape hatch and the `doctor` remediation for a bloated write-ahead log. (#297)
+
+### Changed
+- `cctally-test-all`: faster full-suite runs — `reconcile` now runs inside the parallel pool instead of as a serial barrier, and a three-role worker model (`CCTALLY_TEST_JOBS` outer budget + derived/overridable `CCTALLY_INNER_JOBS` / `CCTALLY_PYTEST_JOBS`) caps the nested per-harness fan-out that made individual harnesses 2–8× slower under load. Adds a slowest-harness timing report. (#292)
+- `doctor` gains a read-only `cache.db WAL size` check that warns when the WAL exceeds 256 MB and points to `cctally db checkpoint`. (#297)
+
+### Fixed
+- Prevent the cache.db write-ahead log from bloating to multi-GB under heavy concurrent multi-agent syncs, which surfaced as `Error: database is locked` on every `cctally` command — cap the WAL file (`PRAGMA journal_size_limit`), force a best-effort TRUNCATE checkpoint at end-of-sync, and raise `busy_timeout` from 5s to 15s. Existing bloated caches self-heal on the next sync (or run `cctally db checkpoint`). (#297)
+
 ## [1.67.1] - 2026-07-12
 
 ### Fixed

package/bin/_cctally_cache.py

@@ -107,7 +107,7 @@ import sqlite3
 import sys
 import time
 from dataclasses import dataclass, field
-from typing import Any, Callable, Iterator
+from typing import Any, Callable, Iterator, NamedTuple
 
 
 def _cctally():
@@ -316,6 +316,96 @@ clear_conversation_messages = _cctally_db_sib.clear_conversation_messages
 _set_cache_meta = _cctally_db_sib._set_cache_meta
 
 
+# cache.db WAL hardening (#297). See
+# docs/superpowers/specs/2026-07-13-cache-db-wal-hardening-design.md.
+# `journal_size_limit` bounds the *persistent* WAL file so a checkpoint that
+# actually resets the WAL truncates the file back down to this cap — it is
+# containment/recovery under transient reader contention, NOT a reader-proof
+# hard cap (a pinned long-lived reader still defeats it until it releases).
+CACHE_WAL_SIZE_LIMIT_BYTES = 128 * 1024 * 1024  # 134217728
+# End-of-sync physical-size shrink trigger: only force a TRUNCATE checkpoint
+# once the -wal file has grown past this, so normal small syncs stay cheap.
+CACHE_WAL_CHECKPOINT_TRIGGER_BYTES = 64 * 1024 * 1024  # 67108864
+# Near-nonblocking busy_timeout for the auto end-of-sync checkpoint. It runs
+# BEFORE the ingest flock is released, so a long wait here would stall every
+# above-threshold sync's flock under the heavy-reader contention that motivates
+# the fix — fail fast and rely on journal_size_limit + the next checkpoint.
+CHECKPOINT_AUTO_BUSY_TIMEOUT_MS = 100
+# The explicit `cctally db checkpoint` command may wait this long.
+CHECKPOINT_CMD_BUSY_TIMEOUT_MS = 15000
+
+
+class CheckpointResult(NamedTuple):
+    """Outcome of a single ``PRAGMA wal_checkpoint(TRUNCATE)`` (#297).
+
+    ``busy`` is the checkpoint's own busy flag (a reader/writer held it off);
+    ``truncated`` means the WAL was actually reset AND the -wal file is now
+    zero-length/absent — a checkpoint can copy some frames yet still report
+    ``busy`` (partial), which is NOT ``truncated``.
+    """
+
+    db: str
+    wal_bytes_before: int
+    wal_bytes_after: int
+    frames_checkpointed: int
+    busy: bool
+    truncated: bool
+
+
+def _wal_file_size(db_path) -> int:
+    """Best-effort size of the -wal sidecar in bytes; 0 if absent/unreadable."""
+    try:
+        return os.path.getsize(f"{db_path}-wal")
+    except OSError:
+        return 0
+
+
+def _run_wal_truncate(conn, db_path, *, db_label: str) -> "CheckpointResult":
+    """Run a best-effort TRUNCATE checkpoint on an already-open connection.
+
+    PRECONDITION: ``conn`` has NO active transaction (autocommit). The
+    ``db checkpoint`` command passes a fresh raw connection; the end-of-sync
+    caller has committed all ingest work first. ``PRAGMA
+    wal_checkpoint(TRUNCATE)`` returns ``(busy, log, checkpointed)``: ``busy=0``
+    means the WAL was reset. Measures the -wal size before and after so callers
+    can report the shrink without re-deriving it.
+    """
+    before = _wal_file_size(db_path)
+    row = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()
+    busy = bool(row[0]) if row else True
+    frames = int(row[2]) if (row and row[2] is not None) else 0
+    after = _wal_file_size(db_path)
+    truncated = (not busy) and after == 0
+    return CheckpointResult(db_label, before, after, frames, busy, truncated)
+
+
+def _maybe_truncate_wal(conn, db_path) -> None:
+    """End-of-sync best-effort WAL drain (#297).
+
+    Threshold-gated (only once the -wal file has grown past
+    ``CACHE_WAL_CHECKPOINT_TRIGGER_BYTES``) and run under a near-nonblocking
+    short busy_timeout so it NEVER stalls the held sync flock — the checkpoint
+    runs BEFORE the flock is released, so a long busy wait here would stall
+    every above-threshold sync under exactly the heavy-reader contention that
+    motivates the fix. Restores the prior busy_timeout. Fail-soft: a checkpoint
+    error must never fail the sync (observability/hygiene, not correctness).
+
+    PRECONDITION: ``conn`` has no active transaction — the caller has committed
+    all ingest work by this point.
+    """
+    try:
+        if _wal_file_size(db_path) <= CACHE_WAL_CHECKPOINT_TRIGGER_BYTES:
+            return
+        prior = conn.execute("PRAGMA busy_timeout").fetchone()[0]
+        try:
+            conn.execute(f"PRAGMA busy_timeout={CHECKPOINT_AUTO_BUSY_TIMEOUT_MS}")
+            _run_wal_truncate(conn, db_path, db_label="cache.db")
+        finally:
+            conn.execute(f"PRAGMA busy_timeout={prior}")
+    except sqlite3.DatabaseError:
+        pass  # best-effort; observability/hygiene must never fail a sync
+
+
 _PARSE_HEALTH_SCHEMA = 1
 
 
@@ -1875,6 +1965,11 @@ def sync_cache(
         # write has materialized the -wal/-shm sidecars. open_cache_db hardens
         # cache.db + the data dir; this finishes the job for the sidecars.
         _harden_cache_sidecars()
+        # #297: forced end-of-sync WAL drain. Threshold-gated + short-timeout +
+        # best-effort. Runs here — all ingest work is committed (no active txn)
+        # and the flock is still held, so the short busy_timeout keeps it from
+        # stalling the lock under heavy-reader contention.
+        _maybe_truncate_wal(conn, _cctally_core.CACHE_DB_PATH)
         return stats
     finally:
         try:
@@ -3262,6 +3357,12 @@ def sync_codex_cache(
             skip_reasons=stats.skip_reasons,
             rebuild=rebuild,
         )
+        # #297: forced end-of-sync WAL drain (Codex half). Claude and Codex
+        # ingests use SEPARATE flocks but commit into the SAME cache.db WAL, so
+        # the fail-fast short timeout naturally dedupes concurrent attempts —
+        # whoever wins truncates; the other gets `busy` immediately and moves
+        # on. All Codex ingest work is committed here (no active txn).
+        _maybe_truncate_wal(conn, _cctally_core.CACHE_DB_PATH)
         return stats
     finally:
         try:
@@ -3513,7 +3614,14 @@ def open_cache_db() -> sqlite3.Connection:
         eprint(f"[cache] could not chmod cache.db 0600 ({exc}); continuing")
 
     conn.execute("PRAGMA journal_mode=WAL")
-    conn.execute("PRAGMA busy_timeout=5000")
+    # #297: 15 s (was 5 s) lets a writer wait out a slow-but-normal sync (>5 s)
+    # instead of instantly erroring with "database is locked". Covers plain
+    # writer-vs-writer SQLITE_BUSY, not SQLITE_BUSY_SNAPSHOT (defended by
+    # write-lock-up-front at the write sites, #87).
+    conn.execute("PRAGMA busy_timeout=15000")
+    # #297: bound the persistent WAL file so a resetting checkpoint truncates it
+    # back to this cap instead of leaving it at its multi-GB high-water size.
+    conn.execute(f"PRAGMA journal_size_limit={CACHE_WAL_SIZE_LIMIT_BYTES}")
     # Re-derivable DB under WAL: NORMAL risks at most the tail transaction on
     # power loss, and cache.db can always be rebuilt (cache-sync --rebuild).
     # Fewer ingest fsyncs than the default FULL. Matches stats.db

package/bin/_cctally_core.py

@@ -213,6 +213,14 @@ def _real_prod_data_dir() -> pathlib.Path:
 _init_paths_from_env()
 
 
+# stats.db WAL cap (#297). Bounds the persistent WAL file so a resetting
+# checkpoint truncates it back down instead of leaving it at its high-water
+# size. stats.db writes are small (its -wal was observed at 0 bytes even under
+# the contention that bloated cache.db to multi-GB), so a tighter 16 MB cap is
+# ample. See docs/superpowers/specs/2026-07-13-cache-db-wal-hardening-design.md.
+STATS_WAL_SIZE_LIMIT_BYTES = 16 * 1024 * 1024  # 16777216
+
+
 # === Telemetry constants (non-path; see spec 2026-07-07) =============
 #
 # These are static (not APP_DIR-derived) so they live outside
@@ -939,16 +947,17 @@ def open_db() -> sqlite3.Connection:
         conn.execute("PRAGMA journal_mode=WAL")
         conn.execute("PRAGMA synchronous=NORMAL")
         # Explicit for intent + symmetry with open_cache_db (bin/_cctally_cache.py).
-        # sqlite3.connect()'s default timeout=5.0 ALREADY maps to busy_timeout=5000,
-        # so this is not a behavior change — it makes the multi-writer retry window
-        # an explicit contract beside the WAL pragmas instead of an inherited
-        # default a reader has to know about. NOTE: busy_timeout does NOT absorb
-        # SQLITE_BUSY_SNAPSHOT (a WAL read-then-write transaction whose snapshot is
-        # invalidated by a competing commit raises "database is locked" instantly,
-        # bypassing the busy handler). The write paths defend against that by taking
-        # the write lock up front — BEGIN IMMEDIATE, or a write as the transaction's
-        # first DML. See cctally-dev#87.
-        conn.execute("PRAGMA busy_timeout=5000")
+        # #297: raised 5000 -> 15000 so a writer waits out a slow-but-normal
+        # sync (>5 s) instead of instantly erroring with "database is locked".
+        # NOTE: busy_timeout does NOT absorb SQLITE_BUSY_SNAPSHOT (a WAL
+        # read-then-write transaction whose snapshot is invalidated by a
+        # competing commit raises "database is locked" instantly, bypassing the
+        # busy handler). The write paths defend against that by taking the write
+        # lock up front — BEGIN IMMEDIATE, or a write as the transaction's first
+        # DML. See cctally-dev#87.
+        conn.execute("PRAGMA busy_timeout=15000")
+        # #297: bound the persistent WAL file (symmetry with open_cache_db).
+        conn.execute(f"PRAGMA journal_size_limit={STATS_WAL_SIZE_LIMIT_BYTES}")
         conn.execute("SELECT 1").fetchone()
     except sqlite3.DatabaseError as exc:
         try:

package/bin/_cctally_db.py

@@ -5453,3 +5453,66 @@ def cmd_db_recover(args: argparse.Namespace) -> int:
         return 0
     finally:
         conn.close()
+
+
+def cmd_db_checkpoint(args: argparse.Namespace) -> int:
+    """Fast, non-destructive WAL drain (TRUNCATE checkpoint) for cache.db /
+    stats.db (#297).
+
+    Opens a RAW existing-file-only connection — NOT open_cache_db()/open_db(),
+    which apply schema, run the migration dispatcher, can DELETE Codex rows on a
+    column-add, and CREATE a missing DB; a checkpoint may do none of those.
+    Precedent: cmd_db_recover's raw connect above. It flushes WAL frames into
+    the main DB and truncates the -wal file — it changes no data, no schema, no
+    user_version — so there is no prod guard and no --yes.
+
+    Exit 0 when drained / already-small / the DB is absent; 3 (staged) if the
+    target stayed busy or was not fully truncated through the timeout — an
+    actionable "something is still holding it" signal.
+    """
+    import _cctally_cache
+    from _lib_json_envelope import stamp_schema_version
+
+    which = args.db  # "cache" | "stats"
+    if which == "cache":
+        path, label = _cctally_core.CACHE_DB_PATH, "cache.db"
+    else:
+        path, label = _cctally_core.DB_PATH, "stats.db"
+    timeout = int(getattr(args, "busy_timeout_ms", None)
+                  or _cctally_cache.CHECKPOINT_CMD_BUSY_TIMEOUT_MS)
+    as_json = bool(getattr(args, "json", False))
+
+    # Absent file → nothing to drain; a missing re-derivable cache is not an
+    # error (mirrors cmd_db_recover / cmd_db_unskip). Do NOT connect — mode=rw
+    # would refuse to create it, but skip it entirely for a clean message.
+    if not path.exists():
+        if as_json:
+            payload = {"db": label, "walBytesBefore": 0, "walBytesAfter": 0,
+                       "framesCheckpointed": 0, "busy": False, "truncated": True,
+                       "present": False}
+            print(json.dumps(stamp_schema_version(payload, version=1)))
+        else:
+            print(f"cctally: no {label} database file present; nothing to drain.")
+        return 0
+
+    conn = sqlite3.connect(f"file:{path}?mode=rw", uri=True)
+    try:
+        conn.execute(f"PRAGMA busy_timeout={timeout}")
+        result = _cctally_cache._run_wal_truncate(conn, path, db_label=label)
+    finally:
+        conn.close()
+
+    if as_json:
+        payload = {"db": result.db, "walBytesBefore": result.wal_bytes_before,
+                   "walBytesAfter": result.wal_bytes_after,
+                   "framesCheckpointed": result.frames_checkpointed,
+                   "busy": result.busy, "truncated": result.truncated,
+                   "present": True}
+        print(json.dumps(stamp_schema_version(payload, version=1)))
+    else:
+        mb_b = result.wal_bytes_before / (1024 * 1024)
+        mb_a = result.wal_bytes_after / (1024 * 1024)
+        state = "drained" if result.truncated else ("still busy" if result.busy else "partial")
+        print(f"cctally: {result.db} WAL {mb_b:.1f} MB -> {mb_a:.1f} MB "
+              f"({result.frames_checkpointed} frames; {state}).")
+    return 0 if result.truncated else 3

package/bin/_cctally_doctor.py

@@ -473,6 +473,17 @@ def doctor_gather_state(
     except Exception:
         locks_held = None
 
+    # ── cache.db WAL size (#297) — read-only backstop ────────────────
+    # Gathered OUTSIDE the deep/quick_check branch (above) so the WAL-size
+    # check runs in both shallow and deep gather modes. Best-effort getsize;
+    # None on OSError/race (doctor never blocks or raises), 0 when absent.
+    cache_db_wal_bytes: "int | None"
+    try:
+        _wal = pathlib.Path(f"{_cctally_core.CACHE_DB_PATH}-wal")
+        cache_db_wal_bytes = _wal.stat().st_size if _wal.exists() else 0
+    except OSError:
+        cache_db_wal_bytes = None
+
     # ── Safety ───────────────────────────────────────────────────────
     # `dashboard.bind` is read via the same chokepoint that powers
     # `cctally config get dashboard.bind` — `_config_known_value`
@@ -646,6 +657,8 @@ def doctor_gather_state(
         stats_db_quick_check=stats_db_quick_check,
         cache_db_quick_check=cache_db_quick_check,
         locks_held=locks_held,
+        # #297: cache.db WAL size backstop (gathered outside the deep branch).
+        cache_db_wal_bytes=cache_db_wal_bytes,
     )
 
 

package/bin/_cctally_parser.py

@@ -2576,6 +2576,29 @@ def _build_db_parser(subparsers, name, *, help_text, xref=None):
         help="Required for --db stats (non-re-derivable; may need a re-record)",
     )
     db_recover.set_defaults(func=c.cmd_db_recover)
+    db_checkpoint = db_sub.add_parser(
+        "checkpoint",
+        help="Drain the WAL (TRUNCATE checkpoint) — fast, non-destructive",
+    )
+    db_checkpoint.add_argument(
+        "--db",
+        choices=("cache", "stats"),
+        default="cache",
+        help="Which DB to checkpoint (default: cache)",
+    )
+    db_checkpoint.add_argument(
+        "--json",
+        action="store_true",
+        help="Emit JSON to stdout",
+    )
+    db_checkpoint.add_argument(
+        "--busy-timeout-ms",
+        dest="busy_timeout_ms",
+        type=int,
+        default=15000,
+        help=argparse.SUPPRESS,
+    )
+    db_checkpoint.set_defaults(func=c.cmd_db_checkpoint)
 
 def _build_doctor_parser(subparsers, name, *, help_text, xref=None):
     """Build the `doctor` parser (registered via _REGISTRATION; #279 S6 W3).

package/bin/_lib_doctor.py

@@ -194,6 +194,11 @@ class DoctorState:
     # (name -> True held / False free / None unreadable). Probe never
     # creates files (doctor read-only contract). None = probe errored.
     locks_held: Optional[dict] = None
+    # #297: size in bytes of cache.db-wal, gathered read-only in
+    # doctor_gather_state (getsize, OSError/absent -> None). Warns above
+    # DOCTOR_WAL_WARN_BYTES (2x the WAL cap) — only when the journal_size_limit
+    # + forced-checkpoint machinery has genuinely failed to contain the WAL.
+    cache_db_wal_bytes: Optional[int] = None
 
 
 @dataclasses.dataclass(frozen=True)
@@ -1340,6 +1345,36 @@ def _check_db_lock_state(s: DoctorState) -> CheckResult:
     )
 
 
+# #297: WARN when cache.db-wal exceeds 2x the Section 1 cap (128 MiB), so this
+# only fires when the journal_size_limit + forced-checkpoint machinery has
+# genuinely failed to contain the WAL — never in normal operation.
+DOCTOR_WAL_WARN_BYTES = 256 * 1024 * 1024  # 268435456
+
+
+def _check_db_wal_size(s: DoctorState) -> CheckResult:
+    """Read-only backstop that makes a genuine cache.db WAL wedge visible and
+    points at `cctally db checkpoint` (#297).
+
+    The exact byte count lives ONLY in the (fingerprint-excluded) details block;
+    the summary stays a stable string so a below-threshold byte count that
+    drifts tick-to-tick does not flip the doctor fingerprint — only an OK<->WARN
+    crossing does.
+    """
+    wal = s.cache_db_wal_bytes
+    details = {"cache_db_wal_bytes": wal}
+    if isinstance(wal, int) and wal > DOCTOR_WAL_WARN_BYTES:
+        return CheckResult(
+            id="db.wal_size", title="cache.db WAL size", severity="warn",
+            summary="oversized — cache.db WAL far above its cap",
+            remediation="Run `cctally db checkpoint` to drain the WAL.",
+            details=details,
+        )
+    return CheckResult(
+        id="db.wal_size", title="cache.db WAL size", severity="ok",
+        summary="within limit", remediation=None, details=details,
+    )
+
+
 # Each entry is (category_id, category_title, ((check_id, evaluator_fn_name), ...)).
 # The dotted check_id is the stable JSON-contract ID (spec §5.2) AND the
 # fingerprint identity-slice key (spec §5.5). When an evaluator raises,
@@ -1370,6 +1405,7 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
         ("db.migrations.applied", "_check_db_migrations_applied"),
         ("db.migrations.pending", "_check_db_migrations_pending"),
         ("db.lock_state", "_check_db_lock_state"),
+        ("db.wal_size", "_check_db_wal_size"),
     )),
     ("data", "Data", (
         ("data.latest_snapshot_age", "_check_data_latest_snapshot_age"),

package/bin/cctally

@@ -831,6 +831,7 @@ _db_path_for_label = _cctally_db._db_path_for_label
 cmd_db_skip = _cctally_db.cmd_db_skip
 cmd_db_unskip = _cctally_db.cmd_db_unskip
 cmd_db_recover = _cctally_db.cmd_db_recover
+cmd_db_checkpoint = _cctally_db.cmd_db_checkpoint
 
 
 # Session-entry cache subsystem (Claude + Codex) — read-through delta

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cctally",
-  "version": "1.67.1",
+  "version": "1.68.0",
   "description": "Claude Code usage tracker and local dashboard for Pro/Max subscription limits - weekly cost-per-percent trend, quota forecasts, threshold alerts. ccusage-compatible.",
   "homepage": "https://github.com/omrikais/cctally",
   "repository": {