cctally 1.79.1 → 1.80.0

package/bin/_cctally_doctor.py

@@ -28,6 +28,7 @@ import datetime as dt
 import fcntl
 import json
 import math
+import os
 import pathlib
 import shutil
 import sqlite3
@@ -44,6 +45,23 @@ def _cctally():
     return sys.modules["cctally"]
 
 
+def _journal_heal_incident(kind: str, name: str, now_utc: dt.datetime) -> dict:
+    """One auto-heal artifact record for the doctor journal leg (§9). Parses the
+    ``%Y%m%dT%H%M%SZ`` timestamp trailing the name (quarantine dir
+    ``<db>.db-<ts>`` or forensics file ``<db>.db-corruption-forensics-<ts>.json``)
+    into an age; a name that doesn't parse degrades to ``age_s=None``."""
+    base = name[:-5] if name.endswith(".json") else name
+    ts = base.rsplit("-", 1)[-1]
+    age_s = None
+    try:
+        parsed = dt.datetime.strptime(ts, "%Y%m%dT%H%M%SZ").replace(
+            tzinfo=dt.timezone.utc)
+        age_s = int((now_utc - parsed).total_seconds())
+    except ValueError:
+        pass
+    return {"kind": kind, "name": name, "age_s": age_s}
+
+
 def _gather_statusline_pipeline(c, *, now_utc: dt.datetime) -> dict:
     """Read the #318 statusline pipeline without creating or pruning files."""
     now_epoch = int(now_utc.timestamp())
@@ -290,6 +308,11 @@ def doctor_gather_state(
         stats_db_status = {"path": str(_cctally_core.DB_PATH), "user_version": 0,
                            "registry_size": len(c._STATS_MIGRATIONS),
                            "migrations": [], "_open_error": str(exc)}
+    # stats.db is the epoch-versioned journal index (DB journal redesign §7.1):
+    # feed the epoch constant to the pure kernel so db.version_ahead classifies
+    # uv==1000 as HEALTHY (not a #145 version-ahead FAIL). registry_size stays the
+    # frozen legacy head (13) and serves as the legacy-range boundary.
+    stats_db_status["epoch"] = _cctally_core.STATS_INDEX_EPOCH
     try:
         cache_db_status = c._db_status_for(_cctally_core.CACHE_DB_PATH, c._CACHE_MIGRATIONS, "cache.db")
         if not _cctally_core.CACHE_DB_PATH.exists():
@@ -863,6 +886,129 @@ def doctor_gather_state(
         pricing_coverage = None
 
     # ── Meta ─────────────────────────────────────────────────────────
+    # ── Journal (DB journal redesign §9) ─────────────────────────────
+    # Read-only legs over the append-only journal: presence + appendability,
+    # torn-tail/malformed counts (deep-gated — reads whole segments), the ingest
+    # cursor lag vs. the high-water, and the auto-heal incident history. Every
+    # probe degrades to its always-OK posture on any error (the pure kernel then
+    # reports "no journal" / "not scanned").
+    import _lib_journal as _jl
+    journal_present = False
+    journal_appendable = None
+    journal_segment_count = 0
+    journal_malformed_count = None
+    journal_torn_tail_count = None
+    journal_cursor_lag_bytes = None
+    journal_hw_segment = None
+    journal_cursor_segment = None
+    try:
+        jdir = _cctally_core.JOURNAL_DIR
+        journal_present = jdir.exists()
+        if journal_present:
+            try:
+                journal_appendable = os.access(str(jdir), os.W_OK)
+            except OSError:
+                journal_appendable = None
+            import _cctally_journal as _jr
+            try:
+                segs = _jr.list_segments()  # canonical (segment) order
+            except Exception:
+                segs = []
+            journal_segment_count = len(segs)
+            sizes: dict = {}
+            for seg in segs:
+                try:
+                    sizes[seg] = (jdir / seg).stat().st_size
+                except OSError:
+                    sizes[seg] = 0
+            if segs:
+                journal_hw_segment = segs[-1]
+            # deep-gated malformed / torn-tail scan (reads the whole journal;
+            # the dashboard's per-rebuild gather stays deep=False so it never
+            # pays this at the 10× envelope — mirrors the quick_check legs).
+            if deep and segs:
+                malformed = 0
+                torn = 0
+                for seg in segs:
+                    try:
+                        data = (jdir / seg).read_bytes()
+                    except OSError:
+                        continue
+                    if not data:
+                        continue
+                    if not data.endswith(b"\n"):
+                        torn += 1
+                    # every element except the last is a complete line; the last
+                    # is either "" (ended in \n) or the torn partial — not a
+                    # mid-file line, so it is never counted as malformed.
+                    for raw in data.split(b"\n")[:-1]:
+                        if raw and _jl.decode_line(raw) is None:
+                            malformed += 1
+                journal_malformed_count = malformed
+                journal_torn_tail_count = torn
+            # ingest cursor lag: unconsumed bytes between the stats index cursor
+            # and the journal high-water, in canonical (segment, offset) order.
+            cursor = None
+            try:
+                if _cctally_core.DB_PATH.exists():
+                    jc = sqlite3.connect(
+                        f"file:{_cctally_core.DB_PATH}?mode=ro", uri=True)
+                    try:
+                        crow = jc.execute(
+                            "SELECT segment, offset FROM journal_cursor "
+                            "WHERE id = 1").fetchone()
+                        if crow:
+                            cursor = (crow[0], int(crow[1]))
+                    except sqlite3.OperationalError:
+                        pass  # pre-cutover DB has no journal_cursor table
+                    finally:
+                        jc.close()
+            except sqlite3.Error:
+                cursor = None
+            if cursor is not None and segs:
+                cseg, coff = cursor
+                journal_cursor_segment = cseg
+                order = {s: i for i, s in enumerate(segs)}
+                if cseg in order:
+                    ci = order[cseg]
+                    lag = max(0, sizes.get(segs[ci], 0) - coff)
+                    for s in segs[ci + 1:]:
+                        lag += sizes.get(s, 0)
+                    journal_cursor_lag_bytes = lag
+    except Exception:
+        pass
+    # Auto-heal incident history — independent of journal presence (a corruption
+    # incident can predate cutover). None only if BOTH dirs were unreadable.
+    journal_heal_incidents = None
+    _incidents: list = []
+    _incident_read_ok = False
+    try:
+        qroot = _cctally_core.APP_DIR / "quarantine"
+        if qroot.exists():
+            _incident_read_ok = True
+            for entry in qroot.iterdir():
+                if entry.is_dir():
+                    _incidents.append(
+                        _journal_heal_incident("quarantine", entry.name, now_utc))
+    except OSError:
+        pass
+    try:
+        logdir = _cctally_core.LOG_DIR
+        if logdir.exists():
+            _incident_read_ok = True
+            for entry in logdir.iterdir():
+                n = entry.name
+                if "-corruption-forensics-" in n and n.endswith(".json"):
+                    _incidents.append(
+                        _journal_heal_incident("forensics", n, now_utc))
+    except OSError:
+        pass
+    if _incident_read_ok:
+        # most-recent first; unparseable ages (None) sort last.
+        _incidents.sort(key=lambda d: (d["age_s"] is None,
+                                       d["age_s"] if d["age_s"] is not None else 0))
+        journal_heal_incidents = _incidents
+
     cctally_version_tuple = _lib_changelog._read_latest_changelog_version()
     cctally_version = (
         cctally_version_tuple[0] if cctally_version_tuple else "unknown"
@@ -944,6 +1090,16 @@ def doctor_gather_state(
         # #311: precomputed statusLine.refreshInterval classification.
         statusline_refresh_state=statusline_refresh_state,
         statusline_pipeline=statusline_pipeline,
+        # DB journal redesign §9: append-only journal legs.
+        journal_present=journal_present,
+        journal_appendable=journal_appendable,
+        journal_segment_count=journal_segment_count,
+        journal_malformed_count=journal_malformed_count,
+        journal_torn_tail_count=journal_torn_tail_count,
+        journal_cursor_lag_bytes=journal_cursor_lag_bytes,
+        journal_hw_segment=journal_hw_segment,
+        journal_cursor_segment=journal_cursor_segment,
+        journal_heal_incidents=journal_heal_incidents,
     )
 
 

package/bin/_cctally_five_hour.py

@@ -1461,7 +1461,9 @@ def cmd_five_hour_breakdown(args: argparse.Namespace) -> int:
         conn.close()
 
 
-def _backfill_five_hour_blocks(conn: sqlite3.Connection) -> int:
+def _backfill_five_hour_blocks(
+    conn: sqlite3.Connection, *, only_missing: bool = False,
+) -> int:
     """One-shot historical backfill of five_hour_blocks from existing
     weekly_usage_snapshots data. Idempotent via UNIQUE(five_hour_window_key)
     + INSERT OR IGNORE. Per spec §4.3, five_hour_milestones is NEVER
@@ -1475,6 +1477,14 @@ def _backfill_five_hour_blocks(conn: sqlite3.Connection) -> int:
     snapshot keys we're about to insert are jitter-forked. On failure,
     rolls back and returns 0; gate fires again on next open_db() so
     partial state is recoverable.
+
+    ``only_missing`` (Task 8, spec §5.4 rebuild): scope the window scan to
+    windows that have snapshots but NO five_hour_blocks row yet — the OPEN
+    (never-closed) window's projection. On a journal rebuild the CLOSED blocks
+    are already materialized from ``five_hour_block_close`` evts, so this
+    re-materializes only the open block (block-only, no milestones — the
+    journaled 5h-milestone evts carry their own values and resolve block_id
+    against this row), keeping the pass O(open windows) instead of O(all blocks).
     """
     _c = _cctally()
     inserted = 0
@@ -1483,16 +1493,18 @@ def _backfill_five_hour_blocks(conn: sqlite3.Connection) -> int:
         # 5h percent. The percent guard is critical: MAX(five_hour_percent)
         # over a NULL-only window is NULL, which would trip the
         # final_five_hour_percent NOT NULL constraint at insert time.
-        keys = [
-            int(r[0]) for r in conn.execute(
-                """
+        keys_sql = """
                 SELECT DISTINCT five_hour_window_key
                   FROM weekly_usage_snapshots
                  WHERE five_hour_window_key IS NOT NULL
                    AND five_hour_percent     IS NOT NULL
-                """
-            ).fetchall()
-        ]
+        """
+        if only_missing:
+            keys_sql += (
+                "   AND five_hour_window_key NOT IN "
+                "(SELECT five_hour_window_key FROM five_hour_blocks)"
+            )
+        keys = [int(r[0]) for r in conn.execute(keys_sql).fetchall()]
 
         now_iso = now_utc_iso()
         now_dt = parse_iso_datetime(now_iso, "now")

package/bin/_cctally_forecast.py

@@ -1907,16 +1907,30 @@ def cmd_budget(args: argparse.Namespace) -> int:
             or getattr(args, "format", None)
         )
         if interactive_status and codex_cfg.get("alerts_enabled"):
-            c.maybe_record_codex_budget_milestone({})
-            # #135: the opportunistic Codex PROJECTED-pace backstop, scoped to
-            # the codex_budget_usd metric so it never pops a weekly_pct / Claude
-            # budget_usd notification from a bare `cctally budget`. The projected
-            # leg self-syncs (skip_sync=False); since codex_inputs was just
-            # built above, that delta-sync is a no-op here — correct and cheap.
-            if codex_cfg.get("projected_enabled"):
-                c.maybe_record_projected_alert(
-                    {}, only_metrics={"codex_budget_usd"}
-                )
+            # Task 7 Item 4: the interactive on-demand Codex budget (+ #135
+            # projected) backstop routes through the single-flight ingest cycle
+            # (the `codex_apply` seam) instead of opening its own stats
+            # connection — so its `budget_milestones`/`projected_milestones`
+            # (vendor=codex) crossings are journaled by the harvest and their
+            # alerts dispatch post-commit (set-then-dispatch). OPPORTUNISTIC +
+            # cycle-swallowed, preserving "never raises into the status render".
+            # The passed-conn helpers re-raise on error (invariant ii), which the
+            # opportunistic cycle catches. The #135 projected leg stays scoped to
+            # the codex_budget_usd metric so a bare `cctally budget` never pops a
+            # weekly_pct / Claude budget_usd notification.
+            import _cctally_journal as _jr_codex
+            projected_on = codex_cfg.get("projected_enabled")
+
+            def _codex_backstop_leg(ictx):
+                c.maybe_record_codex_budget_milestone(
+                    {}, conn=ictx.conn, alert_sink=ictx.pending_alerts)
+                if projected_on:
+                    c.maybe_record_projected_alert(
+                        {}, only_metrics={"codex_budget_usd"},
+                        conn=ictx.conn, alert_sink=ictx.pending_alerts)
+
+            _jr_codex.run_stats_ingest(
+                mode="opportunistic", codex_apply=_codex_backstop_leg)
 
     if target is None:
         # Global Claude budget unset → friendly message, then (if configured)
@@ -2119,9 +2133,12 @@ def _cmd_budget_set_project(args: argparse.Namespace) -> int:
     # budget mid-week (already over) doesn't storm. Scoped to the TOUCHED
     # project so it never latches a sibling's already-crossed-but-not-yet-
     # dispatched threshold (which would permanently suppress that alert).
-    c._reconcile_project_budget_milestones_on_write(
-        validated, touched_projects={root}
-    )
+    # 6f writer reroute: routed THROUGH the ingest cycle (opportunistic +
+    # exception-wrapped) so the latched crossings are journaled (rebuild-
+    # replayable) instead of written on a bespoke stats.db connection.
+    import _cctally_journal as _jr
+    _jr.reconcile_budget_config(
+        validated, axes={"project_budget"}, touched_projects={root})
 
     basename = os.path.basename(root) or root
     if getattr(args, "json", False):
@@ -2179,9 +2196,11 @@ def _cmd_budget_unset_project(args: argparse.Namespace) -> int:
     # the map, so this is a no-op for `root` — and it must NOT scan the
     # remaining projects: scanning them would latch a sibling's already-crossed-
     # but-not-yet-dispatched threshold, permanently suppressing its real alert.
-    c._reconcile_project_budget_milestones_on_write(
-        validated, touched_projects={root}
-    )
+    # 6f writer reroute: routed THROUGH the ingest cycle (opportunistic +
+    # exception-wrapped) so the latched crossings are journaled.
+    import _cctally_journal as _jr
+    _jr.reconcile_budget_config(
+        validated, axes={"project_budget"}, touched_projects={root})
 
     basename = os.path.basename(root) or root
     if getattr(args, "json", False):
@@ -2258,7 +2277,10 @@ def _cmd_budget_set(args: argparse.Namespace, period=None) -> int:
     # Shared with `config set budget.*` + dashboard POST /api/settings via
     # _reconcile_budget_on_config_write (gated on _budget_alerts_active — a
     # budget with alerts off or no thresholds records nothing).
-    c._reconcile_budget_on_config_write(validated)
+    # 6f writer reroute: routed THROUGH the ingest cycle (opportunistic +
+    # exception-wrapped) so the latched crossings are journaled.
+    import _cctally_journal as _jr
+    _jr.reconcile_budget_config(validated, axes={"budget"})
     if getattr(args, "json", False):
         print(json.dumps({
             "schemaVersion": _BUDGET_JSON_SCHEMA_VERSION,
@@ -2363,7 +2385,10 @@ def _cmd_budget_set_codex(args: argparse.Namespace, period=None) -> int:
     # Runs OUTSIDE the config_writer_lock (open_db has its own locking). Gated on
     # codex alerts_enabled + thresholds — a Codex budget with alerts off records
     # nothing.
-    c._reconcile_codex_budget_on_config_write(validated)
+    # 6f writer reroute: routed THROUGH the ingest cycle (opportunistic +
+    # exception-wrapped) so the latched crossings are journaled.
+    import _cctally_journal as _jr
+    _jr.reconcile_budget_config(validated, axes={"codex_budget"})
 
     codex = validated["codex"]
     amount_usd = codex["amount_usd"]