cctally 1.60.0 → 1.62.0

package/bin/_cctally_update.py

@@ -196,7 +196,7 @@ import threading
 import time
 import urllib.error
 import urllib.request
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from typing import Any, Callable
 
 
@@ -1872,12 +1872,22 @@ class _DashboardUpdateCheckThread(threading.Thread):
     via ``update.check.enabled = false`` is honoured inside the gate
     so the thread becomes a no-op without needing teardown.
 
-    After a successful check, republishes the current snapshot via the
-    SSE hub so long-open dashboard tabs in ``--no-sync`` mode pick up
-    the fresh ``latest_version`` written to ``update-state.json``. The
-    snapshot itself is unchanged — ``snapshot_to_envelope`` re-reads
-    the state file per envelope build, so a bare publish is enough to
-    refresh the badge for every live subscriber.
+    After a successful check (or an out-of-band ``current_version``
+    self-heal), republishes the current snapshot via the SSE hub so
+    long-open dashboard tabs in ``--no-sync`` mode pick up the fresh
+    ``latest_version`` / ``current_version`` written to
+    ``update-state.json``. Since M4 (#268) the SSE envelope reads
+    update-state / update-suppress off the snapshot's
+    ``envelope_precompute`` rather than re-reading the state file per
+    envelope build, so a BARE publish of the held snapshot would surface
+    the stale precompute. The republish therefore rebuilds
+    ``envelope_precompute`` (small config/state JSON I/O, no DB, no
+    module-cache mutation — see :meth:`_republish_with_fresh_envelope`)
+    on a fresh ``dataclasses.replace`` snapshot before publishing, and
+    persists it to the snapshot ref so the held snapshot stays current.
+    Under ``--no-sync`` this thread is the ONLY refresher of that
+    precompute (the data-sync thread that otherwise rebuilds it is
+    disabled).
     """
 
     daemon = True
@@ -1917,14 +1927,14 @@ class _DashboardUpdateCheckThread(threading.Thread):
                 ):
                     snap = self._ref.get()
                     if snap is not None:
-                        self._hub.publish(snap)
+                        self._republish_with_fresh_envelope(snap)
                 config = load_config()
                 if c._is_update_check_due(config):
                     c._do_update_check()
                     if self._hub is not None and self._ref is not None:
                         snap = self._ref.get()
                         if snap is not None:
-                            self._hub.publish(snap)
+                            self._republish_with_fresh_envelope(snap)
             except Exception as e:
                 # Log but never propagate — this thread must keep
                 # ticking so a transient registry hiccup doesn't
@@ -1940,6 +1950,36 @@ class _DashboardUpdateCheckThread(threading.Thread):
                     pass
             self._stop_event.wait(c.UPDATE_DASHBOARD_CHECK_POLL_S)
 
+    def _republish_with_fresh_envelope(self, snap) -> None:
+        """Republish ``snap`` with a freshly-rebuilt ``envelope_precompute``.
+
+        Since M4 (#268) the SSE envelope reads update-state / update-suppress
+        off ``snap.envelope_precompute`` rather than re-reading the JSON files
+        per client. ``_do_update_check`` / ``_self_heal_current_version`` write
+        ``update-state.json`` OUT OF BAND (advancing no DB signature and no
+        config render key), so a bare re-publish of the HELD snapshot would
+        surface the stale precompute — freezing the version banner on a
+        ``--no-sync`` dashboard whose data-sync thread (the only other
+        precompute refresher) is disabled.
+
+        ``_tui_precompute_envelope_config`` reads only the config / update-state
+        / update-suppress files and mutates NO module cache, so calling it from
+        this thread is safe (no shared-cache-mutation / Codex F7 hazard).
+        ``dataclasses.replace`` yields a NEW snapshot sharing ``prior``'s
+        immutable rows, so an SSE client serializing the previously-published
+        snapshot can't observe a torn value. The fresh snapshot is persisted to
+        the ref so the held snapshot stays current for the next reader.
+        """
+        c = _cctally()
+        fresh = replace(
+            snap,
+            envelope_precompute=(
+                c._cctally_tui._tui_precompute_envelope_config(load_config())
+            ),
+        )
+        self._ref.set(fresh)
+        self._hub.publish(fresh)
+
 
 def cmd_update(args) -> int:
     """`cctally update` entry point — routes by mode flag (spec §4.1)."""

package/bin/_cctally_weekrefs.py

@@ -31,6 +31,7 @@ from __future__ import annotations
 import datetime as dt
 import sqlite3
 import sys
+import threading
 from dataclasses import replace
 
 from _cctally_core import (
@@ -220,6 +221,68 @@ def _apply_reset_events_to_weekrefs(
     return out
 
 
+# === #269 M4.4 — backfill-rescan process guard ==============================
+#
+# `_backfill_week_reset_events` rescans ALL `weekly_usage_snapshots` on every
+# `open_db()`. Its output is NOT a pure function of the snapshots alone: the
+# in-place-credit branch also reads existing `week_reset_events`, and the
+# reset-events-regenerate contract deletes `week_reset_events` and re-runs the
+# backfill WITHOUT a snapshot insert. So a `MAX(weekly_usage_snapshots.id)`-only
+# guard would skip a needed regeneration. Guard instead on a process-level memo
+# of BOTH `MAX(weekly_usage_snapshots.id)` AND a `week_reset_events` signature
+# `(COUNT(*), MAX(rowid))`, scoped per DB FILE identity: skip the rescan only
+# when both are unchanged since the last successful backfill. Byte-identical (a
+# snapshot insert advances the WUS id; a reset-event delete/clear moves the
+# reset-event sig; a backfill's own inserts move the sig so the next open is a
+# clean skip). Production snapshots are append-only, so the WUS id moves on
+# every new capture. In-memory / temp DBs (no file identity) are never memoized
+# (always rescanned) — a `:memory:` identity could otherwise collide across
+# distinct databases.
+
+_BACKFILL_MEMO_LOCK = threading.Lock()
+_BACKFILL_RESET_EVENTS_MEMO: dict = {}  # db_path -> (max_wus_id, (count, max_rowid))
+
+
+def _reset_backfill_reset_events_memo() -> None:
+    """Drop the backfill-guard memo (test hook + isolation)."""
+    with _BACKFILL_MEMO_LOCK:
+        _BACKFILL_RESET_EVENTS_MEMO.clear()
+
+
+def _backfill_db_identity(conn: sqlite3.Connection) -> "str | None":
+    """The `main` schema's on-disk file path, or None for in-memory/temp DBs
+    (empty file string) — the memo key. None ⇒ never memoize (always rescan)."""
+    try:
+        for _seq, name, file in conn.execute("PRAGMA database_list"):
+            if name == "main":
+                return file or None
+    except sqlite3.Error:
+        pass
+    return None
+
+
+def _backfill_reset_events_signature(
+    conn: sqlite3.Connection,
+) -> "tuple[int, tuple[int, int]]":
+    """`(MAX(weekly_usage_snapshots.id), (COUNT, MAX(rowid)) over
+    week_reset_events)` — two O(1) MAX/COUNT descents vs the full-table rescan.
+    Degrades to zero legs on a missing table so it never raises."""
+    try:
+        max_wus = conn.execute(
+            "SELECT COALESCE(MAX(id), 0) FROM weekly_usage_snapshots"
+        ).fetchone()[0]
+    except sqlite3.DatabaseError:
+        max_wus = 0
+    try:
+        row = conn.execute(
+            "SELECT COUNT(*), COALESCE(MAX(rowid), 0) FROM week_reset_events"
+        ).fetchone()
+        wre = (int(row[0]), int(row[1]))
+    except sqlite3.DatabaseError:
+        wre = (0, 0)
+    return (int(max_wus), wre)
+
+
 def _backfill_week_reset_events(conn: sqlite3.Connection) -> None:
     """One-shot scan over historical snapshots to synthesize reset events
     for past mid-week resets the tool lived through before this feature
@@ -244,6 +307,26 @@ def _backfill_week_reset_events(conn: sqlite3.Connection) -> None:
     window. See ``_is_reset_drop`` for the full rationale.
     """
     c = _cctally()
+    # #269 M4.4: skip the full rescan when neither the snapshots nor the
+    # reset events moved since the last successful backfill on this DB file.
+    #
+    # #271 §9d rider (from the #269 final review): the skip signature
+    # (`_backfill_reset_events_signature`, a MAX(id)/COUNT descent) is blind to a
+    # NON-max `weekly_usage_snapshots` DELETE — dropping a middle row leaves
+    # MAX(id) and could leave COUNT unchanged vs a prior state, so a rescan may
+    # be skipped. That is byte-safe because this backfill is ADD-ONLY
+    # (`INSERT OR IGNORE INTO week_reset_events`): a skipped rescan can only fail
+    # to *add* an event, never *remove* a needed one, matching the pre-#269
+    # idempotent behavior exactly.
+    db_id = _backfill_db_identity(conn)
+    sig = _backfill_reset_events_signature(conn)
+    if db_id is not None:
+        with _BACKFILL_MEMO_LOCK:
+            if _BACKFILL_RESET_EVENTS_MEMO.get(db_id) == sig:
+                # Preserve the original's transaction-flush behavior on the
+                # skip path (harmless when there is nothing pending).
+                conn.commit()
+                return
     try:
         rows = conn.execute(
             "SELECT captured_at_utc, week_end_at, weekly_percent "
@@ -372,6 +455,13 @@ def _backfill_week_reset_events(conn: sqlite3.Connection) -> None:
     # (e.g. _backfill_five_hour_blocks) don't trip "cannot start a
     # transaction within a transaction".
     conn.commit()
+    # #269 M4.4: store the POST-backfill signature (the scan's own inserts move
+    # the reset-event sig, so the next open with no snapshot change is a clean
+    # skip). Only after a completed scan — the early `except` return above
+    # deliberately does not memoize a partial state.
+    if db_id is not None:
+        with _BACKFILL_MEMO_LOCK:
+            _BACKFILL_RESET_EVENTS_MEMO[db_id] = _backfill_reset_events_signature(conn)
 
 
 # Minimum weekly_percent drop that counts as a goodwill reset (percentage
@@ -458,11 +548,20 @@ def _week_ref_has_reset_event(
     return row is not None
 
 
-def _compute_cost_for_weekref(ref: WeekRef) -> float | None:
+def _compute_cost_for_weekref(
+    ref: WeekRef, *, skip_sync: bool = False
+) -> float | None:
     """Live-compute USD cost over `ref`'s (possibly reset-adjusted) range
     straight from session_entries. Mirrors what cmd_sync_week writes into
     weekly_cost_snapshots, minus the cache write — used for reset-affected
     weeks where the cached range disagrees with the effective range.
+
+    ``skip_sync`` (default ``False``) is threaded to ``_sum_cost_for_range``
+    so the caller can read the cache without triggering a JSONL ingest.
+    The #268 dashboard/TUI sync-thread rebuild passes ``True`` (it ingests
+    once at the top of the rebuild); ``build_trend_view`` calls this once per
+    reset-event week, so without the flag each reset week re-globbed the whole
+    ``~/.claude/projects`` tree — the CPU peg the sync-once refactor removes.
     """
     c = _cctally()
     if not ref.week_start_at or not ref.week_end_at:
@@ -474,7 +573,7 @@ def _compute_cost_for_weekref(ref: WeekRef) -> float | None:
         return None
     if end <= start:
         return 0.0
-    return c._sum_cost_for_range(start, end, mode="auto")
+    return c._sum_cost_for_range(start, end, mode="auto", skip_sync=skip_sync)
 
 
 def _apply_overlap_clamp_to_weekrefs(refs: list[WeekRef]) -> list[WeekRef]:

package/bin/_lib_aggregators.py

@@ -106,6 +106,94 @@ class BucketUsage:
     model_breakdowns: list[dict[str, Any]]  # Sorted by cost desc
 
 
+def _new_bucket_acc() -> dict[str, Any]:
+    """A fresh running accumulator for one bucket (the pre-``BucketUsage`` shape).
+
+    The single per-bucket accumulation state shared by the full-pass
+    ``_aggregate_buckets`` and #271's incremental current-bucket accumulator, so
+    the byte-identity proof reduces to "same primitive, entries appended in the
+    same order" (spec §6).
+    """
+    return {
+        "input": 0, "output": 0, "cache_create": 0, "cache_read": 0,
+        "cost": 0.0, "models": {}, "models_order": [],
+    }
+
+
+def _fold_entry(acc: dict[str, Any], entry: UsageEntry, mode: str = "auto") -> None:
+    """Fold one entry into ``acc`` (the exact per-entry logic of ``_aggregate_buckets``).
+
+    Carries over verbatim: the ``<synthetic>`` skip (never touches ``acc`` for
+    them, so it is safe over an unfiltered delta stream), the ``-fast``
+    ``display_model`` suffix, the ``_calculate_entry_cost`` call, the four
+    integer token ``+=``, the running float ``cost +=``, the per-model
+    sub-bucket sums, and the first-seen ``models_order.append``. Order-sensitive:
+    the running ``cost`` is a left-fold, so callers MUST fold in the same order
+    the full pass would (the pinned ``(timestamp_utc, id)`` order, #271 §5).
+    """
+    if entry.model == "<synthetic>":
+        return
+    usage = entry.usage
+    display_model = f"{entry.model}-fast" if usage.get("speed") == "fast" else entry.model
+    inp = int(usage.get("input_tokens", 0) or 0)
+    out = int(usage.get("output_tokens", 0) or 0)
+    cc = int(usage.get("cache_creation_input_tokens", 0) or 0)
+    cr = int(usage.get("cache_read_input_tokens", 0) or 0)
+    cost = _calculate_entry_cost(
+        entry.model, usage, mode=mode, cost_usd=entry.cost_usd,
+    )
+    acc["input"] += inp
+    acc["output"] += out
+    acc["cache_create"] += cc
+    acc["cache_read"] += cr
+    acc["cost"] += cost
+    model_bucket = acc["models"].setdefault(display_model, {
+        "input": 0, "output": 0, "cache_create": 0, "cache_read": 0, "cost": 0.0,
+    })
+    model_bucket["input"] += inp
+    model_bucket["output"] += out
+    model_bucket["cache_create"] += cc
+    model_bucket["cache_read"] += cr
+    model_bucket["cost"] += cost
+    if display_model not in acc["models_order"]:
+        acc["models_order"].append(display_model)
+
+
+def _finalize_bucket(bucket_key: str, acc: dict[str, Any]) -> BucketUsage:
+    """Build the immutable ``BucketUsage`` from a running accumulator.
+
+    Copies ``models`` / ``model_breakdowns`` so the returned row shares no
+    mutable state with ``acc`` (F7 — a live accumulator may keep being folded,
+    e.g. #271's persisted current-bucket state).
+    """
+    model_breakdowns = [
+        {
+            "modelName": model,
+            "inputTokens": mb["input"],
+            "outputTokens": mb["output"],
+            "cacheCreationTokens": mb["cache_create"],
+            "cacheReadTokens": mb["cache_read"],
+            "cost": mb["cost"],
+        }
+        for model, mb in acc["models"].items()
+    ]
+    model_breakdowns.sort(key=lambda m: m["cost"], reverse=True)
+    total_tokens = (
+        acc["input"] + acc["output"] + acc["cache_create"] + acc["cache_read"]
+    )
+    return BucketUsage(
+        bucket=bucket_key,
+        input_tokens=acc["input"],
+        output_tokens=acc["output"],
+        cache_creation_tokens=acc["cache_create"],
+        cache_read_tokens=acc["cache_read"],
+        total_tokens=total_tokens,
+        cost_usd=acc["cost"],
+        models=list(acc["models_order"]),
+        model_breakdowns=model_breakdowns,
+    )
+
+
 def _aggregate_buckets(
     entries: list[UsageEntry],
     key_fn: Callable[[UsageEntry], str],
@@ -117,84 +205,23 @@ def _aggregate_buckets(
     The returned list is sorted by bucket key ascending — callers reverse
     for --order desc.  Model breakdowns within each bucket are sorted by
     descending cost, matching upstream ccusage.
+
+    The per-entry accumulation is the shared ``_fold_entry`` primitive (spec
+    §6): a ``<synthetic>`` entry is skipped BEFORE keying (so a synthetic-only
+    key never materializes an empty bucket), and each surviving entry is folded
+    into its bucket's running accumulator, finalized per key in ascending order.
     """
     by_bucket: dict[str, dict[str, Any]] = {}
-    models_order: dict[str, list[str]] = {}
-
     for entry in entries:
         if entry.model == "<synthetic>":
             continue
-        usage = entry.usage
-        display_model = f"{entry.model}-fast" if usage.get("speed") == "fast" else entry.model
         key = key_fn(entry)
-        bucket = by_bucket.setdefault(key, {
-            "input": 0,
-            "output": 0,
-            "cache_create": 0,
-            "cache_read": 0,
-            "cost": 0.0,
-            "models": {},
-        })
-        order = models_order.setdefault(key, [])
+        acc = by_bucket.get(key)
+        if acc is None:
+            by_bucket[key] = acc = _new_bucket_acc()
+        _fold_entry(acc, entry, mode)
 
-        inp = int(usage.get("input_tokens", 0) or 0)
-        out = int(usage.get("output_tokens", 0) or 0)
-        cc = int(usage.get("cache_creation_input_tokens", 0) or 0)
-        cr = int(usage.get("cache_read_input_tokens", 0) or 0)
-        cost = _calculate_entry_cost(
-            entry.model, usage, mode=mode, cost_usd=entry.cost_usd,
-        )
-
-        bucket["input"] += inp
-        bucket["output"] += out
-        bucket["cache_create"] += cc
-        bucket["cache_read"] += cr
-        bucket["cost"] += cost
-
-        model_bucket = bucket["models"].setdefault(display_model, {
-            "input": 0,
-            "output": 0,
-            "cache_create": 0,
-            "cache_read": 0,
-            "cost": 0.0,
-        })
-        model_bucket["input"] += inp
-        model_bucket["output"] += out
-        model_bucket["cache_create"] += cc
-        model_bucket["cache_read"] += cr
-        model_bucket["cost"] += cost
-
-        if display_model not in order:
-            order.append(display_model)
-
-    result: list[BucketUsage] = []
-    for key in sorted(by_bucket.keys()):
-        b = by_bucket[key]
-        model_breakdowns = [
-            {
-                "modelName": model,
-                "inputTokens": mb["input"],
-                "outputTokens": mb["output"],
-                "cacheCreationTokens": mb["cache_create"],
-                "cacheReadTokens": mb["cache_read"],
-                "cost": mb["cost"],
-            }
-            for model, mb in b["models"].items()
-        ]
-        model_breakdowns.sort(key=lambda m: m["cost"], reverse=True)
-        total_tokens = b["input"] + b["output"] + b["cache_create"] + b["cache_read"]
-        result.append(BucketUsage(
-            bucket=key,
-            input_tokens=b["input"],
-            output_tokens=b["output"],
-            cache_creation_tokens=b["cache_create"],
-            cache_read_tokens=b["cache_read"],
-            total_tokens=total_tokens,
-            cost_usd=b["cost"],
-            models=models_order[key],
-            model_breakdowns=model_breakdowns,
-        ))
-    return result
+    return [_finalize_bucket(key, by_bucket[key]) for key in sorted(by_bucket.keys())]
 
 
 def _aggregate_daily(