cctally 1.99.0 → 1.100.0

package/bin/_cctally_update.py

@@ -196,7 +196,7 @@ import threading
 import time
 import urllib.error
 import urllib.request
-from dataclasses import dataclass, replace
+from dataclasses import dataclass
 from typing import Any, Callable
 
 
@@ -2368,10 +2368,12 @@ class _DashboardUpdateCheckThread(threading.Thread):
     ``envelope_precompute`` rather than re-reading the state file per
     envelope build, so a BARE publish of the held snapshot would surface
     the stale precompute. The republish therefore rebuilds
-    ``envelope_precompute`` (small config/state JSON I/O, no DB, no
-    module-cache mutation — see :meth:`_republish_with_fresh_envelope`)
-    on a fresh ``dataclasses.replace`` snapshot before publishing, and
-    persists it to the snapshot ref so the held snapshot stays current.
+    ``envelope_precompute`` and the doctor summary on a fresh
+    ``dataclasses.replace`` snapshot before publishing, and persists it to
+    the snapshot ref so the held snapshot stays current. The doctor refresh
+    matters because ``safety.update_state`` reads the file this thread just
+    changed; retaining the old payload made ``/api/data`` disagree with the
+    live ``/api/doctor`` endpoint until a later data rebuild.
     Under ``--no-sync`` this thread is the ONLY refresher of that
     precompute (the data-sync thread that otherwise rebuilds it is
     disabled).
@@ -2385,11 +2387,13 @@ class _DashboardUpdateCheckThread(threading.Thread):
         *,
         hub: "SSEHub | None" = None,
         snapshot_ref: "_SnapshotRef | None" = None,
+        runtime_bind: "str | None" = None,
     ) -> None:
         super().__init__(name="cctally-update-check")
         self._stop_event = stop_event
         self._hub = hub
         self._ref = snapshot_ref
+        self._runtime_bind = runtime_bind
 
     def run(self) -> None:
         c = _cctally()
@@ -2412,16 +2416,12 @@ class _DashboardUpdateCheckThread(threading.Thread):
                     and self._hub is not None
                     and self._ref is not None
                 ):
-                    snap = self._ref.get()
-                    if snap is not None:
-                        self._republish_with_fresh_envelope(snap)
+                    self._republish_with_fresh_envelope()
                 config = load_config()
                 if c._is_update_check_due(config):
                     c._do_update_check()
                     if self._hub is not None and self._ref is not None:
-                        snap = self._ref.get()
-                        if snap is not None:
-                            self._republish_with_fresh_envelope(snap)
+                        self._republish_with_fresh_envelope()
             except Exception as e:
                 # Log but never propagate — this thread must keep
                 # ticking so a transient registry hiccup doesn't
@@ -2437,8 +2437,8 @@ class _DashboardUpdateCheckThread(threading.Thread):
                     pass
             self._stop_event.wait(c.UPDATE_DASHBOARD_CHECK_POLL_S)
 
-    def _republish_with_fresh_envelope(self, snap) -> None:
-        """Republish ``snap`` with a freshly-rebuilt ``envelope_precompute``.
+    def _republish_with_fresh_envelope(self) -> None:
+        """Patch the latest snapshot with fresh config and doctor precomputes.
 
         Since M4 (#268) the SSE envelope reads update-state / update-suppress
         off ``snap.envelope_precompute`` rather than re-reading the JSON files
@@ -2450,26 +2450,32 @@ class _DashboardUpdateCheckThread(threading.Thread):
         precompute refresher) is disabled.
 
         ``_tui_precompute_envelope_config`` reads only the config / update-state
-        / update-suppress files and mutates NO module cache, so calling it from
-        this thread is safe (no shared-cache-mutation / Codex F7 hazard).
-        ``dataclasses.replace`` yields a NEW snapshot sharing ``prior``'s
-        immutable rows, so an SSE client serializing the previously-published
-        snapshot can't observe a torn value. The fresh snapshot is persisted to
-        the ref so the held snapshot stays current for the next reader.
+        / update-suppress files and mutates NO module cache. The doctor memo is
+        invalidated deliberately before recomputation: the update-state write
+        changed an input to ``safety.update_state``, so the ordinary 30-second
+        memo would otherwise hand this republish the stale summary again.
+        ``_SnapshotRef.replace_fields`` applies only these three narrow fields
+        to the latest held object under its lock. That atomic patch matters
+        because a full sync can publish while the precomputes above are being
+        gathered; writing a whole snapshot read before that sync would regress
+        every data field and can restore a partial hydration seed.
         """
         c = _cctally()
-        fresh = replace(
-            snap,
+        c._load_sibling("_lib_snapshot_cache").reset_doctor_memo()
+        doctor_payload = c._cctally_tui._tui_precompute_doctor_payload(
+            _now_utc(), self._runtime_bind,
+        )
+        fresh = self._ref.replace_fields(
             envelope_precompute=(
                 c._cctally_tui._tui_precompute_envelope_config(load_config())
             ),
+            doctor_payload=doctor_payload,
             # #278 §1.4.1: a version-banner refresh republishes complete data;
             # force the hydration latch clear so a republish that happens to
             # carry a prior hydrating seed/partial doesn't freeze the client's
             # loading skeletons.
             hydrating=False,
         )
-        self._ref.set(fresh)
         self._hub.publish(fresh)
 
 

package/bin/_lib_dashboard_sources.py

@@ -93,7 +93,15 @@ CapabilityStatus = Literal[
 # optional `spendWindow` bounds, which a cycle-bounded card omits. No client
 # branches on this number; after an in-place `execvp` update a still-loaded old
 # client renders the new figures under old copy until it reloads.
-SOURCE_SCHEMA_VERSION = 9
+# 9 -> 10 (#583 S3): `sources.all.data.providers` no longer carries the two
+# provider data objects. They are published once, on the physical
+# `sources.claude` / `sources.codex` entries, which is where every production
+# consumer already falls back to. The key itself is RETAINED with both members
+# null so a still-loaded v9 bundle does not throw — this is the first
+# DESTRUCTIVE entry in this ledger rather than an additive one, and the null
+# stub is what makes it survivable across an in-place `execvp` update. Removing
+# the stub needs a separate change once no v9 bundle can be live.
+SOURCE_SCHEMA_VERSION = 10
 DEFAULT_SOURCE = "claude"
 SOURCE_ORDER = ("claude", "codex", "all")
 SOURCE_FRESHNESS_DOMAINS = ("hero", "quota", "sessions")
@@ -1217,13 +1225,24 @@ def compose_all_state(
                if combined is None else {}),
             "alerts": {"rows": combined_alerts},
             # #556 S2 §3.5.1: the ONE public copy of the shared range and of
-            # both aggregate outcomes. The rows stay on the provider domains
-            # under `providers` below, so nothing is published twice.
+            # both aggregate outcomes. The rows stay on the provider domains,
+            # published on the physical `sources.claude` / `sources.codex`
+            # entries since #583 S3, so nothing is published twice.
             "aggregates": aggregates,
-            "providers": {
-                "claude": claude.data,
-                "codex": codex.data,
-            },
+            # #583 S3 §4: the provider data is published ONCE, on the physical
+            # `sources.claude` / `sources.codex` entries. This mirror duplicated
+            # about 1.56 MB of a 3.25 MB envelope by reference.
+            #
+            # The key is RETAINED with null members rather than removed.
+            # `dashboard/web/src/lib/dashboardPresentation.ts` reads
+            # `data?.providers.claude` — the optional chain guards `data`, not
+            # `providers` — so an absent key throws a TypeError inside a render
+            # for any tab still running the v9 bundle when `cctally update`
+            # `execvp`s the server underneath it. Both members are declared
+            # nullable in `types/envelope.ts`, so null is a legal v9 value and
+            # every v9 consumer's `?? sources.<p>.data` fallback resolves.
+            # Retiring the stub is filed as a residual, not done here.
+            "providers": {"claude": None, "codex": None},
         },
         domain_freshness={
             domain: (

package/bin/_lib_doctor.py

@@ -232,6 +232,11 @@ class DoctorState:
     # DOCTOR_WAL_WARN_BYTES (2x the WAL cap) — only when the journal_size_limit
     # + forced-checkpoint machinery has genuinely failed to contain the WAL.
     cache_db_wal_bytes: Optional[int] = None
+    # #583 S4 (F39): size in bytes of conversations.db-wal, gathered the same
+    # read-only way as cache_db_wal_bytes above (absent -> 0, OSError -> None).
+    # STORE_POLICY gives conversations.db the same 128 MiB journal_size_limit,
+    # so it warns above the SAME DOCTOR_WAL_WARN_BYTES threshold.
+    conversations_db_wal_bytes: Optional[int] = None
     # #496 S5b: the durable incomplete-quota-projection flag carried inside the
     # published stats generation. True = every quota-projection read is refused
     # until a reconciliation runs; False = the generation is complete; None =
@@ -2294,6 +2299,37 @@ def _check_db_wal_size(s: DoctorState) -> CheckResult:
     )
 
 
+def _check_db_conversations_wal_size(s: DoctorState) -> CheckResult:
+    """Read-only backstop for the transcript store's WAL (#583 S4 / F39).
+
+    Sibling of `_check_db_wal_size`, sharing DOCTOR_WAL_WARN_BYTES because
+    STORE_POLICY gives conversations.db the same 128 MiB journal_size_limit as
+    cache.db — so the same "only fires when containment has genuinely failed"
+    reasoning applies. Same fingerprint discipline: the exact byte count lives
+    ONLY in the excluded details block and the summary is a stable string, so
+    below-threshold drift cannot flip the fingerprint while an OK<->WARN
+    crossing does.
+    """
+    wal = s.conversations_db_wal_bytes
+    details = {"conversations_db_wal_bytes": wal}
+    if isinstance(wal, int) and wal > DOCTOR_WAL_WARN_BYTES:
+        return CheckResult(
+            id="db.conversations_wal_size",
+            title="conversations.db WAL size", severity="warn",
+            summary="oversized — conversations.db WAL far above its cap",
+            remediation=(
+                "Run `cctally db checkpoint --db conversations` "
+                "to drain the WAL."
+            ),
+            details=details,
+        )
+    return CheckResult(
+        id="db.conversations_wal_size",
+        title="conversations.db WAL size", severity="ok",
+        summary="within limit", remediation=None, details=details,
+    )
+
+
 # #315: conservative advisory threshold. A quarter of cache.db being free is
 # large enough to make an explicit, guarded VACUUM useful without nagging for
 # ordinary page churn. This is a ratio, so it remains page-size independent.
@@ -3449,6 +3485,7 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
         ("db.migrations.pending", "_check_db_migrations_pending"),
         ("db.lock_state", "_check_db_lock_state"),
         ("db.wal_size", "_check_db_wal_size"),
+        ("db.conversations_wal_size", "_check_db_conversations_wal_size"),
         ("db.reclaimable", "_check_db_reclaimable"),
         ("db.conversations_reclaimable", "_check_db_conversations_reclaimable"),
         ("db.retained_artifacts", "_check_db_retained_artifacts"),

package/bin/_lib_jsonl.py

@@ -65,12 +65,14 @@ class UsageEntry:
                        # crashes loudly at construction instead).
 
 
-@dataclass
+@dataclass(frozen=True)
 class CodexEntry:
     """One emitted Codex `token_count` event row.
 
     Mirrors the columns of codex_session_entries. `last_token_usage` fields
-    are used (per-turn deltas), not the cumulative totals.
+    are used (per-turn deltas), not the cumulative totals. Instances are
+    immutable because dashboard folds intentionally alias one entry between
+    the merged parent and its account partition.
     """
     timestamp: dt.datetime
     session_id: str

package/bin/_lib_perf.py

@@ -15,6 +15,7 @@ and fields may change without a version bump.
 """
 from __future__ import annotations
 
+import contextlib
 import os
 import sys
 import threading
@@ -29,8 +30,18 @@ def enabled() -> bool:
 
 
 def set_enabled(value: bool) -> None:
-    """Flip tracing at runtime (tests; not used by the dashboard, which reads
-    the env at import time)."""
+    """Flip tracing immediately (tests, and ``apply_pending`` below).
+
+    Runtime arming goes through ``request_enabled`` + ``apply_pending`` instead
+    (#583 S1 §2.2), which defers the flip to a rebuild boundary. This entry
+    point stays for tests.
+
+    It deliberately does NOT clear a captured root arm: an already-open root is
+    wholly traced or wholly untraced whatever the global does, and that is the
+    property the per-root capture exists to give. Every caller that wants the
+    new value to take effect calls ``reset_thread()`` afterwards, which is what
+    both production root sites already do.
+    """
     global _ENABLED
     _ENABLED = bool(value)
 
@@ -38,6 +49,97 @@ def set_enabled(value: bool) -> None:
 _tls = threading.local()
 
 
+# ── runtime arming: an atomic mailbox, applied per root (#583 S1 §2.2) ──────
+# `request_enabled` records the latest desired state and `apply_pending`
+# consumes it, both under one lock performing a LATEST-VALUE EXCHANGE — read
+# and clear inside one critical section. An unsynchronised test-and-clear loses
+# a request arriving between the two steps, which is the race
+# `_SnapshotRef.take_sync_request` already documents in this repository.
+_UNSET = object()
+_MAILBOX_LOCK = threading.Lock()
+_PENDING = _UNSET
+
+
+def request_enabled(value: bool) -> None:
+    """Record the desired tracing state. Applies at the next boundary."""
+    global _PENDING
+    with _MAILBOX_LOCK:
+        _PENDING = bool(value)
+
+
+def apply_pending() -> bool:
+    """Consume any pending request and apply it. Returns whether it changed.
+
+    Called ONLY from `_make_run_sync_now_locked._locked`, after the cache
+    connection closes and immediately before the authoritative build, so a
+    request arriving mid-ingest cannot split one ingest across two tracing
+    states. A2 partial builds never consume it.
+    """
+    global _PENDING
+    with _MAILBOX_LOCK:
+        value, _PENDING = _PENDING, _UNSET
+    if value is _UNSET:
+        return False
+    changed = bool(value) != _ENABLED
+    set_enabled(bool(value))
+    return changed
+
+
+def pending_state() -> "tuple[bool, bool]":
+    """``(requested, applied)``. Equal once the request has taken effect."""
+    with _MAILBOX_LOCK:
+        pending = _PENDING
+    applied = _ENABLED
+    return (applied if pending is _UNSET else bool(pending), applied)
+
+
+def applies_at() -> str:
+    """When a pending request takes effect. Names the boundary, not a clock.
+
+    Without this the operator cannot tell a request from its effect, and
+    ``--trace off`` appears to succeed while tracing is still applied until the
+    next authoritative build.
+    """
+    requested, applied = pending_state()
+    return "next_authoritative_build" if requested != applied else "none"
+
+
+def root_armed() -> "bool | None":
+    """The tracing state this thread's open root captured, or None if no root
+    scope is open on this thread."""
+    return getattr(_tls, "root_armed", None)
+
+
+@contextlib.contextmanager
+def isolated_thread_state():
+    """Run a nested build against empty, private phase state (#583 S1 §2.1).
+
+    ``_make_a2_progress_cb`` calls ``build_partial()`` synchronously from
+    inside ``sync_cache``'s still-open ``walk`` phase, and that build reaches
+    ``_tui_build_snapshot_once``'s unconditional ``reset_thread()``. Each
+    ``Phase`` retains its original list in ``Phase._stack`` while
+    ``reset_thread()`` rebinds ``_tls.stack`` and clears ``_tls.root``, so
+    without isolation the later phases attach to a different list and the outer
+    phase closes into a detached or fragmented root.
+
+    Saves the EXACT stack and root object references — by identity, because
+    the open phases hold that same list — and restores them in ``finally``,
+    including when the body raises.
+    """
+    saved_stack = getattr(_tls, "stack", None)
+    saved_root = getattr(_tls, "root", None)
+    saved_armed = getattr(_tls, "root_armed", None)
+    _tls.stack = []
+    _tls.root = None
+    _tls.root_armed = None
+    try:
+        yield
+    finally:
+        _tls.stack = saved_stack
+        _tls.root = saved_root
+        _tls.root_armed = saved_armed
+
+
 def _stack():
     s = getattr(_tls, "stack", None)
     if s is None:
@@ -90,6 +192,9 @@ class Phase:
             stack[-1].children.append(self)
         else:
             _tls.root = self          # outermost phase closed -> the build root
+            # The root scope ends with its root, so a later phase on this
+            # thread reads the global again rather than a stale capture.
+            _tls.root_armed = None
         return False
 
     def to_dict(self):
@@ -123,7 +228,14 @@ _NULL_PHASE = _NullPhase()
 
 
 def phase(name):
-    if not _ENABLED:
+    # Per-root capture (#583 S1 §2.2). `_ENABLED` is process-global and HTTP
+    # handlers open their own roots on their own threads, so a flip landing
+    # mid-request would otherwise create traced phases beneath an untraced root
+    # or drop later children from a traced one. A root captures the armed state
+    # at its own creation and every phase under it consults that value; a
+    # thread with no root scope open falls back to the global.
+    armed = getattr(_tls, "root_armed", None)
+    if not (_ENABLED if armed is None else armed):
         return _NULL_PHASE
     return Phase(name, _stack())
 
@@ -133,8 +245,25 @@ def current_root():
 
 
 def reset_thread():
+    """Start a fresh root scope on this thread, capturing the armed state.
+
+    Every root in this tree is opened immediately after a `reset_thread()` —
+    `_tui_build_snapshot_once` and the dashboard's `_perf_scope` are the two
+    sites — so this is where a root's tracing decision is taken. The scope ends
+    when the outermost phase closes, or when the next `reset_thread()` recaptures.
+
+    KNOWN LIMIT, recorded rather than fixed. A DISARMED capture is not cleared
+    by the closing phase, because no `Phase` is created to close: a thread that
+    captures `False` and then never calls `reset_thread()` again keeps reading
+    that stale value and stays untraced. Unreachable today — the dashboard
+    threads per request and resets at the top of every build — but it would
+    bite a thread-POOLED server, where a worker outlives many requests. Fixing
+    it needs an explicit root scope that exists when tracing is off, not a
+    smarter capture point.
+    """
     _tls.stack = []
     _tls.root = None
+    _tls.root_armed = _ENABLED
 
 
 def flush_stderr(root):

package/bin/_lib_source_analytics.py

@@ -221,8 +221,8 @@ def _totals(entries: Iterable[QualifiedCodexEntry]) -> TokenTotals:
     # `build_codex_project_result`, over the complete population, before any
     # subset is taken -- which is the only place it can be correct. Guarding
     # this call on "the subset holds more than one project identity" would
-    # have kept nearly all of the cost, because the expensive callers are the
-    # per-block totals over a population spanning every project.
+    # have kept roughly three-quarters of the cost, because the expensive
+    # callers are the per-block totals over a population spanning every project.
     values = tuple(entries)
     return TokenTotals(
         input_tokens=sum(entry.input_tokens for entry in values),