cctally 1.88.2 → 1.89.1

package/bin/_lib_dashboard_sources.py

@@ -29,7 +29,11 @@ CapabilityStatus = Literal[
 # (`UpdateRunningModal.tsx`, `store/sse.ts`), so an old client demonstrably
 # does meet a new server; `docs/cli-contract.md` calls changing a value's
 # meaning breaking. Version-aware client reaction is deliberately deferred.
-SOURCE_SCHEMA_VERSION = 2
+# 2 -> 3 (public #5): the Codex source gained the additive `ingest_backlog`
+# field. Additive and omitted-when-zero, so the normal payload is byte-identical
+# — but the same `execvp` transition applies, so the bump ships as the signal it
+# has always been rather than as a mechanism the client branches on.
+SOURCE_SCHEMA_VERSION = 3
 DEFAULT_SOURCE = "claude"
 SOURCE_ORDER = ("claude", "codex", "all")
 SOURCE_FRESHNESS_DOMAINS = ("hero", "quota", "sessions")

package/bin/_lib_doctor.py

@@ -189,6 +189,13 @@ class DoctorState:
     # by the next whole-tree sync that defers none. None = key absent (the
     # normal state) or cache unreadable — the check degrades OK.
     codex_torn_deferred: Optional[dict] = None
+    # public #5 spec §5: what the hook's BUDGETED ingest leg has left to read.
+    # Mirrors the `codex_ingest_backlog` cache_meta record — `{files, bytes,
+    # since}` — or None when the backlog is empty (the record is deleted, never
+    # zeroed). A non-zero backlog is NORMAL right after a heavy burst, so the
+    # leg distinguishes draining from stuck on the `since` stamp rather than on
+    # the counts.
+    codex_ingest_backlog: Optional[dict] = None
     # The byte-zero Codex replay's stall signal. `codex_replay_pending` mirrors
     # the `codex_replay_from_zero_pending` cache_meta marker; `codex_replay_blocked`
     # is `{"at": iso, "files_failed": N, "files_deferred_torn": N}`, written by a
@@ -197,6 +204,12 @@ class DoctorState:
     # state) or cache unreadable — the check degrades OK.
     codex_replay_pending: Optional[bool] = None
     codex_replay_blocked: Optional[dict] = None
+    # `codex_replay_deferred` is `{"since": iso, "at": iso}`, written when a
+    # BUDGETED tick (the Codex hook) declined the replay outright. A hook-only
+    # install produces this and nothing else — the decline returns before the
+    # walk, so `files_failed`/`files_deferred_torn` are both zero, no `blocked`
+    # record is ever written, and the ingest-backlog leg reads a drained store.
+    codex_replay_deferred: Optional[dict] = None
     # #279 S2 (F5b): PRAGMA quick_check(1) results, gathered ONLY under
     # doctor_gather_state(deep=True) (CLI cmd_doctor) — the dashboard
     # rebuild loop calls the gather every rebuild and quick_check on a
@@ -229,6 +242,13 @@ class DoctorState:
     codex_quota_windows: Optional[list[dict]] = None
     codex_hook_roots: Optional[list[dict]] = None
     codex_lifecycle_activity_24h: Optional[dict] = None
+    # public #5: 24h outcome counts for the detached `_codex-quota-verify`
+    # worker, parsed from the same bounded log. The lifecycle parser above
+    # cannot supply these — worker lines carry no `source_root_key`, so its
+    # root filter drops every one. Shape:
+    # {"success_count_24h", "error_count_24h", "spawn_failure_count_24h",
+    #  "last_success_at"}. None/absent degrades the check to OK.
+    codex_quota_verify_activity: Optional[dict] = None
     # #311: precomputed five-state classification of settings.json's
     # statusLine.refreshInterval (unavailable/absent/foreign/present/missing),
     # computed by doctor_gather_state via the setup I/O-layer classifier so the
@@ -1115,6 +1135,26 @@ def _check_data_codex_cache(s: DoctorState) -> CheckResult:
     )
 
 
+def _age_seconds(stamp: object, now_utc: dt.datetime) -> Optional[int]:
+    """Seconds between an ISO-8601 UTC stamp and ``now_utc``; None if unusable."""
+    if not stamp:
+        return None
+    try:
+        parsed = dt.datetime.fromisoformat(str(stamp).replace("Z", "+00:00"))
+    except (ValueError, TypeError):
+        return None
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=dt.timezone.utc)
+    return int((now_utc - parsed.astimezone(dt.timezone.utc)).total_seconds())
+
+
+#: How long a budgeted decline may stand before it stops being self-healing.
+#: The hook hands the drain to a detached worker on every decline (throttled
+#: hourly), so a deferral older than this means the hand-off is not landing.
+#: Same hour as `CODEX_INGEST_BACKLOG_STUCK_SECONDS`, for the same reason.
+CODEX_REPLAY_DEFERRED_WARN_SECONDS = 3600
+
+
 def _check_data_codex_replay(s: DoctorState) -> CheckResult:
     """WARN while a byte-zero Codex replay is STALLED rather than merely pending.
 
@@ -1131,14 +1171,27 @@ def _check_data_codex_replay(s: DoctorState) -> CheckResult:
     between the migration and the next sync, and it clears on its own. The WARN
     needs the durable `blocked` record, which only a completed-but-unsuccessful
     whole-tree walk writes.
+
+    The `deferred` record is the second stall shape (public #5). A BUDGETED tick
+    cannot perform the replay at all, so the Codex hook declines it — and on a
+    hook-only install no unbudgeted caller exists, which froze every Codex sync
+    while `files_failed` and `files_deferred_torn` both stayed zero and no
+    `blocked` record was ever written. The hook now hands the drain to a
+    detached worker, so a RECENT deferral is the ordinary self-healing state and
+    stays `ok`; one that has stood for over an hour means the hand-off is not
+    landing and is the only signal that the install is frozen.
     """
     blocked = s.codex_replay_blocked or {}
+    deferred = s.codex_replay_deferred or {}
     at = blocked.get("at")
+    deferred_since = deferred.get("since")
     details = {
         "pending": bool(s.codex_replay_pending),
         "blocked_at": at,
         "files_failed": blocked.get("files_failed"),
         "files_deferred_torn": blocked.get("files_deferred_torn"),
+        "deferred_since": deferred_since,
+        "deferred_at": deferred.get("at"),
     }
     if s.codex_replay_pending and isinstance(at, str) and at:
         return CheckResult(
@@ -1150,15 +1203,105 @@ def _check_data_codex_replay(s: DoctorState) -> CheckResult:
                          "is truncated), then `cctally cache-sync --source codex`"),
             details=details,
         )
+    if s.codex_replay_pending and isinstance(deferred_since, str) and deferred_since:
+        age_s = _age_seconds(deferred_since, s.now_utc)
+        if age_s is not None and age_s > CODEX_REPLAY_DEFERRED_WARN_SECONDS:
+            return CheckResult(
+                id="data.codex_replay", title="Codex transcript replay",
+                severity="warn",
+                summary=(f"the Codex hook has deferred it since "
+                         f"{deferred_since} — a budgeted tick cannot run the "
+                         "replay, and all Codex ingest is frozen until an "
+                         "unbudgeted sync does"),
+                remediation="Run `cctally cache-sync --source codex`",
+                details=details,
+            )
+        return CheckResult(
+            id="data.codex_replay", title="Codex transcript replay",
+            severity="ok",
+            summary=(f"pending; the Codex hook deferred it at "
+                     f"{deferred.get('at') or deferred_since} and handed the "
+                     "drain to a background worker"),
+            remediation=None, details=details,
+        )
     return CheckResult(
         id="data.codex_replay", title="Codex transcript replay",
         severity="ok",
-        summary="pending (clears on the next Codex sync)"
+        # NOT "clears on the next Codex sync": a budgeted tick — the Codex
+        # hook — declines the replay rather than running it, so on a hook-only
+        # install the next sync is precisely what does not clear it.
+        summary="pending (clears on the next unbudgeted Codex sync)"
                 if s.codex_replay_pending else "none pending",
         remediation=None, details=details,
     )
 
 
+def _check_data_codex_quota_verification(s: DoctorState) -> CheckResult:
+    """WARN when the detached Codex quota verification is not landing.
+
+    Public #5 moved every whole-history projection pass off the blocking hook
+    path, so on a hook-only install ALL of it now depends on the
+    `_codex-quota-verify` worker. Two of its routes make that dependency sharp
+    rather than cosmetic: the catch-all (a rebuilt stats index, an
+    interpretation bump, a reset ledger, a ledgerless cache) performs no
+    projection work at all, which leaves the projection transiently MISSING —
+    no blocks, no milestones, no alerts — until the worker converges it.
+    Nothing else observes that. `data.codex_quota` is a different question
+    entirely: it reports the freshness of the local rollout OBSERVATIONS, which
+    stay perfectly fresh while the projection derived from them is absent.
+
+    The worker's own streams are `/dev/null` and its exit code is observed by
+    nobody, so it writes its outcome to `hook-tick.log` for exactly this leg to
+    read. The predicate is "failures with no success in 24h": one failed
+    hand-off is ordinary (a contended spawn, a transient lock) and self-heals on
+    the next throttle window, while a worker that has not landed once in a day
+    is the persistent condition — and because the deadline only moves when a
+    pass COMMITS, such a worker re-spawns forever with nothing to show for it.
+
+    Silence is OK, not a failure: an install with no Codex hooks never hands off
+    at all, and every non-hook caller runs the pass inline.
+    """
+    activity = s.codex_quota_verify_activity or {}
+    successes = int(activity.get("success_count_24h") or 0)
+    errors = int(activity.get("error_count_24h") or 0)
+    spawn_failures = int(activity.get("spawn_failure_count_24h") or 0)
+    last_success = activity.get("last_success_at")
+    details = {
+        "success_count_24h": successes,
+        "error_count_24h": errors,
+        "spawn_failure_count_24h": spawn_failures,
+        "last_success_at": (
+            _iso_z(last_success) if isinstance(last_success, dt.datetime)
+            else (last_success if isinstance(last_success, str) else None)
+        ),
+    }
+    failures = errors + spawn_failures
+    if failures and not successes:
+        return CheckResult(
+            id="data.codex_quota_verification",
+            title="Codex quota verification",
+            severity="warn",
+            summary=(
+                f"{failures} failed hand-off(s) and no completed pass in 24h — "
+                "the Codex quota projection is not being verified"),
+            remediation=(
+                "Run `cctally cache-sync --source codex` (every non-hook caller "
+                "runs the pass inline), then check hook-tick.log for "
+                "`op=quota-verify`"),
+            details=details,
+        )
+    return CheckResult(
+        id="data.codex_quota_verification",
+        title="Codex quota verification",
+        severity="ok",
+        summary=(
+            f"{successes} completed, {failures} failed in 24h"
+            if successes or failures else "no hand-off in the last 24h"),
+        remediation=None,
+        details=details,
+    )
+
+
 def _check_data_codex_project_metadata(s: DoctorState) -> CheckResult:
     """Report the identity-safe all-history Codex metadata partition."""
     if s.codex_project_metadata_error is not None:
@@ -2621,6 +2764,61 @@ def _check_accounts_codex_identity(s: DoctorState) -> CheckResult:
     )
 
 
+#: A backlog younger than this is draining, not stuck. A heavy Codex burst
+#: legitimately leaves work for the next few ticks; only a backlog that has
+#: stayed non-zero CONTINUOUSLY past this is worth an operator's attention.
+CODEX_INGEST_BACKLOG_STUCK_SECONDS = 3600
+
+
+def _check_data_codex_ingest_backlog(s: DoctorState) -> CheckResult:
+    """WARN when the hook's budgeted Codex ingest has been behind for over an hour.
+
+    The hook's ingest leg is bounded in wall clock (public #5 spec §4), so it
+    can legitimately leave a backlog — that is the mechanism working, not a
+    fault. What is worth reporting is a backlog that never drains: a store
+    whose per-tick budget is smaller than its per-tick growth, or a walk that
+    has stopped making progress.
+
+    Per the CLI contract a WARN alone does not change `doctor`'s exit code.
+    """
+    record = s.codex_ingest_backlog or {}
+    try:
+        files = int(record.get("files") or 0)
+    except (TypeError, ValueError):
+        files = 0
+    try:
+        pending_bytes = int(record.get("bytes") or 0)
+    except (TypeError, ValueError):
+        pending_bytes = 0
+    since_text = record.get("since")
+    age_s = _age_seconds(since_text, s.now_utc)
+    details = {"files": files, "bytes": pending_bytes, "since": since_text,
+               "age_s": age_s}
+    if files <= 0:
+        return CheckResult(
+            id="data.codex_ingest_backlog", title="Codex ingest backlog",
+            severity="ok", summary="no Codex ingest backlog",
+            remediation=None, details=details,
+        )
+    if age_s is not None and age_s > CODEX_INGEST_BACKLOG_STUCK_SECONDS:
+        return CheckResult(
+            id="data.codex_ingest_backlog", title="Codex ingest backlog",
+            severity="warn",
+            summary=(f"{files} Codex rollout(s) / {pending_bytes} byte(s) still "
+                     f"unread after {age_s // 3600}h — the hook's budget is not "
+                     "draining them"),
+            remediation="Run `cctally cache-sync --source codex`",
+            details=details,
+        )
+    return CheckResult(
+        id="data.codex_ingest_backlog", title="Codex ingest backlog",
+        severity="ok",
+        summary=(f"{files} Codex rollout(s) / {pending_bytes} byte(s) queued — "
+                 "draining"),
+        remediation=None, details=details,
+    )
+
+
 def _check_accounts_attribution(s: DoctorState) -> CheckResult:
     """WARN when Claude usage is flowing but landing in `unattributed` despite a
     resolved active account (the stamping pipeline is broken), or while the
@@ -2738,7 +2936,10 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
         # after the pair rather than between them.
         ("data.codex_project_metadata", "_check_data_codex_project_metadata"),
         ("data.codex_replay", "_check_data_codex_replay"),
+        ("data.codex_ingest_backlog", "_check_data_codex_ingest_backlog"),
         ("data.codex_quota", "_check_data_codex_quota"),
+        ("data.codex_quota_verification",
+         "_check_data_codex_quota_verification"),
         ("data.parse_health", "_check_data_parse_health"),
         ("data.forked_buckets", "_check_data_forked_buckets"),
         ("data.post_credit_milestones", "_check_data_post_credit_milestones"),

package/bin/_lib_jsonl.py

@@ -986,12 +986,21 @@ def _iter_codex_fused_records_with_offsets(
     initial_total_tokens: int = 0,
     source_root_key: str | None = None,
     state: _CodexIterState | None = None,
+    stop_before=None,
 ):
     """Yield typed facts for every complete, valid Codex JSONL object.
 
     Binary readers retain physical byte offsets and strictly reject invalid UTF-8,
     non-finite JSON constants, and non-object JSON.  The mutable state carries
     the shipped accounting resume contract alongside the latest thread facts.
+
+    ``stop_before`` (public #5) is an optional predicate called with the byte
+    offset of each line BEFORE it is read; returning True ends the iteration
+    there, leaving ``fh`` positioned exactly at that offset so the caller can
+    persist it as a resume point. It sits inside the reader rather than in the
+    consuming loop because most lines yield nothing at all — a long run of
+    non-emitting records would otherwise never hand control back, which is
+    precisely the case a wall-clock budget has to survive.
     """
     if state is None:
         state = _CodexIterState()
@@ -1003,6 +1012,9 @@ def _iter_codex_fused_records_with_offsets(
 
     while True:
         line_offset = fh.tell()
+        if stop_before is not None and stop_before(line_offset):
+            fh.seek(line_offset)
+            return
         line = fh.readline()
         if not line:
             return

package/bin/_lib_quota_alert_axes.py

@@ -0,0 +1,188 @@
+"""Pure kernel deciding what an alert pass has to look at (public #5 §3).
+
+Alert state is NOT a function of window dirtiness, so a projector bounded by the
+change ledger would silently stop honouring four of the five things that can
+make an alert eligible. Each axis below is a way the answer changes with no row
+in ``quota_window_snapshots`` moving at all:
+
+1. **Physical-window dirtiness** — the ledger. The only axis the ledger sees.
+2. **Policy scope** — the resolved rule changed. ``quota_rule_fingerprint``
+   already hashes the resolved thresholds, gates, root and logical limit, and
+   is persisted per identity in ``quota_alert_arming``; comparing it against a
+   freshly resolved one detects the change. An exact-rule change is scoped to
+   its identities, a default/global change marks everything.
+3. **Delivery gate** — the global or quota switch flipped. DISABLING must
+   enumerate every Codex arming row, delete it and journal a disarm, and must do
+   so even with zero dirty windows and zero lifecycle-eligible roots; that is
+   why it needs no observation load and why the caller runs it before the
+   eligibility fast path. ENABLING is the opposite: it needs a semantic pass
+   over the affected identities regardless of physical dirtiness, because
+   activation has to write ``suppressed_backfill`` terminal rows for
+   already-satisfied thresholds instead of dispatching history.
+4. **Scheduled time** — an observation captured in the future is skipped today,
+   and becomes eligible when wall time passes it with no mutation to observe.
+   Persisting that boundary and treating ``now >= boundary`` as dirty is what
+   closes it. Unlike axes 2 and 3 this one fires on WALL CLOCK rather than on a
+   configuration change, which is why the hook path defers it
+   (``defer_scheduled``) instead of paying an unannounced whole-history pass on
+   a blocking tick.
+5. **Durable lifecycle state** — the existing arming rows and terminal events,
+   unchanged. Represented here only as the fingerprints axis 2 compares.
+
+The scope this returns is deliberately a SUPERSET decision: widening a bounded
+pass is always safe, missing an identity is not.
+"""
+from __future__ import annotations
+
+import datetime as dt
+from dataclasses import dataclass
+from typing import Iterable, Mapping
+
+
+#: Ordered weakest to strongest; a later value absorbs an earlier one.
+SCOPE_NONE = "none"
+SCOPE_GROUPS = "groups"
+SCOPE_ROOTS = "roots"
+SCOPE_ALL = "all"
+
+_ORDER = {SCOPE_NONE: 0, SCOPE_GROUPS: 1, SCOPE_ROOTS: 2, SCOPE_ALL: 3}
+
+#: Recorded in place of ``"scheduled"`` when axis 4 came due but the caller
+#: cannot honour it inline. A caller that sees it MUST carry the stored
+#: boundary through untouched — the axis is postponed, not satisfied.
+REASON_SCHEDULED_DEFERRED = "scheduled_deferred"
+
+
+@dataclass(frozen=True)
+class AlertDirtyScope:
+    """What the pass has to load, and whether it has to disarm first."""
+
+    disarm_all: bool
+    scope: str
+    roots: frozenset[str]
+    reasons: tuple[str, ...]
+
+    def widens(self, ledger_scope: str) -> bool:
+        """True when the alert axes demand more than the ledger already gives."""
+        return _ORDER[self.scope] > _ORDER[ledger_scope]
+
+
+def _strongest(a: str, b: str) -> str:
+    return a if _ORDER[a] >= _ORDER[b] else b
+
+
+def alert_dirty_scope(
+    *,
+    ledger_groups: Iterable[object],
+    stored_fingerprints: Mapping[tuple, str],
+    resolved_fingerprints: Mapping[tuple, str],
+    gate_before: "bool | None",
+    gate_after: bool,
+    now: dt.datetime,
+    next_evaluation_at: "dt.datetime | None",
+    defer_scheduled: bool = False,
+) -> AlertDirtyScope:
+    """Resolve the five axes into one decision.
+
+    ``stored_fingerprints`` / ``resolved_fingerprints`` are keyed by the arming
+    identity tuple ``(source, source_root_key, account_key, logical_limit_key,
+    observed_slot, window_minutes)``; the ROOT is element 1, which is what an
+    exact-rule change is scoped to.
+
+    ``defer_scheduled`` is the hook path's (``full_pass="defer"``). Axes 2 and 3
+    are driven by a configuration change the user just made, so widening for
+    them is bounded and expected; axis 4 is driven by WALL CLOCK, which makes it
+    the one route into a whole-history pass that can land on a blocking hook
+    tick with nothing to have predicted it. Under this flag it is recorded as
+    ``REASON_SCHEDULED_DEFERRED`` and does NOT strengthen the scope — and the
+    caller owes the stored boundary a carry-through, because a deferral that
+    lets the boundary be recomputed is a silent drop.
+    """
+    reasons: list[str] = []
+    if not gate_after:
+        # Nothing to evaluate: delivery is off, so the only work is making sure
+        # no arming boundary survives to turn disabled-period evidence into a
+        # later alert. It needs no observations at all.
+        return AlertDirtyScope(
+            disarm_all=True, scope=SCOPE_NONE, roots=frozenset(),
+            reasons=("gate_disabled",),
+        )
+
+    scope = SCOPE_NONE
+    roots: set[str] = set()
+
+    if any(True for _ in ledger_groups):
+        scope = _strongest(scope, SCOPE_GROUPS)
+        reasons.append("window_dirty")
+
+    if gate_before is not True:
+        # Enabling — or a first pass with no recorded gate state, which cannot
+        # be distinguished from one and must not be assumed to be a no-op.
+        scope = _strongest(scope, SCOPE_ALL)
+        reasons.append("gate_enabled")
+
+    changed_roots = {
+        identity[1]
+        for identity, fingerprint in resolved_fingerprints.items()
+        if stored_fingerprints.get(identity) != fingerprint
+    }
+    if changed_roots:
+        scope = _strongest(scope, SCOPE_ROOTS)
+        roots |= changed_roots
+        reasons.append("rule_changed")
+
+    if next_evaluation_at is not None and now >= next_evaluation_at:
+        # A future-clocked observation just became eligible. Which identity it
+        # belongs to is not recorded — only the instant — so the honest scope is
+        # everything, and on the hook path "everything" is precisely what may
+        # not run.
+        if defer_scheduled:
+            reasons.append(REASON_SCHEDULED_DEFERRED)
+        else:
+            scope = _strongest(scope, SCOPE_ALL)
+            reasons.append("scheduled")
+
+    return AlertDirtyScope(
+        disarm_all=False, scope=scope, roots=frozenset(roots),
+        reasons=tuple(reasons),
+    )
+
+
+def next_evaluation_boundary(
+    *, capture_times: Iterable[dt.datetime], now: dt.datetime,
+    stored: "dt.datetime | None", retain_due: bool = False,
+) -> "dt.datetime | None":
+    """The earliest still-future capture the projector must come back for.
+
+    A bounded pass only sees the dirty windows, so the STORED boundary is
+    retained whenever it is still in the future: dropping it would forget a
+    future-clocked observation sitting in a window this pass never loaded. Once
+    wall time passes it the axis fires, the pass widens to everything, and the
+    boundary is recomputed from complete evidence — so a retained value can only
+    ever cost one extra pass, never a missed one.
+
+    ``retain_due`` keeps a boundary that is ALREADY due, which is the case where
+    "recomputed from complete evidence" is a lie: a reporting-only pass never
+    reaches a threshold decision at all, and a BOUNDED hook tick that declined
+    the widening deliberately did not look. Either would otherwise retire the
+    axis on behalf of an evaluation nobody performed.
+
+    A due value sorts before every future candidate, so it stays until a pass
+    that genuinely looked at everything retires it — in practice a hook tick
+    that widened to whole-history for axis 2 or 3, since carrying alert
+    eligibility is what separates such a pass from a reporting-only one and the
+    hook is the only production caller that carries it.
+
+    It does NOT stay "until a pass that can act on it does", and that gap is
+    open rather than closed: on a hook-only install with a steady enabled gate,
+    unchanged rules and a quiet ledger, no qualifying pass ever runs and the
+    instant is retained indefinitely. The cost is bounded — the tick stays
+    bounded and fast, and the window is re-evaluated as soon as it goes
+    ledger-dirty again, which for a live window is continuous — so the exposure
+    is a future-clocked capture in a window that then goes permanently quiet
+    never qualifying a threshold. Under-alerting, never a stall or a burst.
+    """
+    candidates = [value for value in capture_times if value > now]
+    if stored is not None and (retain_due or stored > now):
+        candidates.append(stored)
+    return min(candidates) if candidates else None