cctally 1.107.0 → 1.108.0

package/bin/_cctally_dashboard_envelope.py

@@ -40,6 +40,7 @@ from __future__ import annotations
 import bisect
 import datetime as dt
 import importlib.util as _ilu
+import json
 import os
 import sqlite3
 import sys
@@ -383,14 +384,23 @@ def _envelope_rows_weekly(
     # collide on the duplicate (week, threshold) pair. Older clients
     # tolerate longer ids — the id is opaque to them; only the React
     # key uniqueness invariant matters.
+    # The LEFT JOIN resolves the segment number into the instant its cycle
+    # began (#750 S3). ``reset_event_id`` is a ``week_reset_events.id``, whose
+    # AUTOINCREMENT never issues 0, so the pre-credit sentinel matches no row
+    # and the mapper falls back to the week's own start below. The join is on
+    # the id alone: the id IS the identity the milestone recorded, and adding
+    # an account predicate could only turn a legitimately-stamped row into a
+    # silent NULL.
     rows = conn.execute(
         f"""
-        SELECT week_start_date, week_start_at, percent_threshold,
-               captured_at_utc, alerted_at, cumulative_cost_usd,
-               reset_event_id, account_key
-        FROM {descriptor.milestone_table}
-        WHERE alerted_at IS NOT NULL
-        ORDER BY {_CANON_ALERTED_AT} DESC
+        SELECT m.week_start_date, m.week_start_at, m.percent_threshold,
+               m.captured_at_utc, m.alerted_at, m.cumulative_cost_usd,
+               m.reset_event_id, m.account_key,
+               e.effective_reset_at_utc
+        FROM {descriptor.milestone_table} m
+        LEFT JOIN week_reset_events e ON e.id = m.reset_event_id
+        WHERE m.alerted_at IS NOT NULL
+        ORDER BY {_CANON_ALERTED_AT_M} DESC
         LIMIT ?
         """,
         (limit,),
@@ -420,6 +430,20 @@ def _envelope_rows_weekly(
                 # key stays on the wire and the reader degrades to day
                 # granularity instead of inventing a clock reading.
                 "week_start_at":       r["week_start_at"] or "",
+                # The start instant of the BILLING CYCLE this crossing
+                # belongs to (#750 S3). A credit ends one cycle and begins
+                # another without moving either week boundary, so a week
+                # credited twice publishes three rows carrying one
+                # `week_start_date` and the week alone identifies none of
+                # them. Segment 0 has no reset event and its cycle begins
+                # with the week. Empty string when the row retains neither
+                # instant, mirroring `week_start_at` above. This is a
+                # SEPARATE field on purpose: `week_start_at` is what both
+                # scope kernels add seven days to, and the week's end does
+                # not move when a cycle inside it does.
+                "cycle_start_at":      (
+                    r["effective_reset_at_utc"] or r["week_start_at"] or ""
+                ),
                 "cumulative_cost_usd": cumulative,
                 "dollars_per_percent": dpp,
                 # Round-3: parallel to the 5h context block below — both
@@ -850,22 +874,53 @@ def _build_meter_rate_change_array(
     SCOPE (the account, decorated under R8 exactly as the alert rows are).
     """
     account_fields = _alert_account_resolver(conn)
+    # #747: the stored `severity` column is unconstrained, so an older writer,
+    # a hand-repaired row or a future kernel can put a token here that no
+    # `RateChangeSeverity` border rule matches; such a row renders on the base
+    # amber border with nothing reporting it. Clamp to `info` through the
+    # kernel's own tuple — the same rule `_cctally_alerts._dispatch_alert_
+    # notification` already applies on the notifier path, applied at the second
+    # site rather than written out a third time.
+    #
+    # Clamped, not asserted: an assertion in the envelope builder would let one
+    # malformed stored row break the entire payload, which is a worse outcome
+    # than rendering that row at `info`.
+    #
+    # An unresolvable kernel degrades to NO clamp, which is the behaviour that
+    # shipped before this change. Degrading to an empty vocabulary instead
+    # would send every row — including every correct one — to `info`.
+    try:
+        _severities = frozenset(
+            sys.modules["cctally"]._load_sibling(
+                "_lib_meter_rate_change").RATE_CHANGE_SEVERITIES)
+    except Exception:                                  # noqa: BLE001
+        _severities = None
     try:
         rows = conn.execute(
             "SELECT provider, account_key, effective_from,"
             "       previous_units_per_point, new_units_per_point, severity,"
-            "       detected_at_utc, created_at_utc"
+            "       detected_at_utc, created_at_utc,"
+            "       withholding_status, detector_input_causes,"
+            "       composition_provenance, baseline_withheld_days"
             "  FROM meter_rate_change_events"
             " ORDER BY unixepoch(effective_from) DESC, id DESC"
             " LIMIT ?", (int(limit),)).fetchall()
     except sqlite3.Error:
-        # A store predating epoch 1011 has no such table, and the rebuild
-        # that creates it is deferred to a background worker. An empty array
-        # renders as "no change recorded", which is the truth on that store.
+        # A store predating epoch 1012 does not have the shape this SELECT
+        # asks for: below 1011 there is no `meter_rate_change_events` table at
+        # all, and at 1011 the table exists but the four #690 disclosure
+        # columns do not. Both raise `sqlite3.Error` and both are answered the
+        # same way, because the rebuild that supplies either is deferred to a
+        # background worker. An empty array renders as "no change recorded",
+        # which is the truth on that store, rather than emptying the envelope.
         return []
     out: list[dict] = []
     for (provider, account_key, effective_from, previous, new, severity,
-         detected_at, created_at) in rows:
+         detected_at, created_at, status, causes, provenance,
+         baseline_days) in rows:
+        severity_token = str(severity or "info")
+        if _severities is not None and severity_token not in _severities:
+            severity_token = "info"
         entry = {
             # Opaque React key. It is never parsed — the same contract the
             # alert `id` carries.
@@ -873,19 +928,55 @@ def _build_meter_rate_change_array(
             "family": "meter_rate_change",
             "provider": str(provider),
             "owner": str(provider),
-            "severity": str(severity or "info"),
+            "severity": severity_token,
             "effective_from": str(effective_from),
             "detected_at": str(detected_at or created_at or ""),
             "recorded_at": str(created_at or ""),
             "previous_units_per_point": (
                 None if previous is None else float(previous)),
             "new_units_per_point": None if new is None else float(new),
+            # #690: the disclosure evidence, published ALWAYS and nulled
+            # rather than dropped. `null`, `[]` and `0` are three distinct
+            # states the client must be able to tell apart: null means legacy
+            # or unrecoverable evidence, `[]` means assessed with no such
+            # origin, and 0 means a clean baseline. The two array columns are
+            # decoded from their canonical JSON rather than forwarded as
+            # strings, so the client receives typed arrays; a column that
+            # will not decode degrades to null, which is the honest "cannot
+            # be read" answer and never an empty measurement.
+            "withholding_status": (
+                None if status is None else str(status)),
+            "detector_input_causes": _decode_evidence_array(causes),
+            "composition_provenance": _decode_evidence_array(provenance),
+            "baseline_withheld_days": (
+                None if baseline_days is None else int(baseline_days)),
         }
         entry.update(account_fields(str(provider), account_key))
         out.append(entry)
     return out
 
 
+def _decode_evidence_array(value) -> "list | None":
+    """One stored evidence column as a typed list, or None (#690).
+
+    The column holds a canonical JSON array of enum VALUES. Decoding here
+    rather than forwarding the string keeps the wire contract typed and keeps
+    the null-versus-empty distinction intact on the client: `null` and `[]`
+    are different answers and a string `"[]"` would be neither.
+
+    Anything that will not decode to a list degrades to None. That is the
+    honest "cannot be read" answer; returning `[]` would publish an empty
+    MEASUREMENT — assessed, no such origin — for a column nobody could read.
+    """
+    if value is None:
+        return None
+    try:
+        decoded = json.loads(value)
+    except (TypeError, ValueError):
+        return None
+    return decoded if isinstance(decoded, list) else None
+
+
 def _build_alerts_envelope_array(
     conn: sqlite3.Connection,
     limit: int = 100,

package/bin/_cctally_diff.py

@@ -118,11 +118,25 @@ def cmd_diff(args: argparse.Namespace) -> int:
     # windows for Claude instead of attempting to reinterpret the original
     # tokens against its subscription-week anchor.  Ordinary Claude invocations
     # keep the established anchor/parser path byte-for-byte.
+    # #341 --account: resolve the render filter (provider=claude; fail closed
+    # with exit 3 when the entry cache is unavailable). None = merged.
+    #
+    # #750 S3 B3: this resolves BEFORE the anchor, not a hundred lines after
+    # both windows are built. `docs/accounts-gotchas.md` states the rule — the
+    # account resolves before the interval is constructed — because an
+    # account-filtered read whose interval came from the merged boundary set
+    # buckets one account's dollars into a window that belongs to neither.
+    acct_key, acct_exit = c.resolve_account_filter(
+        args, "claude", needs_cache=True)
+    if acct_exit is not None:
+        return acct_exit
+
     supplied_windows = getattr(args, "_source_analytics_windows", None)
     if supplied_windows is None:
         # Resolve anchors (None when no snapshots exist; week tokens then
         # raise NoAnchorError in the parser).
-        anchor_week_start, anchor_resets_at = dk._diff_resolve_anchor(now_utc)
+        anchor_week_start, anchor_resets_at = dk._diff_resolve_anchor(
+            now_utc, account_key=acct_key)
     else:
         anchor_week_start, anchor_resets_at = None, None
 
@@ -222,13 +236,6 @@ def cmd_diff(args: argparse.Namespace) -> int:
     if args.min_delta_pct is not None:
         threshold = dataclasses.replace(threshold, min_delta_pct=args.min_delta_pct)
 
-    # #341 --account: resolve the render filter (provider=claude; fail closed
-    # with exit 3 when the entry cache is unavailable). None = merged.
-    acct_key, acct_exit = c.resolve_account_filter(
-        args, "claude", needs_cache=True)
-    if acct_exit is not None:
-        return acct_exit
-
     try:
         result = dk._build_diff_result(
             window_a, window_b,

package/bin/_cctally_doctor.py

@@ -31,6 +31,7 @@ import json
 import math
 import os
 import pathlib
+import re
 import shutil
 import sqlite3
 import subprocess
@@ -588,6 +589,61 @@ def _codex_lifecycle_activity_24h(
     return records
 
 
+_CODEX_ACCOUNT_KEY_PATTERN = re.compile(r"^[0-9a-f]{32}$")
+
+
+def _codex_hook_liveness_markers(*, root_keys: set[str]) -> dict[str, dict]:
+    """Enumerate `<root>.last-success` / `<root>.<account>.last-success` mtimes.
+
+    Marker enumeration lives here, never in `bin/_lib_doctor.py`: the pure
+    kernel decides every check without touching the filesystem, the network
+    or the clock, and this check is not the exception.
+
+    A bare `*` glob would also admit backup-like siblings, so the account
+    suffix is filtered against the 32-hex `account_key` shape rather than
+    globbed. Account suffixes are never exposed to the report.
+    """
+    if not root_keys:
+        return {}
+    base = _cctally_core.APP_DIR / "codex-hook-tick"
+    records: dict[str, dict] = {
+        key: {"last_success_at": None, "marker_count": 0, "unavailable": False}
+        for key in root_keys
+    }
+    try:
+        names = sorted(entry.name for entry in os.scandir(base))
+    except FileNotFoundError:
+        # No marker directory yet is `never`, not `unavailable`: the hook has
+        # simply not run here.
+        return records
+    except OSError:
+        for record in records.values():
+            record["unavailable"] = True
+        return records
+    for name in names:
+        if not name.endswith(".last-success"):
+            continue
+        stem = name[: -len(".last-success")]
+        root_key = stem
+        if stem not in records:
+            head, _sep, suffix = stem.rpartition(".")
+            if not head or not _CODEX_ACCOUNT_KEY_PATTERN.match(suffix):
+                continue
+            root_key = head
+            if root_key not in records:
+                continue
+        try:
+            stamp = (base / name).stat().st_mtime
+        except OSError:
+            continue
+        record = records[root_key]
+        record["marker_count"] += 1
+        observed = dt.datetime.fromtimestamp(stamp, dt.timezone.utc)
+        if record["last_success_at"] is None or observed > record["last_success_at"]:
+            record["last_success_at"] = observed
+    return records
+
+
 def _codex_quota_verify_activity_24h(*, now_utc: "dt.datetime") -> dict:
     """Aggregate the detached `_codex-quota-verify` worker's 24h outcomes.
 
@@ -1801,18 +1857,29 @@ def _doctor_gather_state_impl(
 
     with _lib_perf.phase("doctor.codex_hooks"):
         codex_hook_roots: list[dict] = []
+        codex_hook_liveness: dict = {}
         try:
-            codex_binary = str(c._setup_resolve_hook_target(repo_root))
             hook_rows = [
-                c._cctally_setup._codex_hook_row(root, codex_binary)
+                c._cctally_setup._codex_hook_row(root)
                 for root in c._setup_codex_hook_roots()
             ]
             codex_hook_roots = [
-                {"source_root_key": row["source_root_key"], "state": row["state"]}
+                {
+                    "source_root_key": row["source_root_key"],
+                    "state": row["state"],
+                    "requires_review": row["requires_review"],
+                    "remediation": row["remediation"],
+                }
                 for row in sorted(hook_rows, key=lambda row: row["source_root_key"])
             ]
         except Exception:
             codex_hook_roots = []
+        try:
+            codex_hook_liveness = _codex_hook_liveness_markers(
+                root_keys={row["source_root_key"] for row in codex_hook_roots},
+            )
+        except Exception:
+            codex_hook_liveness = {}
 
     with _lib_perf.phase("doctor.codex_lifecycle"):
         try:
@@ -2601,6 +2668,7 @@ def _doctor_gather_state_impl(
         conversations_db_freelist_count=conversations_db_freelist_count,
         codex_quota_windows=codex_quota_windows,
         codex_hook_roots=codex_hook_roots,
+        codex_hook_liveness=codex_hook_liveness,
         codex_lifecycle_activity_24h=codex_lifecycle_activity_24h,
         codex_quota_verify_activity=codex_quota_verify_activity,
         # #311: precomputed statusLine.refreshInterval classification.

package/bin/_cctally_forecast.py

@@ -2134,13 +2134,22 @@ def cmd_report(args: argparse.Namespace) -> int:
         # effective reset moment). Route `current_ref` through the same
         # override so its `week_start_at` reflects the post-credit start;
         # this lets the per-row match below disambiguate the synthesized
-        # pre-credit ref from the live post-credit ref via both
-        # `key` AND `week_start_at`. Order contract from
-        # `_apply_reset_events_to_weekrefs`: post-credit ref lands at
-        # index 0, pre-credit at index 1. Non-credit weeks return the
+        # earlier refs from the live one via both `key` AND `week_start_at`.
+        # Order contract from `_apply_reset_events_to_weekrefs`: the output is
+        # NEWEST-FIRST, so the live segment is at index 0 and every earlier
+        # segment follows it in descending order. #750 S3 made the applier
+        # N-ary, so index 1 is the segment before the live one rather than
+        # "the pre-credit ref" — a week credited twice puts a middle segment
+        # there. Only index 0 is read here. Non-credit weeks return the
         # single input ref unchanged, so this is a no-op on the common
         # path.
-        _adjusted_current = c._apply_reset_events_to_weekrefs(conn, [current_ref])
+        # SCOPED to the requesting account (#750 S3, Unit B review). The match
+        # below disambiguates the live segment by `week_start_at`, so a merged
+        # read lets ANOTHER account's cut shift that value and the current-week
+        # row is then misidentified. `acct_key` is None without `--account`,
+        # which is the explicit merged read this call always made.
+        _adjusted_current = c._apply_reset_events_to_weekrefs(
+            conn, [current_ref], account_key=acct_key)
         if _adjusted_current:
             current_ref = _adjusted_current[0]
 
@@ -2396,12 +2405,28 @@ def cmd_report(args: argparse.Namespace) -> int:
             if milestone_rows:
                 print()
                 print("Percent breakdown (current week):\n")
+                # #750 S3 Unit C (issue #738). This table is a SECOND surface
+                # over the same rows, with its own column set and its own bare
+                # `n/a`, so it had the same defect and shipping the disclosure
+                # on only one of the two would leave the issue half fixed. The
+                # classifier is C1's, reached on the namespace — the predicate
+                # has one home. It groups by `(account_key, reset_event_id)`
+                # itself, which matters here because this table is NOT
+                # segment-filtered and renders every epoch of the week.
+                gap_disclosure = c.classify_observation_gaps(milestone_rows)
+                for note in c.observation_gap_notes(gap_disclosure, tz=tz):
+                    print(note)
+                if gap_disclosure.runs:
+                    print()
                 m_headers = ["#", "Threshold", "Cumulative Cost", "Marginal Cost"]
                 m_rows: list[list[str]] = []
                 for idx, m in enumerate(milestone_rows, start=1):
                     pct = f"{int(m['percent_threshold'])}%"
                     cum = f"${float(m['cumulative_cost_usd']):.6f}"
-                    marg = f"${float(m['marginal_cost_usd']):.6f}" if m["marginal_cost_usd"] is not None else "n/a"
+                    marg = c.observation_gap_marginal_cell(
+                        m["marginal_cost_usd"],
+                        withheld=(idx - 1) in gap_disclosure.withheld_indexes,
+                    )
                     m_rows.append([str(idx), pct, cum, marg])
                 print(c._boxed_table(m_headers, m_rows, ["right", "right", "right", "right"]))