cctally 1.113.0 → 1.115.0

package/bin/_cctally_five_hour.py

@@ -45,6 +45,7 @@ import sqlite3
 import sys
 
 import _lib_accounts  # pure stdlib kernel; UNATTRIBUTED sentinel (#341)
+from _lib_five_hour import latest_five_hour_reading
 from _cctally_core import _command_as_of, eprint, now_utc_iso, open_db, parse_iso_datetime
 
 
@@ -983,9 +984,10 @@ def _latest_seven_day_and_window(
     overlooked (#769 S11, #824). A `weekly_observation_held` row's five-hour
     fields are the writing tick's own, so it is the freshest statement of which
     window is current; excluding it would resolve an older window and report the
-    live block closed. Rows with no ``five_hour_window_key`` are included here
-    too, unchanged, because the latest row's key is the answer even when it is
-    NULL.
+    live block closed. It is the key of the newest row that CARRIES a five-hour
+    reading (#825/#832 spec 4.7 revision 7, ``latest_five_hour_reading``): a
+    retired replica or a weekly-only reading can be the newest row, and its NULL
+    key would report the live block closed and leave its weekly end stale.
 
     **The weekly value is CREDIT-AWARE**, which is the #835 change. It comes from
     the most recent row that is not a post-floor stale replica — the same band
@@ -1065,8 +1067,15 @@ def _latest_seven_day_and_window(
         row = cur.fetchone()
         if row is None:
             return None, None
-        # Axis 1: the five-hour window key, from the latest row, held-inclusive.
-        key = row["five_hour_window_key"]
+        # Axis 1: the five-hour window key, held-inclusive, from the newest
+        # five-hour READING of the newest row's own account (#825/#832 spec
+        # 4.7 revision 7, the one shared selection). The newest row can be a
+        # retired replica or a weekly-only reading, which names no window.
+        # Merged mode scopes the weekly walk to that account too, so an
+        # account-blind selection here would pair another account's window
+        # with this account's weekly value.
+        reading = latest_five_hour_reading(conn, account_key=row["account_key"])
+        key = None if reading is None else reading.window_key
         # Axis 2: the weekly value. The scope, the bands and the floor all come
         # from the NEWEST row and are resolved ONCE, because the walk may not
         # leave that scope — so there is nothing to memoize per candidate.
@@ -2689,10 +2698,10 @@ def cmd_five_hour_breakdown(args: argparse.Namespace) -> int:
                 ])
                 continue
             m = ev
-            cum = f"${m['blockCostUSD']:.6f}"
+            cum = _c._fmt_usd_accounting(m['blockCostUSD'], 6)
             marg = (
                 "n/a" if m["marginalCostUSD"] is None
-                else f"${m['marginalCostUSD']:.6f}"
+                else _c._fmt_usd_accounting(m['marginalCostUSD'], 6)
             )
             # #834 S1 (#836): the EFFECTIVE weekly value, never the raw one. A
             # weekly-clamped tick stored a reading no reader ever saw, and this
@@ -2718,6 +2727,7 @@ def cmd_five_hour_breakdown(args: argparse.Namespace) -> int:
 
 def _backfill_five_hour_blocks(
     conn: sqlite3.Connection, *, only_missing: bool = False,
+    journaled_first: "dict | None" = None,
 ) -> int:
     """One-shot historical backfill of five_hour_blocks from existing
     weekly_usage_snapshots data. Idempotent via UNIQUE(five_hour_window_key)
@@ -2740,9 +2750,58 @@ def _backfill_five_hour_blocks(
     re-materializes only the open block (block-only, no milestones — the
     journaled 5h-milestone evts carry their own values and resolve block_id
     against this row), keeping the pass O(open windows) instead of O(all blocks).
+
+    #825/#832 fix round J4 (#954): the rebuild projection reproduces the open
+    block live ingest left, on its accounting columns too.
+
+    * ``journaled_first`` maps ``(account_key, five_hour_window_key)`` to the
+      window's FIRST WRITER — its first effective `snapshot_accept` decision
+      in journal order (`_cctally_journal.
+      journaled_block_first_observations`) — and supplies the block's
+      immutable start values (`first_observed_at_utc`,
+      `five_hour_resets_at`, `block_start_at`,
+      `seven_day_pct_at_block_start`) whenever it is present. Live ingest's
+      upsert never updates those columns: the writer that materializes the
+      row owns them. Two things separate that writer from the earliest
+      surviving snapshot. A five-hour credit's stale-replica removal can
+      delete its snapshot (the band starts at floor10 of the arming tick, so
+      it reaches the genuine pre-credit reading); the snapshot still observed
+      the block, because the removal retires its weekly VALUE, not the
+      observation (J4). And a reading captured earlier can be appended, and
+      so derived, later (fix round J5, 825-832-CD4-V01): its row survives
+      with the earlier capture, but it updated a block another reading had
+      already opened. Neither survivor-derived pick reproduces live. A
+      window without an entry (a payload without an explicit account) falls
+      back to the earliest surviving snapshot.
+    * The projection stamps the computation-time pricing provenance of the
+      totals it computes, through the live upsert's own helper
+      (`_block_pricing_provenance_json`), so a successor's
+      `five_hour_block_close` freezes it exactly as after live ingest. The
+      ordinary (legacy, pre-journal) backfill keeps NULL: its blocks predate
+      provenance, which `_lib_cost_provenance.historical_close_marker` reads
+      as unproven.
     """
     _c = _cctally()
     inserted = 0
+    journaled_first = (journaled_first or {}) if only_missing else {}
+
+    def _first_row(key, acct):
+        """The window's start row: its first writer (J4/J5), else the
+        earliest surviving snapshot."""
+        journaled = journaled_first.get((acct, key))
+        if journaled is not None:
+            return journaled
+        return conn.execute(
+            """
+            SELECT five_hour_resets_at, captured_at_utc, weekly_percent
+              FROM weekly_usage_snapshots
+             WHERE five_hour_window_key = ? AND account_key = ?
+               AND five_hour_percent IS NOT NULL
+             ORDER BY captured_at_utc ASC, id ASC LIMIT 1
+            """,
+            (key, acct),
+        ).fetchone()
+
     try:
         # Iterate distinct windows that have BOTH a canonical key AND a
         # 5h percent. The percent guard is critical: MAX(five_hour_percent)
@@ -2812,18 +2871,9 @@ def _backfill_five_hour_blocks(
         for key, acct in keys:
             if (acct, key) in seen_keys:
                 continue
-            # Same MIN-captured pick the insert loop makes below, so the
+            # Same start-row pick the insert loop makes below, so the
             # ownership interval and the inserted row cannot disagree.
-            anchor_row = conn.execute(
-                """
-                SELECT five_hour_resets_at
-                  FROM weekly_usage_snapshots
-                 WHERE five_hour_window_key = ? AND account_key = ?
-                   AND five_hour_percent IS NOT NULL
-                 ORDER BY captured_at_utc ASC, id ASC LIMIT 1
-                """,
-                (key, acct),
-            ).fetchone()
+            anchor_row = _first_row(key, acct)
             if anchor_row is None:
                 continue
             try:
@@ -2849,22 +2899,14 @@ def _backfill_five_hour_blocks(
         conn.execute("BEGIN IMMEDIATE")
         try:
             for key, acct in keys:
-                # MIN-captured row defines the immutable block boundary
-                # values (deterministic — picking "any in-window row"
-                # would be nondeterministic under seconds-level Anthropic
-                # ISO jitter that the canonical key collapses). Scoped to
+                # The start row (the first writer, else the MIN-captured
+                # survivor) defines the immutable block boundary values
+                # (deterministic — picking "any in-window row" would be
+                # nondeterministic under seconds-level Anthropic ISO jitter
+                # that the canonical key collapses). Scoped to
                 # (window_key, account_key) so a shared window's per-account
                 # blocks each read their own account's boundary rows (#341).
-                min_row = conn.execute(
-                    """
-                    SELECT five_hour_resets_at, captured_at_utc, weekly_percent
-                      FROM weekly_usage_snapshots
-                     WHERE five_hour_window_key = ? AND account_key = ?
-                       AND five_hour_percent IS NOT NULL
-                     ORDER BY captured_at_utc ASC, id ASC LIMIT 1
-                    """,
-                    (key, acct),
-                ).fetchone()
+                min_row = _first_row(key, acct)
                 max_row = conn.execute(
                     """
                     SELECT captured_at_utc, weekly_percent, five_hour_percent
@@ -2941,9 +2983,15 @@ def _backfill_five_hour_blocks(
                 # JSONL if cache.db is unreadable.
                 # skip_sync=True: backfill runs inside open_db(); a sync_cache
                 # cascade here would reopen cache.db recursively.
+                pricing_evidence = {} if only_missing else None
                 totals = _c._compute_block_totals(
                     block_start_dt, last_obs_dt, skip_sync=True,
                     owner_key=key, windows=ownership.get(acct, []),
+                    pricing_evidence=pricing_evidence,
+                )
+                pricing_provenance_json = (
+                    _c._block_pricing_provenance_json(pricing_evidence)
+                    if pricing_evidence is not None else None
                 )
 
                 cur = conn.execute(
@@ -2963,12 +3011,14 @@ def _backfill_five_hour_blocks(
                       total_cache_create_tokens,
                       total_cache_read_tokens,
                       total_cost_usd,
+                      pricing_provenance_json,
                       is_closed,
                       created_at_utc,
                       last_updated_at_utc,
                       account_key
                     )
-                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
+                            ?)
                     """,
                     (
                         key,
@@ -2985,6 +3035,7 @@ def _backfill_five_hour_blocks(
                         totals["cache_create_tokens"],
                         totals["cache_read_tokens"],
                         totals["cost_usd"],
+                        pricing_provenance_json,
                         is_closed,
                         projection_created_at,
                         projection_updated_at,

package/bin/_cctally_forecast.py

@@ -97,6 +97,7 @@ from _lib_forecast import (
 # exercises). doctor's _iso_z deliberately keeps its own divergent copy.
 _ensure_sibling_loaded("_lib_json_envelope")
 from _lib_json_envelope import _iso_z
+from _lib_five_hour import latest_five_hour_reading
 
 # #620 S1 D11: the one affordance shape every warning state renders.
 _ensure_sibling_loaded("_lib_alert_scope")
@@ -153,10 +154,30 @@ def _shape_week_samples(rows, *, include_held: bool,
     return out
 
 
+def _week_rows_by_capture(cursor, *, by_id: bool) -> list:
+    """One week's sample rows in capture order: ``captured_at_utc`` ascending,
+    then ``id`` when ``by_id`` (the row's trailing column, which this strips).
+
+    #901 Amendment 19 T1 (spec I1): the reader returns every row of the week
+    anyway, so ordering them here costs no structure beyond its own result,
+    where ``ORDER BY captured_at_utc`` sorted the whole week through a temp
+    b-tree — and one subscription week is not a bound. ``list.sort`` is stable
+    over the same scan SQLite's in-memory sorter consumed, so rows tied on one
+    capture instant keep the order they had.
+    """
+    rows = cursor.fetchall()
+    if by_id:
+        rows.sort(key=lambda row: (row[0], row[-1]))
+        return [row[:-1] for row in rows]
+    rows.sort(key=lambda row: row[0])
+    return rows
+
+
 def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime,
                                   *, account_key: "str | None" = None,
                                   include_held: bool = False,
-                                  include_account: bool = False):
+                                  include_account: bool = False,
+                                  include_scope: bool = False):
     """Return (week_start_at, week_end_at, list[(captured_at, percent, five_hr)])
     for the subscription week containing `now_utc`, or None if no snapshot
     exists for the current week.
@@ -206,6 +227,14 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
     ``include_account`` appends the physical account key after the held flag
     for the multi-account Current Week card only. The default wire-independent
     sample shape and merged forecast/cost reads remain unchanged.
+
+    ``include_scope`` (#825/#832 spec 4.7 revision 7) appends a fourth element,
+    ``(week_start_date, week_start_at_texts)``: the week scope the samples were
+    read under, in the form ``latest_five_hour_reading`` takes
+    (``week_start_at_texts`` is ``None`` on the date-only fallback). A caller
+    reading the five-hour axis selects its reading under exactly this scope
+    rather than from the newest sample, which a retired replica or a
+    weekly-only reading leaves without a five-hour value.
     """
     # #341: optional account scoping — same predicate on every leg so the whole
     # window resolution stays consistent (a real key or the `unattributed`
@@ -264,6 +293,8 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
         we_date = dt.date.fromisoformat(drow[1])
         week_start_at = dt.datetime.combine(ws_date, dt.time(0, 0), local_tz).astimezone(dt.timezone.utc)
         week_end_at = dt.datetime.combine(we_date + dt.timedelta(days=1), dt.time(0, 0), local_tz).astimezone(dt.timezone.utc)
+        # One date: `idx_usage_week_time` (week_start_date, captured_at_utc
+        # DESC, id DESC) serves this order outright, with no sorter.
         rows = conn.execute(
             "SELECT captured_at_utc, weekly_percent, five_hour_percent, "
             "       weekly_observation_held" + _owner_column + " "
@@ -275,6 +306,8 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
         samples = _shape_week_samples(rows, include_held=include_held,
                                       include_account=include_account)
         samples = [s for s in samples if s[0] <= now_utc]
+        if include_scope:
+            return week_start_at, week_end_at, samples, (drow[0], None)
         return week_start_at, week_end_at, samples
     row = chosen
     week_start_at = parse_iso_datetime(row[0], "week_start_at")
@@ -293,19 +326,22 @@ def _fetch_current_week_snapshots(conn: sqlite3.Connection, now_utc: dt.datetime
             matching_texts.append(r[0])
     chosen_date = chosen[2]
     placeholders = ",".join("?" * len(matching_texts))
-    rows = conn.execute(
+    rows = _week_rows_by_capture(conn.execute(
         f"SELECT captured_at_utc, weekly_percent, five_hour_percent, "
-        f"       weekly_observation_held{_owner_column} "
+        f"       weekly_observation_held{_owner_column}"
+        f"{', id' if include_account else ''} "
         f"FROM weekly_usage_snapshots "
         f"WHERE (week_start_at IN ({placeholders}) "
         f"       OR (week_start_at IS NULL AND week_start_date = ?))"
-        f"{_acct_pred}{_held_pred} "
-        f"ORDER BY captured_at_utc{_sample_order}",
+        f"{_acct_pred}{_held_pred}",
         tuple(matching_texts) + (chosen_date,) + _acct_p,
-    ).fetchall()
+    ), by_id=include_account)
     samples = _shape_week_samples(rows, include_held=include_held,
                                   include_account=include_account)
     samples = [s for s in samples if s[0] <= now_utc]
+    if include_scope:
+        return (week_start_at, week_end_at, samples,
+                (chosen_date, tuple(matching_texts)))
     return week_start_at, week_end_at, samples
 
 
@@ -353,6 +389,42 @@ def _apply_midweek_reset_override(
     return week_start_at, samples
 
 
+def _current_week_five_hour_percent(conn, now_utc, week_scope, *,
+                                    account_key, week_start_at,
+                                    week_start_before_override):
+    """The percentage of ``_current_week_five_hour_reading``, or ``None``."""
+    reading = _current_week_five_hour_reading(
+        conn, now_utc, week_scope, account_key=account_key,
+        week_start_at=week_start_at,
+        week_start_before_override=week_start_before_override)
+    return None if reading is None else reading.percent
+
+
+def _current_week_five_hour_reading(conn, now_utc, week_scope, *,
+                                    account_key, week_start_at,
+                                    week_start_before_override):
+    """The current week's five-hour reading: the newest five-hour READING
+    under the current-week read's own scope (#825/#832 spec 4.7 revision 7).
+    Its percent and reset come from one row, so a caller that shows both
+    shows one reading.
+
+    ``week_scope`` is ``_fetch_current_week_snapshots(include_scope=True)``'s
+    fourth element. When ``_apply_midweek_reset_override`` moved the week's
+    start, its new start bounds the captures exactly as it bounded the samples.
+    ``account_key`` ``None`` keeps the merged read's account-blind contract.
+    """
+    week_start_date, week_start_at_texts = week_scope
+    floor = (week_start_at if week_start_at != week_start_before_override
+             else None)
+    reading = latest_five_hour_reading(
+        conn, account_key=account_key,
+        as_of_utc=_iso_z(now_utc),
+        captured_at_or_after_utc=None if floor is None else _iso_z(floor),
+        week_start_date=week_start_date,
+        week_start_at_in=week_start_at_texts)
+    return reading
+
+
 def _resolve_current_budget_window(conn, now_utc, *, account_key=None):
     """Return ``(effective_week_start_dt, week_end_dt)`` for the subscription
     week containing ``now_utc``, honoring a mid-week reset re-anchor; or
@@ -786,8 +858,13 @@ def _dpp_candidate_comparability(regime, week_start_at, week_end_at, *,
 
     - `"comparable"` — in regime, and its own population passes the support
       test.
-    - `"incomparable"` — in regime, population read, support test failed.
-      This is drift, and only this.
+    - `"incomparable"` — in regime, population read, and `apply_regime`
+      rejected it. That is drift (the support test failed), or a week outside
+      the regime's coefficient era (`unvalidated-coefficient-era`, #978 spec
+      4.6), whose units were weighted with another era's coefficients and so
+      cannot be compared with this regime's rate either, or a week with no
+      local population to weigh (`no-local-history`). All three mean the
+      same thing to the selector: the candidate is not comparable.
     - `"excluded"` — on the far side of a rate boundary.
     - `"unreadable"` — the population could not be read at all. Folding this
       into `"excluded"` made a locked `cache.db` look like twelve weeks that
@@ -816,7 +893,8 @@ def _dpp_candidate_comparability(regime, week_start_at, week_end_at, *,
             cache_conn_factory=cache_conn_factory)
     except Exception:                                  # noqa: BLE001
         return "unreadable"
-    applied = qcg.apply_regime(regime, entries)
+    applied = qcg.apply_regime(regime, entries,
+                               interval=(week_start_at, week_end_at))
     if isinstance(applied, qcg.ApplyRejection):
         return "incomparable"
     return "comparable"
@@ -1268,7 +1346,12 @@ def _calibrated_week_detail(
             week_start_at, horizon, account_key=account_key)
     except sqlite3.Error:
         return CalibratedWeek(code="unavailable")
-    applied = qcg.apply_regime(read.regime, entries)
+    # The interval is the whole week the projection extrapolates over, not
+    # the entries' span: a week straddling the regime's coefficient era is
+    # refused as `unvalidated-coefficient-era` even when every retained entry
+    # so far is in-era (#978 spec 4.6).
+    applied = qcg.apply_regime(read.regime, entries,
+                               interval=(week_start_at, week_end_at))
     if isinstance(applied, qcg.ApplyRejection):
         return CalibratedWeek(code=applied.value)
 
@@ -1371,12 +1454,14 @@ def _load_forecast_inputs(
     c = _cctally()
     # #769 S11 (#824): ONE read, held-inclusive, split per axis below.
     fetched = _fetch_current_week_snapshots(
-        conn, now_utc, account_key=account_key, include_held=True)
+        conn, now_utc, account_key=account_key, include_held=True,
+        include_scope=True)
     if fetched is None:
         return None
-    week_start_at, week_end_at, observations = fetched
+    week_start_at, week_end_at, observations, week_scope = fetched
     if not observations:
         return None
+    week_start_before_override = week_start_at
 
     # Mid-week reset override: shift week_start_at to the effective
     # reset moment and drop pre-reset samples so elapsed/remaining math
@@ -1400,8 +1485,16 @@ def _load_forecast_inputs(
     p_now = latest[1]
     # The five-hour axis, which is why the read is held-inclusive: a held row
     # exists precisely because its five-hour reading GREW, so the newest
-    # observation of either kind is the current five-hour percentage.
-    five_hr = observations[-1][2]
+    # READING of either kind is the current five-hour percentage. It is
+    # selected through the one shared selection (#825/#832 spec 4.7 revision
+    # 7) under this read's own scope: the account, the week, `now_utc`, and
+    # the mid-week reset's lower bound when the override applied. The newest
+    # observation is not the answer, because a retired replica or a
+    # weekly-only reading carries no five-hour value.
+    five_hr = _current_week_five_hour_percent(
+        conn, now_utc, week_scope, account_key=account_key,
+        week_start_at=week_start_at,
+        week_start_before_override=week_start_before_override)
 
     elapsed_hours = (now_utc - week_start_at).total_seconds() / 3600.0
     total_hours = (week_end_at - week_start_at).total_seconds() / 3600.0
@@ -2518,7 +2611,7 @@ def cmd_report(args: argparse.Namespace) -> int:
                 m_rows: list[list[str]] = []
                 for idx, m in enumerate(milestone_rows, start=1):
                     pct = f"{int(m['percent_threshold'])}%"
-                    cum = f"${float(m['cumulative_cost_usd']):.6f}"
+                    cum = c._fmt_usd_accounting(float(m['cumulative_cost_usd']), 6)
                     marg = c.observation_gap_marginal_cell(
                         m["marginal_cost_usd"],
                         withheld=(idx - 1) in gap_disclosure.withheld_indexes,