cctally 1.116.0 → 1.117.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/CHANGELOG.md +36 -0
  2. package/README.md +4 -6
  3. package/bin/_cctally_account.py +57 -3
  4. package/bin/_cctally_cache.py +5720 -885
  5. package/bin/_cctally_claude_attribution.py +997 -0
  6. package/bin/_cctally_completion.py +101 -28
  7. package/bin/_cctally_core.py +235 -12
  8. package/bin/_cctally_dashboard.py +224 -7
  9. package/bin/_cctally_dashboard_conversation.py +107 -21
  10. package/bin/_cctally_dashboard_envelope.py +34 -0
  11. package/bin/_cctally_db.py +928 -80
  12. package/bin/_cctally_diagnosis_sources.py +78 -13
  13. package/bin/_cctally_doctor.py +211 -86
  14. package/bin/_cctally_five_hour.py +389 -18
  15. package/bin/_cctally_forecast.py +30 -6
  16. package/bin/_cctally_journal.py +2418 -158
  17. package/bin/_cctally_oauth_samples.py +185 -7
  18. package/bin/_cctally_parser.py +38 -0
  19. package/bin/_cctally_project.py +34 -8
  20. package/bin/_cctally_quarantine.py +821 -343
  21. package/bin/_cctally_quota.py +69 -4
  22. package/bin/_cctally_quota_calibration.py +677 -3
  23. package/bin/_cctally_quota_model.py +927 -127
  24. package/bin/_cctally_record.py +922 -445
  25. package/bin/_cctally_rederive.py +119 -15
  26. package/bin/_cctally_statusline.py +1048 -176
  27. package/bin/_cctally_store.py +544 -110
  28. package/bin/_cctally_tui.py +93 -9
  29. package/bin/_cctally_update.py +77 -14
  30. package/bin/_lib_account_protocol.py +227 -19
  31. package/bin/_lib_claude_attribution.py +421 -0
  32. package/bin/_lib_clone_copy.py +30 -0
  33. package/bin/_lib_codex_conversation_query.py +45 -9
  34. package/bin/_lib_codex_conversation_watch.py +72 -13
  35. package/bin/_lib_codex_rollout_container.py +234 -19
  36. package/bin/_lib_codex_worker_lifecycle.py +374 -0
  37. package/bin/_lib_credit.py +22 -13
  38. package/bin/_lib_dashboard_sources.py +163 -10
  39. package/bin/_lib_detector_access_registry.py +1335 -166
  40. package/bin/_lib_doctor.py +276 -92
  41. package/bin/_lib_ingest_frontier.py +269 -36
  42. package/bin/_lib_journal.py +273 -22
  43. package/bin/_lib_jsonl.py +12 -0
  44. package/bin/_lib_meter_rate_change.py +72 -0
  45. package/bin/_lib_outcome_quarantine.py +917 -554
  46. package/bin/_lib_provider_activity.py +24 -6
  47. package/bin/_lib_quota_calibration.py +57 -3
  48. package/bin/_lib_quota_copy.py +7 -2
  49. package/bin/_lib_quota_lineage.py +613 -0
  50. package/bin/_lib_quota_model.py +435 -22
  51. package/bin/_lib_rate_change_delivery.py +16 -0
  52. package/bin/_lib_reclaim_planner.py +6 -4
  53. package/bin/_lib_rederive.py +2046 -21
  54. package/bin/_lib_selector_state.py +131 -3
  55. package/bin/_lib_stale_proof.py +113 -4
  56. package/bin/_lib_statusline.py +72 -3
  57. package/bin/_lib_statusline_candidates.py +246 -42
  58. package/bin/_lib_write_io.py +418 -5
  59. package/bin/cctally +24 -11
  60. package/dashboard/static/assets/{ConversationsView-Dflb_myA.js → ConversationsView-Dcfn8T6f.js} +1 -1
  61. package/dashboard/static/assets/{DoctorModal-DFX5CfIs.js → DoctorModal-BqmXbnhn.js} +1 -1
  62. package/dashboard/static/assets/{ModalRoot-HlxoagIa.js → ModalRoot-Ck3WJK2V.js} +1 -1
  63. package/dashboard/static/assets/{ProjectsDrillPanel-C3PJy1xx.js → ProjectsDrillPanel-B-HZkDnT.js} +1 -1
  64. package/dashboard/static/assets/{SourceDetailModal-BczsWKhD.js → SourceDetailModal-QhJkHB7i.js} +1 -1
  65. package/dashboard/static/assets/{UpdateModal-Os-Lo5cs.js → UpdateModal-CkXEIQcK.js} +1 -1
  66. package/dashboard/static/assets/index-D34eYhAc.css +1 -0
  67. package/dashboard/static/assets/{index-CYZ3NLqF.js → index-HKS1s56o.js} +3 -3
  68. package/dashboard/static/assets/{outlineNavigation-k-zIif6W.js → outlineNavigation-BNGaNPCp.js} +1 -1
  69. package/dashboard/static/assets/{useKeymap-CW7Up0Ay.js → useKeymap-CUgEnnua.js} +1 -1
  70. package/dashboard/static/dashboard.html +3 -3
  71. package/package.json +5 -1
  72. package/dashboard/static/assets/index-BKITk1CT.css +0 -1
@@ -42,6 +42,7 @@ import datetime as dt
42
42
  import json
43
43
  import sqlite3
44
44
  import sys
45
+ from dataclasses import dataclass
45
46
 
46
47
  import _lib_accounts # pure stdlib kernel; UNATTRIBUTED sentinel (#341)
47
48
  import _lib_rebuild_progress # #997 D9 progress ticks (stdlib only)
@@ -1779,9 +1780,359 @@ def cmd_five_hour_breakdown(args: argparse.Namespace) -> int:
1779
1780
  conn.close()
1780
1781
 
1781
1782
 
1783
+ @dataclass(frozen=True)
1784
+ class OpenBlockEnd:
1785
+ """The reading an open five-hour block ends at (#956 class 1, spec
1786
+ 4.5.1): its capture, five-hour percent and weekly percent, as stored and
1787
+ fold-effective. ``origin`` names where it came from: ``"snapshot"`` is the
1788
+ window's latest surviving five-hour reading, ``"opening"`` the opening
1789
+ reading of the window's retained opening fact (`five_hour_block_openings`,
1790
+ #956 class 2, spec 4.5.4) when no reading survives."""
1791
+
1792
+ captured_at_utc: str
1793
+ five_hour_percent: float
1794
+ seven_day_percent: "float | None"
1795
+ origin: str
1796
+
1797
+
1798
+ def select_open_block_end(
1799
+ conn: sqlite3.Connection, *, account_key: str, five_hour_window_key: int,
1800
+ ) -> "OpenBlockEnd | None":
1801
+ """The end reading of one open block, read only (#956 class 1): the
1802
+ window's latest surviving five-hour reading in CAPTURE order, held rows
1803
+ included, ties broken by logical identity (`journal_id`, a NULL one
1804
+ lowest), never by journal order or rowid.
1805
+
1806
+ With no surviving reading (a correction removed every one, or `record-
1807
+ credit --force` removed the synthetics that were the window's only
1808
+ readings) it is the window's opening fact's OPENING reading (#956 spec
1809
+ 4.5.4): its capture, five-hour percent and weekly start value, retained
1810
+ and immutable, so live, every rebuild and rederive agree. ``None`` only
1811
+ for a legacy window with neither (residual R4); the caller then leaves
1812
+ the end columns as they are.
1813
+
1814
+ Live ingest, derivation completion and the rebuild's Phase 2a all read
1815
+ the end here, so an earlier capture appended later, a credit that
1816
+ suppresses the selected reading and a `record-credit` synthetic move
1817
+ every path identically. `latest_five_hour_reading` (the week-scoped
1818
+ selection forecast, statusline and the dashboard read) is deliberately
1819
+ a different function and is not changed."""
1820
+ # The order `unixepoch(captured_at_utc) DESC, journal_id DESC` (NULLs
1821
+ # lowest on both keys, as SQLite sorts them under DESC) is applied here
1822
+ # over the window's rows rather than by an ORDER BY: this runs on every
1823
+ # live tick, and the window-key index cannot serve an expression order,
1824
+ # so SQL would sort through a temp B-tree each time (#902's recurring-
1825
+ # plan guard). One five-hour window holds a bounded number of readings.
1826
+ rows = conn.execute(
1827
+ """
1828
+ SELECT unixepoch(captured_at_utc) AS captured_epoch, journal_id,
1829
+ captured_at_utc, five_hour_percent, weekly_percent
1830
+ FROM weekly_usage_snapshots
1831
+ WHERE account_key = ? AND five_hour_window_key = ?
1832
+ AND five_hour_percent IS NOT NULL
1833
+ """,
1834
+ (account_key, int(five_hour_window_key)),
1835
+ ).fetchall()
1836
+ if not rows:
1837
+ seed = conn.execute(
1838
+ "SELECT first_observed_at_utc, five_hour_percent, "
1839
+ " seven_day_pct_at_block_start "
1840
+ " FROM five_hour_block_openings "
1841
+ " WHERE account_key = ? AND five_hour_window_key = ?",
1842
+ (account_key, int(five_hour_window_key)),
1843
+ ).fetchone()
1844
+ if seed is None:
1845
+ return None
1846
+ return OpenBlockEnd(
1847
+ captured_at_utc=seed[0],
1848
+ five_hour_percent=float(seed[1]),
1849
+ seven_day_percent=None if seed[2] is None else float(seed[2]),
1850
+ origin="opening",
1851
+ )
1852
+ row = max(rows, key=lambda r: (
1853
+ r["captured_epoch"] is not None, r["captured_epoch"] or 0,
1854
+ r["journal_id"] is not None, r["journal_id"] or "",
1855
+ ))
1856
+ return OpenBlockEnd(
1857
+ captured_at_utc=row["captured_at_utc"],
1858
+ five_hour_percent=float(row["five_hour_percent"]),
1859
+ seven_day_percent=(
1860
+ None if row["weekly_percent"] is None
1861
+ else float(row["weekly_percent"])
1862
+ ),
1863
+ origin="snapshot",
1864
+ )
1865
+
1866
+
1867
+ def project_open_block_end(
1868
+ conn: sqlite3.Connection, block, end: "OpenBlockEnd | None", *,
1869
+ ownership, journal_ctx, as_of: str,
1870
+ ) -> None:
1871
+ """Project one OPEN block's end columns and accounting from ``end``
1872
+ (#956 class 1, spec 4.5.1), within the caller's transaction.
1873
+
1874
+ It sets `last_observed_at_utc`, `final_five_hour_percent` and
1875
+ `seven_day_pct_at_block_end` from ``end`` (left as they are when ``end``
1876
+ is None), and recomputes the four token totals, `total_cost_usd`, both
1877
+ child sets, `pricing_provenance_json` and `crossed_seven_day_reset` to
1878
+ the end's capture, the INCLUSIVE observation cutoff. Ownership is
1879
+ `_lib_blocks.resolve_owning_window` over ``ownership`` (OwnedWindow
1880
+ values of the account's block windows, the block's own included), with
1881
+ the half-open ``[start, reset)`` interval. With rederive's
1882
+ `block_source_evidence` sink present it records the owned entries behind
1883
+ these totals. It never touches start columns, never refreshes a closed
1884
+ block (a journaled fact), and stamps `last_updated_at_utc = as_of`.
1885
+
1886
+ Cache inputs are read with ``skip_sync=True``: this runs inside a stats
1887
+ transaction (and inside `open_db()` on the rebuild path), and the live
1888
+ tick's own totals computation already synced the cache."""
1889
+ _c = _cctally()
1890
+ current = conn.execute(
1891
+ "SELECT * FROM five_hour_blocks WHERE id = ?", (int(block["id"]),),
1892
+ ).fetchone()
1893
+ if current is None or int(current["is_closed"]) != 0:
1894
+ return
1895
+ account_key = current["account_key"]
1896
+ window_key = int(current["five_hour_window_key"])
1897
+ block_start_dt = parse_iso_datetime(
1898
+ current["block_start_at"], "five_hour_blocks.block_start_at",
1899
+ )
1900
+ if end is None:
1901
+ last_observed = current["last_observed_at_utc"]
1902
+ final_five_hour = current["final_five_hour_percent"]
1903
+ weekly_end = current["seven_day_pct_at_block_end"]
1904
+ else:
1905
+ last_observed = end.captured_at_utc
1906
+ final_five_hour = float(end.five_hour_percent)
1907
+ weekly_end = end.seven_day_percent
1908
+ cutoff_dt = parse_iso_datetime(
1909
+ last_observed, "five_hour_blocks.last_observed_at_utc",
1910
+ )
1911
+ windows = list(ownership)
1912
+ if not any(w.key == window_key for w in windows):
1913
+ windows.append(_c.OwnedWindow(
1914
+ key=window_key,
1915
+ start=block_start_dt.astimezone(dt.timezone.utc),
1916
+ reset=parse_iso_datetime(
1917
+ current["five_hour_resets_at"],
1918
+ "five_hour_blocks.five_hour_resets_at",
1919
+ ).astimezone(dt.timezone.utc),
1920
+ ))
1921
+ pricing_evidence: dict = {}
1922
+ source_sink = getattr(journal_ctx, "block_source_evidence", None)
1923
+ source_key = (account_key, window_key)
1924
+ owned_entries = (
1925
+ [] if source_sink is not None and source_key in source_sink else None
1926
+ )
1927
+ totals = _c._compute_block_totals(
1928
+ block_start_dt, cutoff_dt, skip_sync=True,
1929
+ owner_key=window_key, windows=windows,
1930
+ pricing_evidence=pricing_evidence, owned_entries=owned_entries,
1931
+ )
1932
+ if owned_entries is not None:
1933
+ source_sink[source_key] = tuple(owned_entries)
1934
+ # The reset-crossing flag over [block_start_at, end], from the same two
1935
+ # sources and predicates as the live sweep
1936
+ # (`_flag_blocks_spanning_week_resets`), computed exactly so an end that
1937
+ # moves backward clears it too.
1938
+ crossed = conn.execute(
1939
+ """
1940
+ SELECT
1941
+ EXISTS (
1942
+ SELECT 1 FROM week_reset_events
1943
+ WHERE account_key = ?
1944
+ AND unixepoch(effective_reset_at_utc)
1945
+ BETWEEN unixepoch(?) AND unixepoch(?)
1946
+ )
1947
+ OR EXISTS (
1948
+ SELECT 1 FROM weekly_usage_snapshots
1949
+ WHERE week_start_at IS NOT NULL
1950
+ AND account_key = ?
1951
+ AND unixepoch(week_start_at) > unixepoch(?)
1952
+ AND unixepoch(week_start_at) <= unixepoch(?)
1953
+ )
1954
+ """,
1955
+ (account_key, current["block_start_at"], last_observed,
1956
+ account_key, current["block_start_at"], last_observed),
1957
+ ).fetchone()[0]
1958
+ conn.execute(
1959
+ """
1960
+ UPDATE five_hour_blocks
1961
+ SET last_observed_at_utc = ?,
1962
+ final_five_hour_percent = ?,
1963
+ seven_day_pct_at_block_end = ?,
1964
+ total_input_tokens = ?,
1965
+ total_output_tokens = ?,
1966
+ total_cache_create_tokens = ?,
1967
+ total_cache_read_tokens = ?,
1968
+ total_cost_usd = ?,
1969
+ pricing_provenance_json = ?,
1970
+ crossed_seven_day_reset = ?,
1971
+ last_updated_at_utc = ?
1972
+ WHERE id = ? AND is_closed = 0
1973
+ """,
1974
+ (
1975
+ last_observed, final_five_hour, weekly_end,
1976
+ totals["input_tokens"], totals["output_tokens"],
1977
+ totals["cache_create_tokens"], totals["cache_read_tokens"],
1978
+ totals["cost_usd"],
1979
+ _c._block_pricing_provenance_json(pricing_evidence),
1980
+ 1 if crossed else 0, as_of, int(current["id"]),
1981
+ ),
1982
+ )
1983
+ # Replace-all children, keyed on the window (not block_id) so an orphan
1984
+ # child of an earlier parent rebuild is cleaned up as well.
1985
+ block_id = int(current["id"])
1986
+ conn.execute(
1987
+ "DELETE FROM five_hour_block_models "
1988
+ "WHERE five_hour_window_key = ? AND account_key = ?",
1989
+ (window_key, account_key),
1990
+ )
1991
+ if totals.get("by_model"):
1992
+ conn.executemany(
1993
+ """
1994
+ INSERT INTO five_hour_block_models (
1995
+ block_id, five_hour_window_key, model,
1996
+ input_tokens, output_tokens,
1997
+ cache_create_tokens, cache_read_tokens,
1998
+ cost_usd, entry_count, account_key
1999
+ )
2000
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
2001
+ """,
2002
+ [
2003
+ (
2004
+ block_id, window_key, model,
2005
+ b["input_tokens"], b["output_tokens"],
2006
+ b["cache_create_tokens"], b["cache_read_tokens"],
2007
+ b["cost_usd"], b["entry_count"], account_key,
2008
+ )
2009
+ for model, b in totals["by_model"].items()
2010
+ ],
2011
+ )
2012
+ conn.execute(
2013
+ "DELETE FROM five_hour_block_projects "
2014
+ "WHERE five_hour_window_key = ? AND account_key = ?",
2015
+ (window_key, account_key),
2016
+ )
2017
+ if totals.get("by_project"):
2018
+ conn.executemany(
2019
+ """
2020
+ INSERT INTO five_hour_block_projects (
2021
+ block_id, five_hour_window_key, project_path,
2022
+ input_tokens, output_tokens,
2023
+ cache_create_tokens, cache_read_tokens,
2024
+ cost_usd, entry_count, account_key
2025
+ )
2026
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
2027
+ """,
2028
+ [
2029
+ (
2030
+ block_id, window_key, project_path,
2031
+ b["input_tokens"], b["output_tokens"],
2032
+ b["cache_create_tokens"], b["cache_read_tokens"],
2033
+ b["cost_usd"], b["entry_count"], account_key,
2034
+ )
2035
+ for project_path, b in totals["by_project"].items()
2036
+ ],
2037
+ )
2038
+
2039
+
2040
+ def _materialize_open_candidate(conn, key, acct, start_row) -> int:
2041
+ """Insert one rebuild open-block candidate with its start columns from
2042
+ ``start_row`` (the first writer, else the earliest survivor) and its end
2043
+ from `select_open_block_end`; zero totals and NULL provenance until
2044
+ `_refresh_every_open_block` projects it in the same transaction. Returns
2045
+ the inserted row count."""
2046
+ end = select_open_block_end(
2047
+ conn, account_key=acct, five_hour_window_key=key)
2048
+ if start_row is None or end is None:
2049
+ return 0 # defensive — should be unreachable per the keys query
2050
+ resets_at = start_row["five_hour_resets_at"]
2051
+ block_start_at = (
2052
+ parse_iso_datetime(resets_at, "resets_at backfill")
2053
+ - dt.timedelta(hours=5)
2054
+ ).isoformat(timespec="seconds")
2055
+ # A rebuild's only-missing row is the trailing unjournaled projection,
2056
+ # never a close decision. Wall time may be past its reset, but only a
2057
+ # later retained observation may close and clock it (#399).
2058
+ cur = conn.execute(
2059
+ """
2060
+ INSERT OR IGNORE INTO five_hour_blocks (
2061
+ five_hour_window_key, five_hour_resets_at, block_start_at,
2062
+ first_observed_at_utc, last_observed_at_utc,
2063
+ final_five_hour_percent,
2064
+ seven_day_pct_at_block_start, seven_day_pct_at_block_end,
2065
+ crossed_seven_day_reset,
2066
+ total_input_tokens, total_output_tokens,
2067
+ total_cache_create_tokens, total_cache_read_tokens,
2068
+ total_cost_usd, pricing_provenance_json, is_closed,
2069
+ created_at_utc, last_updated_at_utc, account_key
2070
+ )
2071
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0, NULL, 0, ?, ?, ?)
2072
+ """,
2073
+ (
2074
+ key, resets_at, block_start_at,
2075
+ start_row["captured_at_utc"], end.captured_at_utc,
2076
+ end.five_hour_percent,
2077
+ start_row["weekly_percent"], end.seven_day_percent,
2078
+ start_row["captured_at_utc"], end.captured_at_utc, acct,
2079
+ ),
2080
+ )
2081
+ return cur.rowcount or 0
2082
+
2083
+
2084
+ def _refresh_every_open_block(conn) -> int:
2085
+ """Project every open block through `select_open_block_end` /
2086
+ `project_open_block_end` (#956 class 1, spec 4.5.3), with each account's
2087
+ ownership context built from every `five_hour_blocks` row as it stands
2088
+ now. Runs inside the caller's transaction. Returns the blocks refreshed.
2089
+ A block whose window has no surviving reading keeps its end columns, and
2090
+ its accounting is recomputed to its stored `last_observed_at_utc`."""
2091
+ _c = _cctally()
2092
+ ownership: dict = {}
2093
+ for row in conn.execute(
2094
+ "SELECT five_hour_window_key, block_start_at, "
2095
+ " five_hour_resets_at, account_key FROM five_hour_blocks"
2096
+ ).fetchall():
2097
+ try:
2098
+ w_start = parse_iso_datetime(
2099
+ row["block_start_at"], "five_hour_blocks.block_start_at",
2100
+ ).astimezone(dt.timezone.utc)
2101
+ w_reset = parse_iso_datetime(
2102
+ row["five_hour_resets_at"],
2103
+ "five_hour_blocks.five_hour_resets_at",
2104
+ ).astimezone(dt.timezone.utc)
2105
+ except ValueError:
2106
+ continue
2107
+ ownership.setdefault(row["account_key"], []).append(_c.OwnedWindow(
2108
+ key=int(row["five_hour_window_key"]), start=w_start,
2109
+ reset=w_reset,
2110
+ ))
2111
+ refreshed = 0
2112
+ for index, block in enumerate(conn.execute(
2113
+ "SELECT id, account_key, five_hour_window_key, last_updated_at_utc "
2114
+ " FROM five_hour_blocks WHERE is_closed = 0"
2115
+ ).fetchall(), 1):
2116
+ if not index & 1023:
2117
+ _lib_rebuild_progress.tick(blocks=index)
2118
+ end = select_open_block_end(
2119
+ conn, account_key=block["account_key"],
2120
+ five_hour_window_key=int(block["five_hour_window_key"]))
2121
+ project_open_block_end(
2122
+ conn, block, end,
2123
+ ownership=ownership.get(block["account_key"], []),
2124
+ journal_ctx=None,
2125
+ as_of=(end.captured_at_utc if end is not None
2126
+ else block["last_updated_at_utc"]),
2127
+ )
2128
+ refreshed += 1
2129
+ return refreshed
2130
+
2131
+
1782
2132
  def _backfill_five_hour_blocks(
1783
2133
  conn: sqlite3.Connection, *, only_missing: bool = False,
1784
2134
  journaled_first: "dict | None" = None,
2135
+ retired_openings: frozenset = frozenset(),
1785
2136
  ) -> int:
1786
2137
  """One-shot historical backfill of five_hour_blocks from existing
1787
2138
  weekly_usage_snapshots data. Idempotent via UNIQUE(five_hour_window_key)
@@ -1834,6 +2185,26 @@ def _backfill_five_hour_blocks(
1834
2185
  ordinary (legacy, pre-journal) backfill keeps NULL: its blocks predate
1835
2186
  provenance, which `_lib_cost_provenance.historical_close_marker` reads
1836
2187
  as unproven.
2188
+
2189
+ #956 class 1 (spec 4.5.3): with ``only_missing`` the end columns and the
2190
+ accounting come from the one capture-order projection live ingest uses
2191
+ (`select_open_block_end` / `project_open_block_end`), not from a
2192
+ backfill-local end row. The pass materializes the missing candidates
2193
+ (start columns only, as above), then builds each account's ownership
2194
+ context from EVERY block row after that materialization, and refreshes
2195
+ EVERY open block, not only those it created: a block another path
2196
+ materialized is refreshed against the same window set. The candidate
2197
+ ranking breaks equal-second ties by `journal_id` (logical identity), never
2198
+ by rowid, which a journal correction can reorder.
2199
+
2200
+ #956 class 2 (spec 4.6.3): a window created by a current binary has a
2201
+ retained opening fact whose fold already materialized its block, so it
2202
+ is never a candidate here and its start columns are the seed's. The
2203
+ candidates are the LEGACY open blocks (created before opening facts, R4):
2204
+ each account's latest-captured window with no block row, except a window
2205
+ in ``retired_openings`` — the `(account_key, five_hour_window_key)` pairs
2206
+ whose opening fact a correction tombstoned (`_cctally_journal.
2207
+ retired_block_openings`): its block was removed, never "never seeded".
1837
2208
  """
1838
2209
  _c = _cctally()
1839
2210
  inserted = 0
@@ -1884,7 +2255,7 @@ def _backfill_five_hour_blocks(
1884
2255
  " SELECT id, ROW_NUMBER() OVER ("
1885
2256
  " PARTITION BY account_key"
1886
2257
  " ORDER BY unixepoch(captured_at_utc) DESC,"
1887
- " id DESC"
2258
+ " journal_id DESC"
1888
2259
  " ) AS rn"
1889
2260
  " FROM weekly_usage_snapshots"
1890
2261
  " WHERE five_hour_window_key IS NOT NULL"
@@ -1899,6 +2270,9 @@ def _backfill_five_hour_blocks(
1899
2270
  " )"
1900
2271
  )
1901
2272
  keys = [(int(r[0]), r[1]) for r in conn.execute(keys_sql).fetchall()]
2273
+ if only_missing and retired_openings:
2274
+ keys = [(key, acct) for key, acct in keys
2275
+ if (acct, key) not in retired_openings]
1902
2276
 
1903
2277
  now_iso = now_utc_iso()
1904
2278
  now_dt = parse_iso_datetime(now_iso, "now")
@@ -1911,10 +2285,10 @@ def _backfill_five_hour_blocks(
1911
2285
  # pair re-prices its overlap twice.
1912
2286
  ownership: dict[Any, list] = {}
1913
2287
  seen_keys: set = set()
1914
- for row in conn.execute(
2288
+ for row in ([] if only_missing else conn.execute(
1915
2289
  "SELECT five_hour_window_key, block_start_at, "
1916
2290
  " five_hour_resets_at, account_key FROM five_hour_blocks"
1917
- ).fetchall():
2291
+ ).fetchall()):
1918
2292
  try:
1919
2293
  w_start = parse_iso_datetime(
1920
2294
  row["block_start_at"], "five_hour_blocks.block_start_at",
@@ -1930,7 +2304,7 @@ def _backfill_five_hour_blocks(
1930
2304
  ownership.setdefault(row["account_key"], []).append(
1931
2305
  _c.OwnedWindow(key=w_key, start=w_start, reset=w_reset)
1932
2306
  )
1933
- for key, acct in keys:
2307
+ for key, acct in ([] if only_missing else keys):
1934
2308
  if (acct, key) in seen_keys:
1935
2309
  continue
1936
2310
  # Same start-row pick the insert loop makes below, so the
@@ -1973,6 +2347,10 @@ def _backfill_five_hour_blocks(
1973
2347
  # (window_key, account_key) so a shared window's per-account
1974
2348
  # blocks each read their own account's boundary rows (#341).
1975
2349
  min_row = _first_row(key, acct)
2350
+ if only_missing:
2351
+ inserted += _materialize_open_candidate(
2352
+ conn, key, acct, min_row)
2353
+ continue
1976
2354
  max_row = conn.execute(
1977
2355
  """
1978
2356
  SELECT captured_at_utc, weekly_percent, five_hour_percent
@@ -2038,26 +2416,16 @@ def _backfill_five_hour_blocks(
2038
2416
  # reset, but only a later retained observation may close and
2039
2417
  # clock it (#399). The ordinary legacy backfill keeps its
2040
2418
  # historical wall-time classification.
2041
- is_closed = (
2042
- 0 if only_missing else (1 if resets_dt < now_dt else 0)
2043
- )
2044
- projection_created_at = first_obs if only_missing else now_iso
2045
- projection_updated_at = last_obs if only_missing else now_iso
2419
+ is_closed = 1 if resets_dt < now_dt else 0
2046
2420
 
2047
2421
  # Token + cost totals — recomputed via the shared helper,
2048
2422
  # which routes through get_entries() and falls back to
2049
2423
  # JSONL if cache.db is unreadable.
2050
2424
  # skip_sync=True: backfill runs inside open_db(); a sync_cache
2051
2425
  # cascade here would reopen cache.db recursively.
2052
- pricing_evidence = {} if only_missing else None
2053
2426
  totals = _c._compute_block_totals(
2054
2427
  block_start_dt, last_obs_dt, skip_sync=True,
2055
2428
  owner_key=key, windows=ownership.get(acct, []),
2056
- pricing_evidence=pricing_evidence,
2057
- )
2058
- pricing_provenance_json = (
2059
- _c._block_pricing_provenance_json(pricing_evidence)
2060
- if pricing_evidence is not None else None
2061
2429
  )
2062
2430
 
2063
2431
  cur = conn.execute(
@@ -2101,10 +2469,10 @@ def _backfill_five_hour_blocks(
2101
2469
  totals["cache_create_tokens"],
2102
2470
  totals["cache_read_tokens"],
2103
2471
  totals["cost_usd"],
2104
- pricing_provenance_json,
2472
+ None,
2105
2473
  is_closed,
2106
- projection_created_at,
2107
- projection_updated_at,
2474
+ now_iso,
2475
+ now_iso,
2108
2476
  acct,
2109
2477
  ),
2110
2478
  )
@@ -2176,6 +2544,9 @@ def _backfill_five_hour_blocks(
2176
2544
  ],
2177
2545
  )
2178
2546
 
2547
+ if only_missing:
2548
+ _refresh_every_open_block(conn)
2549
+
2179
2550
  # Mark child-table migrations done so the upgrade-user gates in
2180
2551
  # open_db() don't re-fire on next open. Inside the same
2181
2552
  # transaction as the per-window inserts so partial-fail leaves
@@ -777,7 +777,8 @@ def _snapshot_columns(conn) -> set:
777
777
  return {str(row[1]) for row in rows}
778
778
 
779
779
 
780
- def _dpp_candidate_regime(current_week_start, *, account_key):
780
+ def _dpp_candidate_regime(current_week_start, *, account_key,
781
+ stats_conn=None):
781
782
  """The S1 regime the TARGET week sits in, or None when there is none.
782
783
 
783
784
  None means "no regime restriction is knowable", which is the inert state
@@ -817,7 +818,8 @@ def _dpp_candidate_regime(current_week_start, *, account_key):
817
818
  c = _cctally()
818
819
  try:
819
820
  qcg = c._load_sibling("_cctally_quota_calibration")
820
- read = qcg.read_calibration_file(account_key=account_key)
821
+ read = qcg.read_calibration_file(account_key=account_key,
822
+ stats_conn=stats_conn)
821
823
  except Exception: # noqa: BLE001
822
824
  return None
823
825
  regime = getattr(read, "regime", None)
@@ -1032,7 +1034,8 @@ def _select_dollars_per_percent(
1032
1034
  # marked `detection-only`, so on an install with no calibration there is
1033
1035
  # no regime information and the restriction is inert. Applying it anyway
1034
1036
  # would exclude every candidate rather than the incomparable ones.
1035
- regime = _dpp_candidate_regime(current_week_start, account_key=account_key)
1037
+ regime = _dpp_candidate_regime(current_week_start, account_key=account_key,
1038
+ stats_conn=conn)
1036
1039
  eligible: list[tuple[dt.datetime, dt.datetime, float]] = []
1037
1040
  drifted = False
1038
1041
  unverified = False
@@ -1250,6 +1253,11 @@ def _pick_p_24h_ago(
1250
1253
  _CALIBRATION_REJECTION_CODES: dict = {
1251
1254
  "calibration-fingerprint-mismatch": "stale",
1252
1255
  "calibration-revision-mismatch": "stale",
1256
+ # #1015 spec 5.4: a correction to the regime's inputs. Its attempt status
1257
+ # (when current) is stated instead, exactly as the detection-only branch
1258
+ # states the stored status. `calibration-lineage-unavailable` falls to
1259
+ # `unavailable` with every other rejection.
1260
+ "recalibration-needed": "recalibration-needed",
1253
1261
  }
1254
1262
 
1255
1263
  #: The cause a `detection-only` refusal states when the stored regime's own
@@ -1283,6 +1291,7 @@ def _calibrated_projection(
1283
1291
  week_end_at: dt.datetime,
1284
1292
  *,
1285
1293
  account_key: "str | None",
1294
+ stats_conn=None,
1286
1295
  ) -> "tuple[float | None, str | None]":
1287
1296
  """`(projected_end_of_week_percent, withheld_code)` from the S1 model.
1288
1297
 
@@ -1290,7 +1299,8 @@ def _calibrated_projection(
1290
1299
  `_calibrated_week_detail` is the one implementation.
1291
1300
  """
1292
1301
  detail = _calibrated_week_detail(
1293
- now_utc, week_start_at, week_end_at, account_key=account_key)
1302
+ now_utc, week_start_at, week_end_at, account_key=account_key,
1303
+ stats_conn=stats_conn)
1294
1304
  return detail.projection_pct, detail.code
1295
1305
 
1296
1306
 
@@ -1300,6 +1310,7 @@ def _calibrated_week_detail(
1300
1310
  week_end_at: dt.datetime,
1301
1311
  *,
1302
1312
  account_key: "str | None",
1313
+ stats_conn=None,
1303
1314
  ) -> CalibratedWeek:
1304
1315
  """The S1 model's view of THIS week, or the typed cause it has none.
1305
1316
 
@@ -1324,9 +1335,21 @@ def _calibrated_week_detail(
1324
1335
  # of the function. That function now returns THREE values, which is why
1325
1336
  # its unreadable-state path returns `None, (), ()`.
1326
1337
  return CalibratedWeek(code="unavailable")
1327
- read = qcg.read_calibration_file(account_key=account_key)
1338
+ # #1015: the stats connection the caller holds, when it holds one, so the
1339
+ # lineage assessment reads the same index; otherwise the reader's
1340
+ # fail-fast opener.
1341
+ read = qcg.read_calibration_file(account_key=account_key,
1342
+ stats_conn=stats_conn)
1328
1343
  if read.regime is None:
1329
1344
  rejection = getattr(read.rejection, "value", None)
1345
+ if rejection == "recalibration-needed":
1346
+ # #1015 spec 5.4: the sweep's withheld attempt over the current
1347
+ # basis names its own status; otherwise the refusal itself.
1348
+ qm = c._load_sibling("_lib_quota_model")
1349
+ stored = read.regime_status
1350
+ return CalibratedWeek(code=(
1351
+ stored if stored in qm.EVIDENCE_CODES
1352
+ else qm.RECALIBRATION_NEEDED))
1330
1353
  if rejection == "calibration-detection-only":
1331
1354
  # S1 held this successor fit below its prediction gate, and the
1332
1355
  # regime records which blocking status it carried. State that
@@ -1522,7 +1545,8 @@ def _load_forecast_inputs(
1522
1545
  p_24h_ago_corrected = corrected_percent_point(p_24h_ago)
1523
1546
  right_censored = p_now_corrected is None
1524
1547
  calibrated = _calibrated_week_detail(
1525
- now_utc, week_start_at, week_end_at, account_key=account_key)
1548
+ now_utc, week_start_at, week_end_at, account_key=account_key,
1549
+ stats_conn=conn)
1526
1550
 
1527
1551
  # Cache is warm for this invocation after the spent_usd lookup. Suppress
1528
1552
  # re-syncs in downstream cost lookups (trailing-4wk-median loop hits