cctally 1.116.0 → 1.117.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +57 -3
- package/bin/_cctally_cache.py +5720 -885
- package/bin/_cctally_claude_attribution.py +997 -0
- package/bin/_cctally_completion.py +101 -28
- package/bin/_cctally_core.py +235 -12
- package/bin/_cctally_dashboard.py +224 -7
- package/bin/_cctally_dashboard_conversation.py +107 -21
- package/bin/_cctally_dashboard_envelope.py +34 -0
- package/bin/_cctally_db.py +928 -80
- package/bin/_cctally_diagnosis_sources.py +78 -13
- package/bin/_cctally_doctor.py +211 -86
- package/bin/_cctally_five_hour.py +389 -18
- package/bin/_cctally_forecast.py +30 -6
- package/bin/_cctally_journal.py +2418 -158
- package/bin/_cctally_oauth_samples.py +185 -7
- package/bin/_cctally_parser.py +38 -0
- package/bin/_cctally_project.py +34 -8
- package/bin/_cctally_quarantine.py +821 -343
- package/bin/_cctally_quota.py +69 -4
- package/bin/_cctally_quota_calibration.py +677 -3
- package/bin/_cctally_quota_model.py +927 -127
- package/bin/_cctally_record.py +922 -445
- package/bin/_cctally_rederive.py +119 -15
- package/bin/_cctally_statusline.py +1048 -176
- package/bin/_cctally_store.py +544 -110
- package/bin/_cctally_tui.py +93 -9
- package/bin/_cctally_update.py +77 -14
- package/bin/_lib_account_protocol.py +227 -19
- package/bin/_lib_claude_attribution.py +421 -0
- package/bin/_lib_clone_copy.py +30 -0
- package/bin/_lib_codex_conversation_query.py +45 -9
- package/bin/_lib_codex_conversation_watch.py +72 -13
- package/bin/_lib_codex_rollout_container.py +234 -19
- package/bin/_lib_codex_worker_lifecycle.py +374 -0
- package/bin/_lib_credit.py +22 -13
- package/bin/_lib_dashboard_sources.py +163 -10
- package/bin/_lib_detector_access_registry.py +1335 -166
- package/bin/_lib_doctor.py +276 -92
- package/bin/_lib_ingest_frontier.py +269 -36
- package/bin/_lib_journal.py +273 -22
- package/bin/_lib_jsonl.py +12 -0
- package/bin/_lib_meter_rate_change.py +72 -0
- package/bin/_lib_outcome_quarantine.py +917 -554
- package/bin/_lib_provider_activity.py +24 -6
- package/bin/_lib_quota_calibration.py +57 -3
- package/bin/_lib_quota_copy.py +7 -2
- package/bin/_lib_quota_lineage.py +613 -0
- package/bin/_lib_quota_model.py +435 -22
- package/bin/_lib_rate_change_delivery.py +16 -0
- package/bin/_lib_reclaim_planner.py +6 -4
- package/bin/_lib_rederive.py +2046 -21
- package/bin/_lib_selector_state.py +131 -3
- package/bin/_lib_stale_proof.py +113 -4
- package/bin/_lib_statusline.py +72 -3
- package/bin/_lib_statusline_candidates.py +246 -42
- package/bin/_lib_write_io.py +418 -5
- package/bin/cctally +24 -11
- package/dashboard/static/assets/{ConversationsView-Dflb_myA.js → ConversationsView-Dcfn8T6f.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-DFX5CfIs.js → DoctorModal-BqmXbnhn.js} +1 -1
- package/dashboard/static/assets/{ModalRoot-HlxoagIa.js → ModalRoot-Ck3WJK2V.js} +1 -1
- package/dashboard/static/assets/{ProjectsDrillPanel-C3PJy1xx.js → ProjectsDrillPanel-B-HZkDnT.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-BczsWKhD.js → SourceDetailModal-QhJkHB7i.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-Os-Lo5cs.js → UpdateModal-CkXEIQcK.js} +1 -1
- package/dashboard/static/assets/index-D34eYhAc.css +1 -0
- package/dashboard/static/assets/{index-CYZ3NLqF.js → index-HKS1s56o.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-k-zIif6W.js → outlineNavigation-BNGaNPCp.js} +1 -1
- package/dashboard/static/assets/{useKeymap-CW7Up0Ay.js → useKeymap-CUgEnnua.js} +1 -1
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +5 -1
- package/dashboard/static/assets/index-BKITk1CT.css +0 -1
|
@@ -42,6 +42,7 @@ import datetime as dt
|
|
|
42
42
|
import json
|
|
43
43
|
import sqlite3
|
|
44
44
|
import sys
|
|
45
|
+
from dataclasses import dataclass
|
|
45
46
|
|
|
46
47
|
import _lib_accounts # pure stdlib kernel; UNATTRIBUTED sentinel (#341)
|
|
47
48
|
import _lib_rebuild_progress # #997 D9 progress ticks (stdlib only)
|
|
@@ -1779,9 +1780,359 @@ def cmd_five_hour_breakdown(args: argparse.Namespace) -> int:
|
|
|
1779
1780
|
conn.close()
|
|
1780
1781
|
|
|
1781
1782
|
|
|
1783
|
+
@dataclass(frozen=True)
|
|
1784
|
+
class OpenBlockEnd:
|
|
1785
|
+
"""The reading an open five-hour block ends at (#956 class 1, spec
|
|
1786
|
+
4.5.1): its capture, five-hour percent and weekly percent, as stored and
|
|
1787
|
+
fold-effective. ``origin`` names where it came from: ``"snapshot"`` is the
|
|
1788
|
+
window's latest surviving five-hour reading, ``"opening"`` the opening
|
|
1789
|
+
reading of the window's retained opening fact (`five_hour_block_openings`,
|
|
1790
|
+
#956 class 2, spec 4.5.4) when no reading survives."""
|
|
1791
|
+
|
|
1792
|
+
captured_at_utc: str
|
|
1793
|
+
five_hour_percent: float
|
|
1794
|
+
seven_day_percent: "float | None"
|
|
1795
|
+
origin: str
|
|
1796
|
+
|
|
1797
|
+
|
|
1798
|
+
def select_open_block_end(
|
|
1799
|
+
conn: sqlite3.Connection, *, account_key: str, five_hour_window_key: int,
|
|
1800
|
+
) -> "OpenBlockEnd | None":
|
|
1801
|
+
"""The end reading of one open block, read only (#956 class 1): the
|
|
1802
|
+
window's latest surviving five-hour reading in CAPTURE order, held rows
|
|
1803
|
+
included, ties broken by logical identity (`journal_id`, a NULL one
|
|
1804
|
+
lowest), never by journal order or rowid.
|
|
1805
|
+
|
|
1806
|
+
With no surviving reading (a correction removed every one, or `record-
|
|
1807
|
+
credit --force` removed the synthetics that were the window's only
|
|
1808
|
+
readings) it is the window's opening fact's OPENING reading (#956 spec
|
|
1809
|
+
4.5.4): its capture, five-hour percent and weekly start value, retained
|
|
1810
|
+
and immutable, so live, every rebuild and rederive agree. ``None`` only
|
|
1811
|
+
for a legacy window with neither (residual R4); the caller then leaves
|
|
1812
|
+
the end columns as they are.
|
|
1813
|
+
|
|
1814
|
+
Live ingest, derivation completion and the rebuild's Phase 2a all read
|
|
1815
|
+
the end here, so an earlier capture appended later, a credit that
|
|
1816
|
+
suppresses the selected reading and a `record-credit` synthetic move
|
|
1817
|
+
every path identically. `latest_five_hour_reading` (the week-scoped
|
|
1818
|
+
selection forecast, statusline and the dashboard read) is deliberately
|
|
1819
|
+
a different function and is not changed."""
|
|
1820
|
+
# The order `unixepoch(captured_at_utc) DESC, journal_id DESC` (NULLs
|
|
1821
|
+
# lowest on both keys, as SQLite sorts them under DESC) is applied here
|
|
1822
|
+
# over the window's rows rather than by an ORDER BY: this runs on every
|
|
1823
|
+
# live tick, and the window-key index cannot serve an expression order,
|
|
1824
|
+
# so SQL would sort through a temp B-tree each time (#902's recurring-
|
|
1825
|
+
# plan guard). One five-hour window holds a bounded number of readings.
|
|
1826
|
+
rows = conn.execute(
|
|
1827
|
+
"""
|
|
1828
|
+
SELECT unixepoch(captured_at_utc) AS captured_epoch, journal_id,
|
|
1829
|
+
captured_at_utc, five_hour_percent, weekly_percent
|
|
1830
|
+
FROM weekly_usage_snapshots
|
|
1831
|
+
WHERE account_key = ? AND five_hour_window_key = ?
|
|
1832
|
+
AND five_hour_percent IS NOT NULL
|
|
1833
|
+
""",
|
|
1834
|
+
(account_key, int(five_hour_window_key)),
|
|
1835
|
+
).fetchall()
|
|
1836
|
+
if not rows:
|
|
1837
|
+
seed = conn.execute(
|
|
1838
|
+
"SELECT first_observed_at_utc, five_hour_percent, "
|
|
1839
|
+
" seven_day_pct_at_block_start "
|
|
1840
|
+
" FROM five_hour_block_openings "
|
|
1841
|
+
" WHERE account_key = ? AND five_hour_window_key = ?",
|
|
1842
|
+
(account_key, int(five_hour_window_key)),
|
|
1843
|
+
).fetchone()
|
|
1844
|
+
if seed is None:
|
|
1845
|
+
return None
|
|
1846
|
+
return OpenBlockEnd(
|
|
1847
|
+
captured_at_utc=seed[0],
|
|
1848
|
+
five_hour_percent=float(seed[1]),
|
|
1849
|
+
seven_day_percent=None if seed[2] is None else float(seed[2]),
|
|
1850
|
+
origin="opening",
|
|
1851
|
+
)
|
|
1852
|
+
row = max(rows, key=lambda r: (
|
|
1853
|
+
r["captured_epoch"] is not None, r["captured_epoch"] or 0,
|
|
1854
|
+
r["journal_id"] is not None, r["journal_id"] or "",
|
|
1855
|
+
))
|
|
1856
|
+
return OpenBlockEnd(
|
|
1857
|
+
captured_at_utc=row["captured_at_utc"],
|
|
1858
|
+
five_hour_percent=float(row["five_hour_percent"]),
|
|
1859
|
+
seven_day_percent=(
|
|
1860
|
+
None if row["weekly_percent"] is None
|
|
1861
|
+
else float(row["weekly_percent"])
|
|
1862
|
+
),
|
|
1863
|
+
origin="snapshot",
|
|
1864
|
+
)
|
|
1865
|
+
|
|
1866
|
+
|
|
1867
|
+
def project_open_block_end(
|
|
1868
|
+
conn: sqlite3.Connection, block, end: "OpenBlockEnd | None", *,
|
|
1869
|
+
ownership, journal_ctx, as_of: str,
|
|
1870
|
+
) -> None:
|
|
1871
|
+
"""Project one OPEN block's end columns and accounting from ``end``
|
|
1872
|
+
(#956 class 1, spec 4.5.1), within the caller's transaction.
|
|
1873
|
+
|
|
1874
|
+
It sets `last_observed_at_utc`, `final_five_hour_percent` and
|
|
1875
|
+
`seven_day_pct_at_block_end` from ``end`` (left as they are when ``end``
|
|
1876
|
+
is None), and recomputes the four token totals, `total_cost_usd`, both
|
|
1877
|
+
child sets, `pricing_provenance_json` and `crossed_seven_day_reset` to
|
|
1878
|
+
the end's capture, the INCLUSIVE observation cutoff. Ownership is
|
|
1879
|
+
`_lib_blocks.resolve_owning_window` over ``ownership`` (OwnedWindow
|
|
1880
|
+
values of the account's block windows, the block's own included), with
|
|
1881
|
+
the half-open ``[start, reset)`` interval. With rederive's
|
|
1882
|
+
`block_source_evidence` sink present it records the owned entries behind
|
|
1883
|
+
these totals. It never touches start columns, never refreshes a closed
|
|
1884
|
+
block (a journaled fact), and stamps `last_updated_at_utc = as_of`.
|
|
1885
|
+
|
|
1886
|
+
Cache inputs are read with ``skip_sync=True``: this runs inside a stats
|
|
1887
|
+
transaction (and inside `open_db()` on the rebuild path), and the live
|
|
1888
|
+
tick's own totals computation already synced the cache."""
|
|
1889
|
+
_c = _cctally()
|
|
1890
|
+
current = conn.execute(
|
|
1891
|
+
"SELECT * FROM five_hour_blocks WHERE id = ?", (int(block["id"]),),
|
|
1892
|
+
).fetchone()
|
|
1893
|
+
if current is None or int(current["is_closed"]) != 0:
|
|
1894
|
+
return
|
|
1895
|
+
account_key = current["account_key"]
|
|
1896
|
+
window_key = int(current["five_hour_window_key"])
|
|
1897
|
+
block_start_dt = parse_iso_datetime(
|
|
1898
|
+
current["block_start_at"], "five_hour_blocks.block_start_at",
|
|
1899
|
+
)
|
|
1900
|
+
if end is None:
|
|
1901
|
+
last_observed = current["last_observed_at_utc"]
|
|
1902
|
+
final_five_hour = current["final_five_hour_percent"]
|
|
1903
|
+
weekly_end = current["seven_day_pct_at_block_end"]
|
|
1904
|
+
else:
|
|
1905
|
+
last_observed = end.captured_at_utc
|
|
1906
|
+
final_five_hour = float(end.five_hour_percent)
|
|
1907
|
+
weekly_end = end.seven_day_percent
|
|
1908
|
+
cutoff_dt = parse_iso_datetime(
|
|
1909
|
+
last_observed, "five_hour_blocks.last_observed_at_utc",
|
|
1910
|
+
)
|
|
1911
|
+
windows = list(ownership)
|
|
1912
|
+
if not any(w.key == window_key for w in windows):
|
|
1913
|
+
windows.append(_c.OwnedWindow(
|
|
1914
|
+
key=window_key,
|
|
1915
|
+
start=block_start_dt.astimezone(dt.timezone.utc),
|
|
1916
|
+
reset=parse_iso_datetime(
|
|
1917
|
+
current["five_hour_resets_at"],
|
|
1918
|
+
"five_hour_blocks.five_hour_resets_at",
|
|
1919
|
+
).astimezone(dt.timezone.utc),
|
|
1920
|
+
))
|
|
1921
|
+
pricing_evidence: dict = {}
|
|
1922
|
+
source_sink = getattr(journal_ctx, "block_source_evidence", None)
|
|
1923
|
+
source_key = (account_key, window_key)
|
|
1924
|
+
owned_entries = (
|
|
1925
|
+
[] if source_sink is not None and source_key in source_sink else None
|
|
1926
|
+
)
|
|
1927
|
+
totals = _c._compute_block_totals(
|
|
1928
|
+
block_start_dt, cutoff_dt, skip_sync=True,
|
|
1929
|
+
owner_key=window_key, windows=windows,
|
|
1930
|
+
pricing_evidence=pricing_evidence, owned_entries=owned_entries,
|
|
1931
|
+
)
|
|
1932
|
+
if owned_entries is not None:
|
|
1933
|
+
source_sink[source_key] = tuple(owned_entries)
|
|
1934
|
+
# The reset-crossing flag over [block_start_at, end], from the same two
|
|
1935
|
+
# sources and predicates as the live sweep
|
|
1936
|
+
# (`_flag_blocks_spanning_week_resets`), computed exactly so an end that
|
|
1937
|
+
# moves backward clears it too.
|
|
1938
|
+
crossed = conn.execute(
|
|
1939
|
+
"""
|
|
1940
|
+
SELECT
|
|
1941
|
+
EXISTS (
|
|
1942
|
+
SELECT 1 FROM week_reset_events
|
|
1943
|
+
WHERE account_key = ?
|
|
1944
|
+
AND unixepoch(effective_reset_at_utc)
|
|
1945
|
+
BETWEEN unixepoch(?) AND unixepoch(?)
|
|
1946
|
+
)
|
|
1947
|
+
OR EXISTS (
|
|
1948
|
+
SELECT 1 FROM weekly_usage_snapshots
|
|
1949
|
+
WHERE week_start_at IS NOT NULL
|
|
1950
|
+
AND account_key = ?
|
|
1951
|
+
AND unixepoch(week_start_at) > unixepoch(?)
|
|
1952
|
+
AND unixepoch(week_start_at) <= unixepoch(?)
|
|
1953
|
+
)
|
|
1954
|
+
""",
|
|
1955
|
+
(account_key, current["block_start_at"], last_observed,
|
|
1956
|
+
account_key, current["block_start_at"], last_observed),
|
|
1957
|
+
).fetchone()[0]
|
|
1958
|
+
conn.execute(
|
|
1959
|
+
"""
|
|
1960
|
+
UPDATE five_hour_blocks
|
|
1961
|
+
SET last_observed_at_utc = ?,
|
|
1962
|
+
final_five_hour_percent = ?,
|
|
1963
|
+
seven_day_pct_at_block_end = ?,
|
|
1964
|
+
total_input_tokens = ?,
|
|
1965
|
+
total_output_tokens = ?,
|
|
1966
|
+
total_cache_create_tokens = ?,
|
|
1967
|
+
total_cache_read_tokens = ?,
|
|
1968
|
+
total_cost_usd = ?,
|
|
1969
|
+
pricing_provenance_json = ?,
|
|
1970
|
+
crossed_seven_day_reset = ?,
|
|
1971
|
+
last_updated_at_utc = ?
|
|
1972
|
+
WHERE id = ? AND is_closed = 0
|
|
1973
|
+
""",
|
|
1974
|
+
(
|
|
1975
|
+
last_observed, final_five_hour, weekly_end,
|
|
1976
|
+
totals["input_tokens"], totals["output_tokens"],
|
|
1977
|
+
totals["cache_create_tokens"], totals["cache_read_tokens"],
|
|
1978
|
+
totals["cost_usd"],
|
|
1979
|
+
_c._block_pricing_provenance_json(pricing_evidence),
|
|
1980
|
+
1 if crossed else 0, as_of, int(current["id"]),
|
|
1981
|
+
),
|
|
1982
|
+
)
|
|
1983
|
+
# Replace-all children, keyed on the window (not block_id) so an orphan
|
|
1984
|
+
# child of an earlier parent rebuild is cleaned up as well.
|
|
1985
|
+
block_id = int(current["id"])
|
|
1986
|
+
conn.execute(
|
|
1987
|
+
"DELETE FROM five_hour_block_models "
|
|
1988
|
+
"WHERE five_hour_window_key = ? AND account_key = ?",
|
|
1989
|
+
(window_key, account_key),
|
|
1990
|
+
)
|
|
1991
|
+
if totals.get("by_model"):
|
|
1992
|
+
conn.executemany(
|
|
1993
|
+
"""
|
|
1994
|
+
INSERT INTO five_hour_block_models (
|
|
1995
|
+
block_id, five_hour_window_key, model,
|
|
1996
|
+
input_tokens, output_tokens,
|
|
1997
|
+
cache_create_tokens, cache_read_tokens,
|
|
1998
|
+
cost_usd, entry_count, account_key
|
|
1999
|
+
)
|
|
2000
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2001
|
+
""",
|
|
2002
|
+
[
|
|
2003
|
+
(
|
|
2004
|
+
block_id, window_key, model,
|
|
2005
|
+
b["input_tokens"], b["output_tokens"],
|
|
2006
|
+
b["cache_create_tokens"], b["cache_read_tokens"],
|
|
2007
|
+
b["cost_usd"], b["entry_count"], account_key,
|
|
2008
|
+
)
|
|
2009
|
+
for model, b in totals["by_model"].items()
|
|
2010
|
+
],
|
|
2011
|
+
)
|
|
2012
|
+
conn.execute(
|
|
2013
|
+
"DELETE FROM five_hour_block_projects "
|
|
2014
|
+
"WHERE five_hour_window_key = ? AND account_key = ?",
|
|
2015
|
+
(window_key, account_key),
|
|
2016
|
+
)
|
|
2017
|
+
if totals.get("by_project"):
|
|
2018
|
+
conn.executemany(
|
|
2019
|
+
"""
|
|
2020
|
+
INSERT INTO five_hour_block_projects (
|
|
2021
|
+
block_id, five_hour_window_key, project_path,
|
|
2022
|
+
input_tokens, output_tokens,
|
|
2023
|
+
cache_create_tokens, cache_read_tokens,
|
|
2024
|
+
cost_usd, entry_count, account_key
|
|
2025
|
+
)
|
|
2026
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2027
|
+
""",
|
|
2028
|
+
[
|
|
2029
|
+
(
|
|
2030
|
+
block_id, window_key, project_path,
|
|
2031
|
+
b["input_tokens"], b["output_tokens"],
|
|
2032
|
+
b["cache_create_tokens"], b["cache_read_tokens"],
|
|
2033
|
+
b["cost_usd"], b["entry_count"], account_key,
|
|
2034
|
+
)
|
|
2035
|
+
for project_path, b in totals["by_project"].items()
|
|
2036
|
+
],
|
|
2037
|
+
)
|
|
2038
|
+
|
|
2039
|
+
|
|
2040
|
+
def _materialize_open_candidate(conn, key, acct, start_row) -> int:
|
|
2041
|
+
"""Insert one rebuild open-block candidate with its start columns from
|
|
2042
|
+
``start_row`` (the first writer, else the earliest survivor) and its end
|
|
2043
|
+
from `select_open_block_end`; zero totals and NULL provenance until
|
|
2044
|
+
`_refresh_every_open_block` projects it in the same transaction. Returns
|
|
2045
|
+
the inserted row count."""
|
|
2046
|
+
end = select_open_block_end(
|
|
2047
|
+
conn, account_key=acct, five_hour_window_key=key)
|
|
2048
|
+
if start_row is None or end is None:
|
|
2049
|
+
return 0 # defensive — should be unreachable per the keys query
|
|
2050
|
+
resets_at = start_row["five_hour_resets_at"]
|
|
2051
|
+
block_start_at = (
|
|
2052
|
+
parse_iso_datetime(resets_at, "resets_at backfill")
|
|
2053
|
+
- dt.timedelta(hours=5)
|
|
2054
|
+
).isoformat(timespec="seconds")
|
|
2055
|
+
# A rebuild's only-missing row is the trailing unjournaled projection,
|
|
2056
|
+
# never a close decision. Wall time may be past its reset, but only a
|
|
2057
|
+
# later retained observation may close and clock it (#399).
|
|
2058
|
+
cur = conn.execute(
|
|
2059
|
+
"""
|
|
2060
|
+
INSERT OR IGNORE INTO five_hour_blocks (
|
|
2061
|
+
five_hour_window_key, five_hour_resets_at, block_start_at,
|
|
2062
|
+
first_observed_at_utc, last_observed_at_utc,
|
|
2063
|
+
final_five_hour_percent,
|
|
2064
|
+
seven_day_pct_at_block_start, seven_day_pct_at_block_end,
|
|
2065
|
+
crossed_seven_day_reset,
|
|
2066
|
+
total_input_tokens, total_output_tokens,
|
|
2067
|
+
total_cache_create_tokens, total_cache_read_tokens,
|
|
2068
|
+
total_cost_usd, pricing_provenance_json, is_closed,
|
|
2069
|
+
created_at_utc, last_updated_at_utc, account_key
|
|
2070
|
+
)
|
|
2071
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0, NULL, 0, ?, ?, ?)
|
|
2072
|
+
""",
|
|
2073
|
+
(
|
|
2074
|
+
key, resets_at, block_start_at,
|
|
2075
|
+
start_row["captured_at_utc"], end.captured_at_utc,
|
|
2076
|
+
end.five_hour_percent,
|
|
2077
|
+
start_row["weekly_percent"], end.seven_day_percent,
|
|
2078
|
+
start_row["captured_at_utc"], end.captured_at_utc, acct,
|
|
2079
|
+
),
|
|
2080
|
+
)
|
|
2081
|
+
return cur.rowcount or 0
|
|
2082
|
+
|
|
2083
|
+
|
|
2084
|
+
def _refresh_every_open_block(conn) -> int:
|
|
2085
|
+
"""Project every open block through `select_open_block_end` /
|
|
2086
|
+
`project_open_block_end` (#956 class 1, spec 4.5.3), with each account's
|
|
2087
|
+
ownership context built from every `five_hour_blocks` row as it stands
|
|
2088
|
+
now. Runs inside the caller's transaction. Returns the blocks refreshed.
|
|
2089
|
+
A block whose window has no surviving reading keeps its end columns, and
|
|
2090
|
+
its accounting is recomputed to its stored `last_observed_at_utc`."""
|
|
2091
|
+
_c = _cctally()
|
|
2092
|
+
ownership: dict = {}
|
|
2093
|
+
for row in conn.execute(
|
|
2094
|
+
"SELECT five_hour_window_key, block_start_at, "
|
|
2095
|
+
" five_hour_resets_at, account_key FROM five_hour_blocks"
|
|
2096
|
+
).fetchall():
|
|
2097
|
+
try:
|
|
2098
|
+
w_start = parse_iso_datetime(
|
|
2099
|
+
row["block_start_at"], "five_hour_blocks.block_start_at",
|
|
2100
|
+
).astimezone(dt.timezone.utc)
|
|
2101
|
+
w_reset = parse_iso_datetime(
|
|
2102
|
+
row["five_hour_resets_at"],
|
|
2103
|
+
"five_hour_blocks.five_hour_resets_at",
|
|
2104
|
+
).astimezone(dt.timezone.utc)
|
|
2105
|
+
except ValueError:
|
|
2106
|
+
continue
|
|
2107
|
+
ownership.setdefault(row["account_key"], []).append(_c.OwnedWindow(
|
|
2108
|
+
key=int(row["five_hour_window_key"]), start=w_start,
|
|
2109
|
+
reset=w_reset,
|
|
2110
|
+
))
|
|
2111
|
+
refreshed = 0
|
|
2112
|
+
for index, block in enumerate(conn.execute(
|
|
2113
|
+
"SELECT id, account_key, five_hour_window_key, last_updated_at_utc "
|
|
2114
|
+
" FROM five_hour_blocks WHERE is_closed = 0"
|
|
2115
|
+
).fetchall(), 1):
|
|
2116
|
+
if not index & 1023:
|
|
2117
|
+
_lib_rebuild_progress.tick(blocks=index)
|
|
2118
|
+
end = select_open_block_end(
|
|
2119
|
+
conn, account_key=block["account_key"],
|
|
2120
|
+
five_hour_window_key=int(block["five_hour_window_key"]))
|
|
2121
|
+
project_open_block_end(
|
|
2122
|
+
conn, block, end,
|
|
2123
|
+
ownership=ownership.get(block["account_key"], []),
|
|
2124
|
+
journal_ctx=None,
|
|
2125
|
+
as_of=(end.captured_at_utc if end is not None
|
|
2126
|
+
else block["last_updated_at_utc"]),
|
|
2127
|
+
)
|
|
2128
|
+
refreshed += 1
|
|
2129
|
+
return refreshed
|
|
2130
|
+
|
|
2131
|
+
|
|
1782
2132
|
def _backfill_five_hour_blocks(
|
|
1783
2133
|
conn: sqlite3.Connection, *, only_missing: bool = False,
|
|
1784
2134
|
journaled_first: "dict | None" = None,
|
|
2135
|
+
retired_openings: frozenset = frozenset(),
|
|
1785
2136
|
) -> int:
|
|
1786
2137
|
"""One-shot historical backfill of five_hour_blocks from existing
|
|
1787
2138
|
weekly_usage_snapshots data. Idempotent via UNIQUE(five_hour_window_key)
|
|
@@ -1834,6 +2185,26 @@ def _backfill_five_hour_blocks(
|
|
|
1834
2185
|
ordinary (legacy, pre-journal) backfill keeps NULL: its blocks predate
|
|
1835
2186
|
provenance, which `_lib_cost_provenance.historical_close_marker` reads
|
|
1836
2187
|
as unproven.
|
|
2188
|
+
|
|
2189
|
+
#956 class 1 (spec 4.5.3): with ``only_missing`` the end columns and the
|
|
2190
|
+
accounting come from the one capture-order projection live ingest uses
|
|
2191
|
+
(`select_open_block_end` / `project_open_block_end`), not from a
|
|
2192
|
+
backfill-local end row. The pass materializes the missing candidates
|
|
2193
|
+
(start columns only, as above), then builds each account's ownership
|
|
2194
|
+
context from EVERY block row after that materialization, and refreshes
|
|
2195
|
+
EVERY open block, not only those it created: a block another path
|
|
2196
|
+
materialized is refreshed against the same window set. The candidate
|
|
2197
|
+
ranking breaks equal-second ties by `journal_id` (logical identity), never
|
|
2198
|
+
by rowid, which a journal correction can reorder.
|
|
2199
|
+
|
|
2200
|
+
#956 class 2 (spec 4.6.3): a window created by a current binary has a
|
|
2201
|
+
retained opening fact whose fold already materialized its block, so it
|
|
2202
|
+
is never a candidate here and its start columns are the seed's. The
|
|
2203
|
+
candidates are the LEGACY open blocks (created before opening facts, R4):
|
|
2204
|
+
each account's latest-captured window with no block row, except a window
|
|
2205
|
+
in ``retired_openings`` — the `(account_key, five_hour_window_key)` pairs
|
|
2206
|
+
whose opening fact a correction tombstoned (`_cctally_journal.
|
|
2207
|
+
retired_block_openings`): its block was removed, never "never seeded".
|
|
1837
2208
|
"""
|
|
1838
2209
|
_c = _cctally()
|
|
1839
2210
|
inserted = 0
|
|
@@ -1884,7 +2255,7 @@ def _backfill_five_hour_blocks(
|
|
|
1884
2255
|
" SELECT id, ROW_NUMBER() OVER ("
|
|
1885
2256
|
" PARTITION BY account_key"
|
|
1886
2257
|
" ORDER BY unixepoch(captured_at_utc) DESC,"
|
|
1887
|
-
"
|
|
2258
|
+
" journal_id DESC"
|
|
1888
2259
|
" ) AS rn"
|
|
1889
2260
|
" FROM weekly_usage_snapshots"
|
|
1890
2261
|
" WHERE five_hour_window_key IS NOT NULL"
|
|
@@ -1899,6 +2270,9 @@ def _backfill_five_hour_blocks(
|
|
|
1899
2270
|
" )"
|
|
1900
2271
|
)
|
|
1901
2272
|
keys = [(int(r[0]), r[1]) for r in conn.execute(keys_sql).fetchall()]
|
|
2273
|
+
if only_missing and retired_openings:
|
|
2274
|
+
keys = [(key, acct) for key, acct in keys
|
|
2275
|
+
if (acct, key) not in retired_openings]
|
|
1902
2276
|
|
|
1903
2277
|
now_iso = now_utc_iso()
|
|
1904
2278
|
now_dt = parse_iso_datetime(now_iso, "now")
|
|
@@ -1911,10 +2285,10 @@ def _backfill_five_hour_blocks(
|
|
|
1911
2285
|
# pair re-prices its overlap twice.
|
|
1912
2286
|
ownership: dict[Any, list] = {}
|
|
1913
2287
|
seen_keys: set = set()
|
|
1914
|
-
for row in conn.execute(
|
|
2288
|
+
for row in ([] if only_missing else conn.execute(
|
|
1915
2289
|
"SELECT five_hour_window_key, block_start_at, "
|
|
1916
2290
|
" five_hour_resets_at, account_key FROM five_hour_blocks"
|
|
1917
|
-
).fetchall():
|
|
2291
|
+
).fetchall()):
|
|
1918
2292
|
try:
|
|
1919
2293
|
w_start = parse_iso_datetime(
|
|
1920
2294
|
row["block_start_at"], "five_hour_blocks.block_start_at",
|
|
@@ -1930,7 +2304,7 @@ def _backfill_five_hour_blocks(
|
|
|
1930
2304
|
ownership.setdefault(row["account_key"], []).append(
|
|
1931
2305
|
_c.OwnedWindow(key=w_key, start=w_start, reset=w_reset)
|
|
1932
2306
|
)
|
|
1933
|
-
for key, acct in keys:
|
|
2307
|
+
for key, acct in ([] if only_missing else keys):
|
|
1934
2308
|
if (acct, key) in seen_keys:
|
|
1935
2309
|
continue
|
|
1936
2310
|
# Same start-row pick the insert loop makes below, so the
|
|
@@ -1973,6 +2347,10 @@ def _backfill_five_hour_blocks(
|
|
|
1973
2347
|
# (window_key, account_key) so a shared window's per-account
|
|
1974
2348
|
# blocks each read their own account's boundary rows (#341).
|
|
1975
2349
|
min_row = _first_row(key, acct)
|
|
2350
|
+
if only_missing:
|
|
2351
|
+
inserted += _materialize_open_candidate(
|
|
2352
|
+
conn, key, acct, min_row)
|
|
2353
|
+
continue
|
|
1976
2354
|
max_row = conn.execute(
|
|
1977
2355
|
"""
|
|
1978
2356
|
SELECT captured_at_utc, weekly_percent, five_hour_percent
|
|
@@ -2038,26 +2416,16 @@ def _backfill_five_hour_blocks(
|
|
|
2038
2416
|
# reset, but only a later retained observation may close and
|
|
2039
2417
|
# clock it (#399). The ordinary legacy backfill keeps its
|
|
2040
2418
|
# historical wall-time classification.
|
|
2041
|
-
is_closed =
|
|
2042
|
-
0 if only_missing else (1 if resets_dt < now_dt else 0)
|
|
2043
|
-
)
|
|
2044
|
-
projection_created_at = first_obs if only_missing else now_iso
|
|
2045
|
-
projection_updated_at = last_obs if only_missing else now_iso
|
|
2419
|
+
is_closed = 1 if resets_dt < now_dt else 0
|
|
2046
2420
|
|
|
2047
2421
|
# Token + cost totals — recomputed via the shared helper,
|
|
2048
2422
|
# which routes through get_entries() and falls back to
|
|
2049
2423
|
# JSONL if cache.db is unreadable.
|
|
2050
2424
|
# skip_sync=True: backfill runs inside open_db(); a sync_cache
|
|
2051
2425
|
# cascade here would reopen cache.db recursively.
|
|
2052
|
-
pricing_evidence = {} if only_missing else None
|
|
2053
2426
|
totals = _c._compute_block_totals(
|
|
2054
2427
|
block_start_dt, last_obs_dt, skip_sync=True,
|
|
2055
2428
|
owner_key=key, windows=ownership.get(acct, []),
|
|
2056
|
-
pricing_evidence=pricing_evidence,
|
|
2057
|
-
)
|
|
2058
|
-
pricing_provenance_json = (
|
|
2059
|
-
_c._block_pricing_provenance_json(pricing_evidence)
|
|
2060
|
-
if pricing_evidence is not None else None
|
|
2061
2429
|
)
|
|
2062
2430
|
|
|
2063
2431
|
cur = conn.execute(
|
|
@@ -2101,10 +2469,10 @@ def _backfill_five_hour_blocks(
|
|
|
2101
2469
|
totals["cache_create_tokens"],
|
|
2102
2470
|
totals["cache_read_tokens"],
|
|
2103
2471
|
totals["cost_usd"],
|
|
2104
|
-
|
|
2472
|
+
None,
|
|
2105
2473
|
is_closed,
|
|
2106
|
-
|
|
2107
|
-
|
|
2474
|
+
now_iso,
|
|
2475
|
+
now_iso,
|
|
2108
2476
|
acct,
|
|
2109
2477
|
),
|
|
2110
2478
|
)
|
|
@@ -2176,6 +2544,9 @@ def _backfill_five_hour_blocks(
|
|
|
2176
2544
|
],
|
|
2177
2545
|
)
|
|
2178
2546
|
|
|
2547
|
+
if only_missing:
|
|
2548
|
+
_refresh_every_open_block(conn)
|
|
2549
|
+
|
|
2179
2550
|
# Mark child-table migrations done so the upgrade-user gates in
|
|
2180
2551
|
# open_db() don't re-fire on next open. Inside the same
|
|
2181
2552
|
# transaction as the per-window inserts so partial-fail leaves
|
package/bin/_cctally_forecast.py
CHANGED
|
@@ -777,7 +777,8 @@ def _snapshot_columns(conn) -> set:
|
|
|
777
777
|
return {str(row[1]) for row in rows}
|
|
778
778
|
|
|
779
779
|
|
|
780
|
-
def _dpp_candidate_regime(current_week_start, *, account_key
|
|
780
|
+
def _dpp_candidate_regime(current_week_start, *, account_key,
|
|
781
|
+
stats_conn=None):
|
|
781
782
|
"""The S1 regime the TARGET week sits in, or None when there is none.
|
|
782
783
|
|
|
783
784
|
None means "no regime restriction is knowable", which is the inert state
|
|
@@ -817,7 +818,8 @@ def _dpp_candidate_regime(current_week_start, *, account_key):
|
|
|
817
818
|
c = _cctally()
|
|
818
819
|
try:
|
|
819
820
|
qcg = c._load_sibling("_cctally_quota_calibration")
|
|
820
|
-
read = qcg.read_calibration_file(account_key=account_key
|
|
821
|
+
read = qcg.read_calibration_file(account_key=account_key,
|
|
822
|
+
stats_conn=stats_conn)
|
|
821
823
|
except Exception: # noqa: BLE001
|
|
822
824
|
return None
|
|
823
825
|
regime = getattr(read, "regime", None)
|
|
@@ -1032,7 +1034,8 @@ def _select_dollars_per_percent(
|
|
|
1032
1034
|
# marked `detection-only`, so on an install with no calibration there is
|
|
1033
1035
|
# no regime information and the restriction is inert. Applying it anyway
|
|
1034
1036
|
# would exclude every candidate rather than the incomparable ones.
|
|
1035
|
-
regime = _dpp_candidate_regime(current_week_start, account_key=account_key
|
|
1037
|
+
regime = _dpp_candidate_regime(current_week_start, account_key=account_key,
|
|
1038
|
+
stats_conn=conn)
|
|
1036
1039
|
eligible: list[tuple[dt.datetime, dt.datetime, float]] = []
|
|
1037
1040
|
drifted = False
|
|
1038
1041
|
unverified = False
|
|
@@ -1250,6 +1253,11 @@ def _pick_p_24h_ago(
|
|
|
1250
1253
|
_CALIBRATION_REJECTION_CODES: dict = {
|
|
1251
1254
|
"calibration-fingerprint-mismatch": "stale",
|
|
1252
1255
|
"calibration-revision-mismatch": "stale",
|
|
1256
|
+
# #1015 spec 5.4: a correction to the regime's inputs. Its attempt status
|
|
1257
|
+
# (when current) is stated instead, exactly as the detection-only branch
|
|
1258
|
+
# states the stored status. `calibration-lineage-unavailable` falls to
|
|
1259
|
+
# `unavailable` with every other rejection.
|
|
1260
|
+
"recalibration-needed": "recalibration-needed",
|
|
1253
1261
|
}
|
|
1254
1262
|
|
|
1255
1263
|
#: The cause a `detection-only` refusal states when the stored regime's own
|
|
@@ -1283,6 +1291,7 @@ def _calibrated_projection(
|
|
|
1283
1291
|
week_end_at: dt.datetime,
|
|
1284
1292
|
*,
|
|
1285
1293
|
account_key: "str | None",
|
|
1294
|
+
stats_conn=None,
|
|
1286
1295
|
) -> "tuple[float | None, str | None]":
|
|
1287
1296
|
"""`(projected_end_of_week_percent, withheld_code)` from the S1 model.
|
|
1288
1297
|
|
|
@@ -1290,7 +1299,8 @@ def _calibrated_projection(
|
|
|
1290
1299
|
`_calibrated_week_detail` is the one implementation.
|
|
1291
1300
|
"""
|
|
1292
1301
|
detail = _calibrated_week_detail(
|
|
1293
|
-
now_utc, week_start_at, week_end_at, account_key=account_key
|
|
1302
|
+
now_utc, week_start_at, week_end_at, account_key=account_key,
|
|
1303
|
+
stats_conn=stats_conn)
|
|
1294
1304
|
return detail.projection_pct, detail.code
|
|
1295
1305
|
|
|
1296
1306
|
|
|
@@ -1300,6 +1310,7 @@ def _calibrated_week_detail(
|
|
|
1300
1310
|
week_end_at: dt.datetime,
|
|
1301
1311
|
*,
|
|
1302
1312
|
account_key: "str | None",
|
|
1313
|
+
stats_conn=None,
|
|
1303
1314
|
) -> CalibratedWeek:
|
|
1304
1315
|
"""The S1 model's view of THIS week, or the typed cause it has none.
|
|
1305
1316
|
|
|
@@ -1324,9 +1335,21 @@ def _calibrated_week_detail(
|
|
|
1324
1335
|
# of the function. That function now returns THREE values, which is why
|
|
1325
1336
|
# its unreadable-state path returns `None, (), ()`.
|
|
1326
1337
|
return CalibratedWeek(code="unavailable")
|
|
1327
|
-
|
|
1338
|
+
# #1015: the stats connection the caller holds, when it holds one, so the
|
|
1339
|
+
# lineage assessment reads the same index; otherwise the reader's
|
|
1340
|
+
# fail-fast opener.
|
|
1341
|
+
read = qcg.read_calibration_file(account_key=account_key,
|
|
1342
|
+
stats_conn=stats_conn)
|
|
1328
1343
|
if read.regime is None:
|
|
1329
1344
|
rejection = getattr(read.rejection, "value", None)
|
|
1345
|
+
if rejection == "recalibration-needed":
|
|
1346
|
+
# #1015 spec 5.4: the sweep's withheld attempt over the current
|
|
1347
|
+
# basis names its own status; otherwise the refusal itself.
|
|
1348
|
+
qm = c._load_sibling("_lib_quota_model")
|
|
1349
|
+
stored = read.regime_status
|
|
1350
|
+
return CalibratedWeek(code=(
|
|
1351
|
+
stored if stored in qm.EVIDENCE_CODES
|
|
1352
|
+
else qm.RECALIBRATION_NEEDED))
|
|
1330
1353
|
if rejection == "calibration-detection-only":
|
|
1331
1354
|
# S1 held this successor fit below its prediction gate, and the
|
|
1332
1355
|
# regime records which blocking status it carried. State that
|
|
@@ -1522,7 +1545,8 @@ def _load_forecast_inputs(
|
|
|
1522
1545
|
p_24h_ago_corrected = corrected_percent_point(p_24h_ago)
|
|
1523
1546
|
right_censored = p_now_corrected is None
|
|
1524
1547
|
calibrated = _calibrated_week_detail(
|
|
1525
|
-
now_utc, week_start_at, week_end_at, account_key=account_key
|
|
1548
|
+
now_utc, week_start_at, week_end_at, account_key=account_key,
|
|
1549
|
+
stats_conn=conn)
|
|
1526
1550
|
|
|
1527
1551
|
# Cache is warm for this invocation after the spent_usd lookup. Suppress
|
|
1528
1552
|
# re-syncs in downstream cost lookups (trailing-4wk-median loop hits
|