cctally 1.108.0 → 1.110.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/README.md +6 -6
- package/bin/_cctally_account.py +4 -0
- package/bin/_cctally_alerts.py +6 -2
- package/bin/_cctally_cache.py +1652 -166
- package/bin/_cctally_config.py +3 -2
- package/bin/_cctally_core.py +594 -33
- package/bin/_cctally_dashboard.py +1585 -201
- package/bin/_cctally_dashboard_conversation.py +494 -48
- package/bin/_cctally_dashboard_envelope.py +167 -11
- package/bin/_cctally_dashboard_share.py +154 -36
- package/bin/_cctally_dashboard_sources.py +2615 -422
- package/bin/_cctally_db.py +678 -5
- package/bin/_cctally_diagnosis_sources.py +238 -26
- package/bin/_cctally_doctor.py +99 -18
- package/bin/_cctally_five_hour.py +1295 -40
- package/bin/_cctally_forecast.py +124 -28
- package/bin/_cctally_journal.py +408 -38
- package/bin/_cctally_milestone_history.py +150 -14
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_percent_breakdown.py +6 -1
- package/bin/_cctally_project.py +48 -76
- package/bin/_cctally_quota.py +39 -7
- package/bin/_cctally_quota_model.py +17 -2
- package/bin/_cctally_record.py +1647 -443
- package/bin/_cctally_rederive.py +899 -30
- package/bin/_cctally_refresh.py +2 -1
- package/bin/_cctally_share.py +24 -5
- package/bin/_cctally_source_analytics.py +1035 -78
- package/bin/_cctally_statusline.py +83 -19
- package/bin/_cctally_store.py +61 -4
- package/bin/_cctally_transcript.py +27 -2
- package/bin/_cctally_tui.py +264 -76
- package/bin/_cctally_weekrefs.py +28 -5
- package/bin/_lib_blocks.py +127 -20
- package/bin/_lib_cache_report.py +20 -5
- package/bin/_lib_codex_conversation_query.py +297 -64
- package/bin/_lib_codex_hooks.py +102 -0
- package/bin/_lib_codex_metadata.py +207 -0
- package/bin/_lib_conversation.py +119 -0
- package/bin/_lib_conversation_dispatch.py +31 -1
- package/bin/_lib_conversation_query.py +315 -51
- package/bin/_lib_conversation_retention.py +413 -25
- package/bin/_lib_dashboard_sources.py +267 -7
- package/bin/_lib_diagnosis.py +10 -0
- package/bin/_lib_diff_kernel.py +12 -1
- package/bin/_lib_doctor.py +230 -8
- package/bin/_lib_ingest_frontier.py +1366 -49
- package/bin/_lib_journal.py +19 -2
- package/bin/_lib_merge_gate.py +307 -0
- package/bin/_lib_pricing.py +588 -29
- package/bin/_lib_record.py +130 -12
- package/bin/_lib_rederive.py +903 -2
- package/bin/_lib_render.py +9 -3
- package/bin/_lib_retained_size.py +27 -2
- package/bin/_lib_segment_summary.py +66 -8
- package/bin/_lib_share_templates.py +28 -18
- package/bin/_lib_snapshot_cache.py +115 -1
- package/bin/_lib_source_retry.py +320 -0
- package/bin/_lib_statusline_candidates.py +400 -36
- package/bin/_lib_tick_stats.py +119 -0
- package/bin/_lib_view_models.py +256 -76
- package/bin/cctally +69 -0
- package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
- package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
- package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
- package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
- package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
- package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
- package/dashboard/static/assets/index-COhxe3vU.css +1 -0
- package/dashboard/static/assets/index-DdU62sov.js +13 -0
- package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
- package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +4 -1
- package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
- package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
- package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
- package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
- package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
- package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
- package/dashboard/static/assets/index-CIWsbux3.js +0 -13
- package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
- package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
|
@@ -38,11 +38,13 @@ Spec: docs/superpowers/specs/2026-05-30-extract-five-hour-statusline-cmd-design.
|
|
|
38
38
|
from __future__ import annotations
|
|
39
39
|
|
|
40
40
|
import argparse
|
|
41
|
+
import collections
|
|
41
42
|
import datetime as dt
|
|
42
43
|
import json
|
|
43
44
|
import sqlite3
|
|
44
45
|
import sys
|
|
45
46
|
|
|
47
|
+
import _lib_accounts # pure stdlib kernel; UNATTRIBUTED sentinel (#341)
|
|
46
48
|
from _cctally_core import _command_as_of, eprint, now_utc_iso, open_db, parse_iso_datetime
|
|
47
49
|
|
|
48
50
|
|
|
@@ -685,7 +687,12 @@ def cmd_blocks(args: argparse.Namespace) -> int:
|
|
|
685
687
|
# additional entries. Without re-aggregation the displayed window
|
|
686
688
|
# said one thing and the cost said another (live data: window
|
|
687
689
|
# 20:50→01:50 with $45 cost vs the real $128).
|
|
688
|
-
_maybe_swap_active_block_to_canonical(
|
|
690
|
+
_maybe_swap_active_block_to_canonical(
|
|
691
|
+
blocks, all_entries, now=now_utc, mode=args.mode,
|
|
692
|
+
competing_windows=_ownership_windows_from_recorded(
|
|
693
|
+
recorded_windows, block_start_overrides, canonical_intervals,
|
|
694
|
+
),
|
|
695
|
+
)
|
|
689
696
|
|
|
690
697
|
# ── Session F (#86): resolve token limit, then filter ────────────────
|
|
691
698
|
# Auto-max baseline over ALL blocks (before --recent/--active filtering),
|
|
@@ -746,12 +753,41 @@ def cmd_blocks(args: argparse.Namespace) -> int:
|
|
|
746
753
|
return 0
|
|
747
754
|
|
|
748
755
|
|
|
756
|
+
def _ownership_windows_from_recorded(
|
|
757
|
+
recorded_windows,
|
|
758
|
+
block_start_overrides=None,
|
|
759
|
+
canonical_intervals=None,
|
|
760
|
+
) -> "list[Any]":
|
|
761
|
+
"""The `_load_recorded_five_hour_windows` result as ownership windows.
|
|
762
|
+
|
|
763
|
+
Resolves each anchor's interval exactly as `_group_entries_into_blocks`
|
|
764
|
+
does — canonical interval first, then the override-supplied start, then
|
|
765
|
+
the legacy ``(R - 5h, R)`` shape — so a consumer outside the grouping
|
|
766
|
+
pass reasons about the same intervals it did.
|
|
767
|
+
"""
|
|
768
|
+
_c = _cctally()
|
|
769
|
+
canonical_intervals = canonical_intervals or {}
|
|
770
|
+
block_start_overrides = block_start_overrides or {}
|
|
771
|
+
out: list[Any] = []
|
|
772
|
+
for anchor in recorded_windows or ():
|
|
773
|
+
if anchor in canonical_intervals:
|
|
774
|
+
start, reset = canonical_intervals[anchor]
|
|
775
|
+
else:
|
|
776
|
+
start = block_start_overrides.get(
|
|
777
|
+
anchor, anchor - dt.timedelta(hours=5),
|
|
778
|
+
)
|
|
779
|
+
reset = anchor
|
|
780
|
+
out.append(_c.OwnedWindow(key=anchor, start=start, reset=reset))
|
|
781
|
+
return out
|
|
782
|
+
|
|
783
|
+
|
|
749
784
|
def _maybe_swap_active_block_to_canonical(
|
|
750
785
|
blocks: list[Any],
|
|
751
786
|
all_entries: list[Any],
|
|
752
787
|
*,
|
|
753
788
|
now: dt.datetime,
|
|
754
789
|
mode: str = "auto",
|
|
790
|
+
competing_windows,
|
|
755
791
|
) -> None:
|
|
756
792
|
"""In-place swap of an ACTIVE heuristic block to its API-anchored
|
|
757
793
|
canonical window — timestamps AND token/cost totals.
|
|
@@ -827,15 +863,27 @@ def _maybe_swap_active_block_to_canonical(
|
|
|
827
863
|
# heuristic anchor reflects real ongoing activity in a later window.
|
|
828
864
|
if block_end_utc <= now.astimezone(dt.timezone.utc):
|
|
829
865
|
return
|
|
830
|
-
# Re-aggregate entries
|
|
866
|
+
# Re-aggregate the entries this canonical window OWNS. Build a fresh
|
|
831
867
|
# Block via ``_build_activity_block`` so every total stays in one code
|
|
832
868
|
# path — no field-by-field assignment that could drift if the dataclass
|
|
833
869
|
# grows new fields. Thread the caller's ``mode`` so the active block's
|
|
834
870
|
# cost honors --mode like the main grouping (Session C / Codex F1).
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
871
|
+
#
|
|
872
|
+
# #751a: selection is `partition_entries_by_owner` over the competing
|
|
873
|
+
# windows the caller grouped with, not a `block_start <= ts < block_end`
|
|
874
|
+
# re-filter. After a reset shift the canonical interval overlaps its
|
|
875
|
+
# predecessor's, and the re-filter re-priced the predecessor's entries
|
|
876
|
+
# here even though grouping had already assigned them.
|
|
877
|
+
target = _c.OwnedWindow(
|
|
878
|
+
key=key, start=block_start_utc, reset=block_end_utc,
|
|
879
|
+
)
|
|
880
|
+
ownership = [target] + [
|
|
881
|
+
w for w in competing_windows
|
|
882
|
+
if w.key != target.key
|
|
883
|
+
and (w.start, w.reset) != (target.start, target.reset)
|
|
838
884
|
]
|
|
885
|
+
owned, _unowned = _c.partition_entries_by_owner(all_entries, ownership)
|
|
886
|
+
canonical_entries = owned[target.key]
|
|
839
887
|
rebuilt = _c._build_activity_block(
|
|
840
888
|
canonical_entries,
|
|
841
889
|
block_start_utc,
|
|
@@ -925,36 +973,1160 @@ def _latest_seven_day_and_window(
|
|
|
925
973
|
"""Return ``(latest_7d_percent, latest_5h_window_key)`` from
|
|
926
974
|
``weekly_usage_snapshots``.
|
|
927
975
|
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
``
|
|
932
|
-
|
|
976
|
+
TWO SELECTIONS OVER ONE ROW SET, and they differ deliberately (#834 S1,
|
|
977
|
+
#835). Either or both elements may be ``None``. Used by
|
|
978
|
+
``cmd_five_hour_blocks`` and ``cmd_five_hour_breakdown`` to override
|
|
979
|
+
``seven_day_pct_at_block_end`` on the active row, and by ``_block_is_active``
|
|
980
|
+
to decide which block is current.
|
|
981
|
+
|
|
982
|
+
**The five-hour window key is HELD-INCLUSIVE**, decided rather than
|
|
983
|
+
overlooked (#769 S11, #824). A `weekly_observation_held` row's five-hour
|
|
984
|
+
fields are the writing tick's own, so it is the freshest statement of which
|
|
985
|
+
window is current; excluding it would resolve an older window and report the
|
|
986
|
+
live block closed. Rows with no ``five_hour_window_key`` are included here
|
|
987
|
+
too, unchanged, because the latest row's key is the answer even when it is
|
|
988
|
+
NULL.
|
|
989
|
+
|
|
990
|
+
**The weekly value is CREDIT-AWARE**, which is the #835 change. It comes from
|
|
991
|
+
the most recent row that is not a post-floor stale replica — the same band
|
|
992
|
+
and floor predicate the stale-replica DELETE applies, applied as a read
|
|
993
|
+
filter. Held-inclusive is still right in principle here, because a held row
|
|
994
|
+
stores the EFFECTIVE weekly value and that is what a block's weekly end
|
|
995
|
+
means. What is not right is reporting a value a credit RETIRED, and #835
|
|
996
|
+
made that reachable by preserving held rows: before it, the DELETE removed
|
|
997
|
+
every post-floor replica, so this read could not see one. Excluding held rows
|
|
998
|
+
outright would be the wrong repair — it would report the block end from an
|
|
999
|
+
older tick, which is the mistake #824's comment here warned against.
|
|
1000
|
+
|
|
1001
|
+
**The weekly walk is BOUNDED three ways, and each bound closes a hole tranche
|
|
1002
|
+
A left open (#834 S1 Gate A R1).**
|
|
1003
|
+
|
|
1004
|
+
*Scope.* The walk descends only within the NEWEST row's own
|
|
1005
|
+
``(week_start_date, account_key)``. A candidate belonging to another week or
|
|
1006
|
+
another account ends the walk with ``None``, because week W-1's percentage is
|
|
1007
|
+
not week W's weekly end and account B's is not account A's, under any reading.
|
|
1008
|
+
Before #835 this read took exactly one row, so the absent predicates could not
|
|
1009
|
+
matter; a walk makes them matter.
|
|
1010
|
+
|
|
1011
|
+
*Floor.* The walk also ends with ``None`` at the first row captured BEFORE the
|
|
1012
|
+
latest credit floor in that scope. This is the soundness bound: a pre-floor row
|
|
1013
|
+
is never classified as retired, so without it the walk terminated on one and
|
|
1014
|
+
returned its value — and that value is by construction the PRE-credit reading,
|
|
1015
|
+
which is exactly the value a retirement band exists to suppress. A sub-one-point
|
|
1016
|
+
credit reaches it immediately, because such a credit is legal
|
|
1017
|
+
(`_doomed_snapshot_rows`) and places the post-credit synthetic inside a band
|
|
1018
|
+
centred on ``from_pct``, leaving every post-floor row in the band. An
|
|
1019
|
+
unparseable capture stamp inside a credited week is treated as pre-floor for
|
|
1020
|
+
the same reason: the row cannot be placed relative to the floor, so it cannot
|
|
1021
|
+
be certified as able to state the current value.
|
|
1022
|
+
|
|
1023
|
+
*Retirement.* Within the scope and at or after the floor, a retired replica is
|
|
1024
|
+
skipped, and a retired value is therefore never returned.
|
|
1025
|
+
|
|
1026
|
+
The floor bound applies to the NEWEST row as well as to the candidates below
|
|
1027
|
+
it, so a newest row captured before the latest floor answers ``None`` where the
|
|
1028
|
+
pre-#835 code answered its value. That state is unreachable through either
|
|
1029
|
+
production credit path: `record-credit` writes a post-credit synthetic snapshot
|
|
1030
|
+
at its own effective instant, and the automatic path's floor is derived from a
|
|
1031
|
+
capture that is itself in the store, so in both cases at least one row exists at
|
|
1032
|
+
or after the floor and it is the newest.
|
|
1033
|
+
|
|
1034
|
+
An UNCREDITED week resolves no bands, so there is no floor, nothing is skipped,
|
|
1035
|
+
and the read returns the newest row's value exactly as it did before #835. A
|
|
1036
|
+
``NULL`` weekly value is not a retired replica, so it still terminates the walk
|
|
1037
|
+
and still answers ``None``. Both contracts are byte-stable against today.
|
|
1038
|
+
|
|
1039
|
+
**The residual, deliberately unfixed and filed as #840.** A genuine later climb
|
|
1040
|
+
back INTO the band is still classified as a replay, so the weekly end reads the
|
|
1041
|
+
newest out-of-band value until the true value passes the band's upper edge.
|
|
1042
|
+
That is preferred over returning a value a credit retired, and no sound
|
|
1043
|
+
discriminator is available: `record-credit`'s own synthetic post-credit
|
|
1044
|
+
snapshot is written before any later replay, so "a below-band row has been
|
|
1045
|
+
observed" would close the band immediately and let every replay through.
|
|
1046
|
+
`test_840_characterization_a_genuine_climb_back_reads_the_older_value`
|
|
1047
|
+
characterizes it.
|
|
933
1048
|
|
|
934
1049
|
``account_key`` (#341, spec §3): ``None`` = merged / byte-stable; a real key /
|
|
935
|
-
``unattributed`` scopes
|
|
1050
|
+
``unattributed`` scopes both selections to that account. The retirement bands
|
|
1051
|
+
are resolved against the NEWEST ROW's own account, which is also the walk's
|
|
1052
|
+
scope, because a credit under one account retires nothing under another.
|
|
936
1053
|
"""
|
|
1054
|
+
_c = _cctally()
|
|
937
1055
|
acct_pred = "" if account_key is None else " WHERE account_key = ?"
|
|
938
1056
|
acct_p: tuple = () if account_key is None else (account_key,)
|
|
939
1057
|
try:
|
|
940
|
-
|
|
941
|
-
"SELECT weekly_percent, five_hour_window_key "
|
|
1058
|
+
cur = conn.execute(
|
|
1059
|
+
"SELECT weekly_percent, five_hour_window_key, captured_at_utc, "
|
|
1060
|
+
" week_start_date, week_start_at, week_end_at, account_key "
|
|
942
1061
|
"FROM weekly_usage_snapshots" + acct_pred +
|
|
943
|
-
" ORDER BY captured_at_utc DESC, id DESC
|
|
1062
|
+
" ORDER BY captured_at_utc DESC, id DESC",
|
|
944
1063
|
acct_p,
|
|
945
|
-
)
|
|
1064
|
+
)
|
|
1065
|
+
row = cur.fetchone()
|
|
1066
|
+
if row is None:
|
|
1067
|
+
return None, None
|
|
1068
|
+
# Axis 1: the five-hour window key, from the latest row, held-inclusive.
|
|
1069
|
+
key = row["five_hour_window_key"]
|
|
1070
|
+
# Axis 2: the weekly value. The scope, the bands and the floor all come
|
|
1071
|
+
# from the NEWEST row and are resolved ONCE, because the walk may not
|
|
1072
|
+
# leave that scope — so there is nothing to memoize per candidate.
|
|
1073
|
+
#
|
|
1074
|
+
# Dropping `LIMIT 1` does cost something at the storage layer: SQLite
|
|
1075
|
+
# materializes the full sort before yielding the first row, measured at
|
|
1076
|
+
# 9.0ms against 3.4ms on a 25,053-row store. The cost is immaterial
|
|
1077
|
+
# because this runs once per invocation, but the earlier claim that the
|
|
1078
|
+
# ordinary case "reads exactly one row, which is the work the single-row
|
|
1079
|
+
# query did" described only the rows the cursor yields, not the work the
|
|
1080
|
+
# query does, and was wrong.
|
|
1081
|
+
scope = (row["week_start_date"], row["account_key"])
|
|
1082
|
+
bands = _c._credit_retirement_bands(
|
|
1083
|
+
conn, week_start_date=row["week_start_date"],
|
|
1084
|
+
week_start_at=row["week_start_at"],
|
|
1085
|
+
week_end_at=row["week_end_at"],
|
|
1086
|
+
account_key=row["account_key"])
|
|
1087
|
+
floor = _c._latest_credit_floor_instant(bands)
|
|
1088
|
+
pct = None
|
|
1089
|
+
while row is not None:
|
|
1090
|
+
if (row["week_start_date"], row["account_key"]) != scope:
|
|
1091
|
+
break
|
|
1092
|
+
if floor is not None:
|
|
1093
|
+
try:
|
|
1094
|
+
captured = parse_iso_datetime(
|
|
1095
|
+
row["captured_at_utc"], "captured_at").timestamp()
|
|
1096
|
+
except (ValueError, TypeError):
|
|
1097
|
+
captured = None
|
|
1098
|
+
if captured is None or captured < floor:
|
|
1099
|
+
break
|
|
1100
|
+
if not _c._weekly_value_is_retired_replica(
|
|
1101
|
+
bands,
|
|
1102
|
+
captured_at_utc=row["captured_at_utc"],
|
|
1103
|
+
weekly_percent=row["weekly_percent"]):
|
|
1104
|
+
pct = row["weekly_percent"]
|
|
1105
|
+
break
|
|
1106
|
+
row = cur.fetchone()
|
|
946
1107
|
except sqlite3.DatabaseError:
|
|
947
1108
|
return None, None
|
|
948
|
-
if row is None:
|
|
949
|
-
return None, None
|
|
950
|
-
pct = row[0]
|
|
951
|
-
key = row[1]
|
|
952
1109
|
return (
|
|
953
1110
|
float(pct) if pct is not None else None,
|
|
954
1111
|
int(key) if key is not None else None,
|
|
955
1112
|
)
|
|
956
1113
|
|
|
957
1114
|
|
|
1115
|
+
def _parsed_block_instant(block, column):
|
|
1116
|
+
"""``block[column]`` when it is present and parses as an instant, else ``None``.
|
|
1117
|
+
|
|
1118
|
+
#834 S1 (#835) Gate A R8. Both observation stamps are declared `TEXT NOT NULL`,
|
|
1119
|
+
so absence is not a live shape — but a fixture, a hand-built store or a legacy
|
|
1120
|
+
projection can still carry an unparseable one, and the two instant helpers
|
|
1121
|
+
below must degrade rather than raise.
|
|
1122
|
+
"""
|
|
1123
|
+
value = block.get(column)
|
|
1124
|
+
if not value:
|
|
1125
|
+
return None
|
|
1126
|
+
try:
|
|
1127
|
+
parse_iso_datetime(value, column)
|
|
1128
|
+
except (ValueError, TypeError):
|
|
1129
|
+
return None
|
|
1130
|
+
return value
|
|
1131
|
+
|
|
1132
|
+
|
|
1133
|
+
def _block_weekly_start_instant(block):
|
|
1134
|
+
"""The instant a block's stored weekly START was captured at, as ISO text.
|
|
1135
|
+
|
|
1136
|
+
#834 S1 (#835) Gate A R8. `first_observed_at_utc`, because that is the stamp
|
|
1137
|
+
of the tick that wrote `seven_day_pct_at_block_start`. The live upsert in
|
|
1138
|
+
bin/_cctally_record.py sets both from one fold dict on the INSERT (both are
|
|
1139
|
+
omitted from its `DO UPDATE`, so the first writer owns the pair), and
|
|
1140
|
+
`_backfill_five_hour_blocks` takes `first_obs` and `pct_start_7d` from its
|
|
1141
|
+
single MIN-captured snapshot row.
|
|
1142
|
+
|
|
1143
|
+
NOT `block_start_at`, which is the window's NOMINAL start — `five_hour_resets_at`
|
|
1144
|
+
minus five hours — and therefore at or before the first tick. The gap is not
|
|
1145
|
+
cosmetic: a block whose nominal start precedes a credit floor but whose first
|
|
1146
|
+
tick landed after it would have the floor's retired value compared against a
|
|
1147
|
+
PRE-floor instant and rendered. It falls back to `block_start_at` only when the
|
|
1148
|
+
observation stamp is absent or unparseable, which keeps the pre-R8 disposition
|
|
1149
|
+
rather than blanking the axis.
|
|
1150
|
+
"""
|
|
1151
|
+
return (_parsed_block_instant(block, "first_observed_at_utc")
|
|
1152
|
+
or block.get("block_start_at"))
|
|
1153
|
+
|
|
1154
|
+
|
|
1155
|
+
def _block_weekly_end_instant(block, *, now_utc):
|
|
1156
|
+
"""The instant a block's stored weekly END was captured at, as an ISO text.
|
|
1157
|
+
|
|
1158
|
+
#834 S1 (#835) Gate A R8. `last_observed_at_utc`, because that is the stamp of
|
|
1159
|
+
the tick that wrote `seven_day_pct_at_block_end`. All four writers of the pair
|
|
1160
|
+
move the two together: the live upsert assigns
|
|
1161
|
+
`last_observed_at_utc = excluded.last_observed_at_utc` and
|
|
1162
|
+
`seven_day_pct_at_block_end = excluded.seven_day_pct_at_block_end` in one
|
|
1163
|
+
`DO UPDATE SET` from one fold dict, `_backfill_five_hour_blocks` takes
|
|
1164
|
+
`last_obs` and `pct_end_7d` from its single MAX-captured snapshot row,
|
|
1165
|
+
`five_hour_block_close` freezes the whole row into one evt, and
|
|
1166
|
+
`_migration_merge_5h_block_duplicates_v1` (bin/_cctally_db.py) copies both
|
|
1167
|
+
from the group row whose `last_observed_at_utc` is MAX.
|
|
1168
|
+
|
|
1169
|
+
Gate A S1 used `five_hour_resets_at` instead, falling back to the current
|
|
1170
|
+
instant while that reset was still in the future. That is the block's END, not
|
|
1171
|
+
a capture instant, and it was wrong in BOTH directions. A crossed-reset block's
|
|
1172
|
+
reset falls in the SUCCESSOR week, so `_block_weekly_axis_weeks` resolved the
|
|
1173
|
+
successor's bounds and the credited week's bands were never consulted. A closed
|
|
1174
|
+
block whose last tick predates the floor had its genuinely effective end
|
|
1175
|
+
withheld, which contradicts the historical-truth rule this filter rests on.
|
|
1176
|
+
|
|
1177
|
+
The reset-or-now form is KEPT as the fallback for an absent or unparseable
|
|
1178
|
+
stamp, so a store that cannot supply a capture instant behaves as it did
|
|
1179
|
+
before R8 rather than losing the axis.
|
|
1180
|
+
"""
|
|
1181
|
+
_c = _cctally()
|
|
1182
|
+
observed = _parsed_block_instant(block, "last_observed_at_utc")
|
|
1183
|
+
if observed is not None:
|
|
1184
|
+
return observed
|
|
1185
|
+
now_iso = _c._iso_z(now_utc)
|
|
1186
|
+
resets = block.get("five_hour_resets_at")
|
|
1187
|
+
if not resets:
|
|
1188
|
+
return now_iso
|
|
1189
|
+
try:
|
|
1190
|
+
moment = parse_iso_datetime(resets, "five_hour_resets_at").timestamp()
|
|
1191
|
+
except (ValueError, TypeError):
|
|
1192
|
+
return now_iso
|
|
1193
|
+
return resets if moment <= now_utc.timestamp() else now_iso
|
|
1194
|
+
|
|
1195
|
+
|
|
1196
|
+
#: One candidate week for `_block_weekly_axis_weeks`, as loaded once per account
|
|
1197
|
+
#: per render. ``lo`` / ``hi`` are the parsed `[week_start_at, week_end_at)`
|
|
1198
|
+
#: epochs, or ``None`` when the week carries no usable bounds.
|
|
1199
|
+
_AxisWeek = collections.namedtuple(
|
|
1200
|
+
"_AxisWeek",
|
|
1201
|
+
"week_start_date week_end_date week_start_at week_end_at lo hi")
|
|
1202
|
+
|
|
1203
|
+
|
|
1204
|
+
def _parsed_week_bounds(week_start_at, week_end_at):
|
|
1205
|
+
"""``(lo, hi)`` epochs for a week's bounds, or ``(None, None)``.
|
|
1206
|
+
|
|
1207
|
+
#834 S1 (#835) Gate A R9. Both loaders need the same verdict on the same two
|
|
1208
|
+
nullable columns — `_account_axis_weeks` to decide whether a week is placed by
|
|
1209
|
+
epoch or by date, `_account_axis_capture_weeks` to decide whether the identity
|
|
1210
|
+
it returns carries bounds for the `week_reset_events` leg — so the parse lives
|
|
1211
|
+
in one place and the two cannot disagree about which weeks have usable bounds.
|
|
1212
|
+
Absent or unparseable on either side yields no bounds, which is what
|
|
1213
|
+
`_credit_retirement_bands` already treats as a leg it cannot scope.
|
|
1214
|
+
"""
|
|
1215
|
+
if week_start_at is None or week_end_at is None:
|
|
1216
|
+
return None, None
|
|
1217
|
+
try:
|
|
1218
|
+
return (parse_iso_datetime(week_start_at, "week_start_at").timestamp(),
|
|
1219
|
+
parse_iso_datetime(week_end_at, "week_end_at").timestamp())
|
|
1220
|
+
except (ValueError, TypeError):
|
|
1221
|
+
return None, None
|
|
1222
|
+
|
|
1223
|
+
|
|
1224
|
+
def _account_axis_weeks(conn, *, account_key, cache):
|
|
1225
|
+
"""Every distinct week ``account_key``'s snapshot rows name, in no order.
|
|
1226
|
+
|
|
1227
|
+
#834 S1 (#835) Gate A R8. ONE query per account per render, replacing one query
|
|
1228
|
+
per axis instant. The per-instant form was measured at 22x the whole render's
|
|
1229
|
+
prior cost on a 25,000-row store with 50 blocks, because its
|
|
1230
|
+
``unixepoch(week_start_at)`` predicate makes `idx_usage_week_start_at_time`
|
|
1231
|
+
unusable — `EXPLAIN QUERY PLAN` reported `SCAN weekly_usage_snapshots` plus
|
|
1232
|
+
`USE TEMP B-TREE FOR ORDER BY`, and the 50-block render performed about 51 full
|
|
1233
|
+
scans and 51 full sorts. An instant-keyed memo could not collapse them: two
|
|
1234
|
+
instants per block are two distinct keys.
|
|
1235
|
+
|
|
1236
|
+
THE WIN IS THE COLLAPSE TO ONE QUERY, NOT RESTORED INDEX USE. This query is
|
|
1237
|
+
equally unindexed: `EXPLAIN QUERY PLAN` reports `SCAN weekly_usage_snapshots`
|
|
1238
|
+
plus a temp B-tree. About 51 unindexed scans became one unindexed scan, which is
|
|
1239
|
+
the whole of the improvement; do not read `idx_usage_week_start_at_time` as being
|
|
1240
|
+
back in play. The scan verdict was measured on an empty store, on a 25,500-row
|
|
1241
|
+
store and again after `ANALYZE` while this query was still a `GROUP BY`; R10
|
|
1242
|
+
replaced that clause with `SELECT DISTINCT`, which changed the B-tree's stated
|
|
1243
|
+
purpose and not the scan (see the R10 paragraph below for the measured plan).
|
|
1244
|
+
|
|
1245
|
+
THE RESIDUAL IS NOT IRREDUCIBLE, and it is accepted as shipped rather than
|
|
1246
|
+
defended as minimal. `idx_usage_week_time` leads on `week_start_date`, so
|
|
1247
|
+
narrowing the projection to that column alone does use it — `SELECT DISTINCT
|
|
1248
|
+
week_start_date` reports `SCAN weekly_usage_snapshots USING INDEX
|
|
1249
|
+
idx_usage_week_time`, measured on the same two stores, and an earlier form said
|
|
1250
|
+
"grouping on that column alone", which is stale terminology since R10 replaced
|
|
1251
|
+
the `GROUP BY` with `SELECT DISTINCT` — and a render only
|
|
1252
|
+
needs the weeks overlapping its blocks' instant range rather than the account's
|
|
1253
|
+
whole history. Both are real reductions and neither is taken, because the
|
|
1254
|
+
property that matters is that the cost is CONSTANT in the number of rendered
|
|
1255
|
+
blocks, which it already is. `week_end_date` and the two bounds are in the
|
|
1256
|
+
projection because the date pass needs all four, which is exactly what a
|
|
1257
|
+
`week_start_date`-only projection would give up.
|
|
1258
|
+
|
|
1259
|
+
ONE COMPARISON SEMANTIC CHANGED WITH THE FORM, unreachably. The per-instant
|
|
1260
|
+
predicate compared SQLite `unixepoch()` integers, which truncate a fractional
|
|
1261
|
+
second; this form compares `parse_iso_datetime(...).timestamp()` floats, which
|
|
1262
|
+
keep it. The two disagree for a week bound carrying a fractional second: a
|
|
1263
|
+
`week_start_at` of `…T00:00:00.500000+00:00` against a probe at
|
|
1264
|
+
`…T00:00:00+00:00` is INSIDE the week under the integer compare and outside it
|
|
1265
|
+
under the float one. No writer can produce that bound.
|
|
1266
|
+
`_canonicalize_optional_iso` returns `isoformat(timespec="seconds")` over a
|
|
1267
|
+
`_normalize_week_boundary_dt` result, which zeroes second and microsecond and
|
|
1268
|
+
snaps to the hour, and `cmd_record_usage`'s pipeline derives both bounds from
|
|
1269
|
+
that same normalizer and writes them with the same `timespec="seconds"`.
|
|
1270
|
+
|
|
1271
|
+
THE SET IS UNORDERED, AND THE QUERY IS A PLAIN `SELECT DISTINCT` (Gate A R10).
|
|
1272
|
+
Through R9 it carried a `MAX(captured_at_utc || char(1) || printf('%020d', id))`
|
|
1273
|
+
aggregate whose SQL alias was `rk` and whose `_AxisWeek` field was `rank` — call
|
|
1274
|
+
it `rk`, because that is the alias, and earlier text here called the column
|
|
1275
|
+
`rank` — introduced to reproduce the per-instant query's
|
|
1276
|
+
``ORDER BY captured_at_utc DESC, id DESC`` tie-break exactly so that which week
|
|
1277
|
+
won an overlap did not change with the query's form, and kept as documentary
|
|
1278
|
+
after R9 deleted the tie-break. Nothing read it: this function has one caller,
|
|
1279
|
+
every pass in `_block_weekly_axis_weeks` builds an order-insensitive set, and
|
|
1280
|
+
the caller ORs a predicate over the result. The grouping was there only to carry
|
|
1281
|
+
the aggregate, so both went.
|
|
1282
|
+
|
|
1283
|
+
NOTHING PINS THE ORDER NOW. There was never a SQL `ORDER BY` on this query: the
|
|
1284
|
+
order came from a Python `weeks.sort(key=lambda w: w.rank, reverse=True)` over the
|
|
1285
|
+
aggregate. Through R8 a consumer tie-break in the epoch pass read it; R9 deleted
|
|
1286
|
+
that tie-break and left the aggregate documentary; R10 deleted the aggregate and
|
|
1287
|
+
the sort together. So the returned order is now whatever SQLite produces, and no
|
|
1288
|
+
test asserts order-insensitivity of the consumer, so a future consumer that cares
|
|
1289
|
+
about order has nothing to fail against. It is safe today because the one consumer
|
|
1290
|
+
ORs a monotone band predicate over the whole set. A consumer that needs an order
|
|
1291
|
+
must establish it itself rather than assume this one.
|
|
1292
|
+
|
|
1293
|
+
The statement-count test cannot see R10's REMOVAL of the aggregate and the
|
|
1294
|
+
grouping, because the statement count is unchanged by it; the removal was
|
|
1295
|
+
measured instead, best of five on a 25,194-row 13-week 50-block store, with the
|
|
1296
|
+
order of the three forms reversed to rule out page-cache bias and the three
|
|
1297
|
+
result sets asserted identical: 13.57 ms with the aggregate, 9.36 ms for the
|
|
1298
|
+
bare `GROUP BY`, 2.34 ms for `SELECT DISTINCT` — so removing both took about 83%
|
|
1299
|
+
off the statement, and the plan changed from `SCAN weekly_usage_snapshots` +
|
|
1300
|
+
`USE TEMP B-TREE FOR GROUP BY` to the same scan + `USE TEMP B-TREE FOR DISTINCT`.
|
|
1301
|
+
It is still an unindexed scan; the `idx_usage_week_time` reduction named above is
|
|
1302
|
+
still not taken, for the reason given there.
|
|
1303
|
+
|
|
1304
|
+
DO NOT QUOTE A SPLIT BETWEEN THE TWO REMOVALS. On those figures the aggregate is
|
|
1305
|
+
31% of the statement and the grouping a further 52%, but the review that checked
|
|
1306
|
+
this measured 12.87-13.09 / 6.30-6.77 / 2.02-2.23 ms on its own 25,194-row store,
|
|
1307
|
+
which makes the aggregate about 50% and the grouping about 34% — the same 83%
|
|
1308
|
+
total, with the two components in the opposite order. The total reproduces and
|
|
1309
|
+
the ranking does not, so treat the split as machine- and data-dependent.
|
|
1310
|
+
|
|
1311
|
+
`week_start_at` / `week_end_at` are both nullable and
|
|
1312
|
+
`_derive_week_from_payload` leaves them `None` on three of its four paths, so
|
|
1313
|
+
a week with no bounds is a live shape; it is retained here with ``lo``/``hi``
|
|
1314
|
+
of ``None`` and placed by date instead (see `_block_weekly_axis_weeks`).
|
|
1315
|
+
"""
|
|
1316
|
+
if account_key in cache:
|
|
1317
|
+
return cache[account_key]
|
|
1318
|
+
try:
|
|
1319
|
+
rows = conn.execute(
|
|
1320
|
+
"SELECT DISTINCT week_start_date, week_end_date, week_start_at, "
|
|
1321
|
+
" week_end_at "
|
|
1322
|
+
" FROM weekly_usage_snapshots "
|
|
1323
|
+
" WHERE account_key = ?",
|
|
1324
|
+
(account_key,),
|
|
1325
|
+
).fetchall()
|
|
1326
|
+
except sqlite3.DatabaseError:
|
|
1327
|
+
rows = []
|
|
1328
|
+
weeks: list = []
|
|
1329
|
+
for r in rows:
|
|
1330
|
+
lo, hi = _parsed_week_bounds(r["week_start_at"], r["week_end_at"])
|
|
1331
|
+
weeks.append(_AxisWeek(
|
|
1332
|
+
week_start_date=r["week_start_date"],
|
|
1333
|
+
week_end_date=r["week_end_date"],
|
|
1334
|
+
week_start_at=r["week_start_at"] if lo is not None else None,
|
|
1335
|
+
week_end_at=r["week_end_at"] if hi is not None else None,
|
|
1336
|
+
lo=lo, hi=hi))
|
|
1337
|
+
cache[account_key] = weeks
|
|
1338
|
+
return weeks
|
|
1339
|
+
|
|
1340
|
+
|
|
1341
|
+
def _account_axis_capture_weeks(conn, *, account_key, cache):
|
|
1342
|
+
"""``{capture second -> the week identities a snapshot row at that instant
|
|
1343
|
+
names}``, for the capture instants ``account_key``'s blocks actually carry.
|
|
1344
|
+
|
|
1345
|
+
#834 S1 (#835) Gate A R9. THE STORE KEEPS A RECORD OF WHICH WEEK A CAPTURE WAS
|
|
1346
|
+
OBSERVED UNDER, and this is the read of it. Round five documented its
|
|
1347
|
+
over-withholding on the premise that it does not. All four writers of
|
|
1348
|
+
`five_hour_blocks.seven_day_pct_at_block_start` / `_at_block_end` take the
|
|
1349
|
+
matching axis STAMP from a `weekly_usage_snapshots` row's `captured_at_utc`:
|
|
1350
|
+
the live upsert passes `captured_at = saved["capturedAt"]` into
|
|
1351
|
+
`first_observed_at_utc` and `last_observed_at_utc` in one `INSERT … DO UPDATE`
|
|
1352
|
+
and that value IS the row's `captured_at_utc` (bin/_cctally_record.py:2351,
|
|
1353
|
+
:2633-2634, :6828), `_backfill_five_hour_blocks` takes them from its MIN- and
|
|
1354
|
+
MAX-captured rows, `five_hour_block_close` freezes the row, and
|
|
1355
|
+
`_migration_merge_5h_block_duplicates_v1` copies from the MAX-captured group
|
|
1356
|
+
row. Every snapshot row carries all four week columns. So a row whose
|
|
1357
|
+
`captured_at_utc` equals an axis instant NAMES the week in force at that
|
|
1358
|
+
capture, with no date-range guess involved.
|
|
1359
|
+
|
|
1360
|
+
HELD ROWS ARE NOT EXCLUDED, which is the one judgement call in this function.
|
|
1361
|
+
The rest of this tranche excludes `weekly_observation_held = 1` from weekly
|
|
1362
|
+
VALUE reads, because a held row carries a reading no fresh observation
|
|
1363
|
+
confirmed. This function reads no value: it reads which week the value on the
|
|
1364
|
+
block's axis came from, and on a held tick that is the BASIS week by
|
|
1365
|
+
construction — the held write copies the basis's four week columns verbatim
|
|
1366
|
+
while keeping the tick's own `capture_at` (bin/_cctally_record.py:7190-7203),
|
|
1367
|
+
and the block's axis takes `fold.weekly.effective_pct`, the same
|
|
1368
|
+
carried-forward value, stamped with that same `capture_at`. The held row is
|
|
1369
|
+
therefore the provenance record of the number on the axis, and the credits
|
|
1370
|
+
that retired that number are the basis week's.
|
|
1371
|
+
|
|
1372
|
+
Two consequences follow. A held row is exactly how a pre-credit weekly value
|
|
1373
|
+
reaches a post-credit capture instant, which is the case this whole tranche
|
|
1374
|
+
exists to handle, and excluding it would drop the only exact evidence of which
|
|
1375
|
+
week's floor applies. And because no predicate names
|
|
1376
|
+
`weekly_observation_held`, a store that stops short of epoch 1015 is served
|
|
1377
|
+
unchanged and `weekly_held_exclusion` is not needed here. Pinned by
|
|
1378
|
+
`tests/test_five_hour_breakdown.py::test_835_r9_a_held_capture_record_names_the_basis_week_it_carried`
|
|
1379
|
+
and `::test_835_r9_a_pre_column_store_resolves_the_capture_record`.
|
|
1380
|
+
|
|
1381
|
+
THE SET IS BOUNDED BY THE ACCOUNT'S BLOCK COUNT, NOT BY ITS SNAPSHOT HISTORY,
|
|
1382
|
+
and the form was chosen on measurement rather than on shape. Re-measured at Gate
|
|
1383
|
+
A R10, because R10 replaced `_account_axis_weeks`' grouped query with a plain
|
|
1384
|
+
`SELECT DISTINCT` and the earlier round's figures for that statement no longer
|
|
1385
|
+
described what ships. All three were timed in ONE session on a 25,194-row,
|
|
1386
|
+
13-week, 50-block store — the store `_account_axis_weeks` documents its own
|
|
1387
|
+
numbers against — best of five each, with the order reversed to rule out
|
|
1388
|
+
page-cache bias:
|
|
1389
|
+
|
|
1390
|
+
- this narrowed query: 2.10-2.27 ms, 50 rows, `SCAN weekly_usage_snapshots`
|
|
1391
|
+
plus a `LIST SUBQUERY` that SEARCHes `five_hour_blocks` on its own index;
|
|
1392
|
+
- `_account_axis_weeks`' `SELECT DISTINCT`: 2.06-2.13 ms, 13 rows;
|
|
1393
|
+
- one ungrouped full-row scan serving BOTH passes from a single statement:
|
|
1394
|
+
14.18-14.38 ms, 25,194 rows, a bare `SCAN weekly_usage_snapshots`.
|
|
1395
|
+
|
|
1396
|
+
So the two statements together cost 4.40 ms against the single statement's
|
|
1397
|
+
14.18 ms and retain 63 rows against 25,194 — a 400th. Two statements per account
|
|
1398
|
+
per render is the cheaper form on both axes, and it keeps the property that
|
|
1399
|
+
matters: the cost is CONSTANT in the number of rendered blocks. Bounded by the
|
|
1400
|
+
account's own `five_hour_blocks` rows rather than by which of them this render
|
|
1401
|
+
shows, which is why no instant range has to be threaded down from the caller.
|
|
1402
|
+
|
|
1403
|
+
TWO EARLIER ROUNDS DISAGREED ABOUT THE SINGLE-STATEMENT FIGURE — this docstring
|
|
1404
|
+
said 22.03 ms and `docs/five-hour-gotchas.md` said 21.97 ms for the same query on
|
|
1405
|
+
the same store — and neither number survives, because both were taken against the
|
|
1406
|
+
superseded grouped form of the week query. The figures above replace both rather
|
|
1407
|
+
than reconciling them.
|
|
1408
|
+
|
|
1409
|
+
THE NARROWING AND THE MATCH USE DIFFERENT READERS, deliberately. SQLite's
|
|
1410
|
+
`unixepoch()` selects the rows; `parse_iso_datetime(...).timestamp()` keys
|
|
1411
|
+
them and keys the probe, so the match itself is Python's and is consistent
|
|
1412
|
+
between map and probe. Both readers truncate to the second, which is how two
|
|
1413
|
+
ISO spellings of one moment — `Z` against `+00:00` — compare equal.
|
|
1414
|
+
|
|
1415
|
+
A NAIVE STAMP IS A LIVE SHAPE, and the two readings of it differ. An earlier
|
|
1416
|
+
form of this paragraph said no writer produces one, citing `now_utc_iso()`.
|
|
1417
|
+
That is wrong: `_coerce_payload_captured_at` (bin/_cctally_record.py:6576-6585)
|
|
1418
|
+
returns the PAYLOAD's `capturedAt` verbatim whenever `parse_iso_datetime`
|
|
1419
|
+
accepts it, `parse_iso_datetime` accepts a naive string and reads it host-local,
|
|
1420
|
+
and `insert_usage_snapshot` then stores that text. Driven with
|
|
1421
|
+
`TZ=America/Los_Angeles`: a payload `capturedAt` of `2026-04-30T13:00:00` is
|
|
1422
|
+
stored as `captured_at_utc = '2026-04-30T13:00:00'`, and `unixepoch()` reads it
|
|
1423
|
+
1777554000 against Python's 1777579200.
|
|
1424
|
+
|
|
1425
|
+
THE FAILURE IS A MIXED PAIR, NOT A NAIVE STAMP, AND IT IS THE MATCH THAT MISSES
|
|
1426
|
+
RATHER THAN THE NARROWING THAT EXCLUDES. The same earlier paragraph stated it
|
|
1427
|
+
the other way round. Each reader is self-consistent across the two sides it
|
|
1428
|
+
compares, so the spelling only matters when the snapshot stamp and the block
|
|
1429
|
+
stamp are spelled DIFFERENTLY. All four pairings, driven on that same non-UTC
|
|
1430
|
+
host: equal naive text narrows IN and the Python match HITS; `Z` against
|
|
1431
|
+
`+00:00` narrows IN and HITS; naive against offset-carrying, in either
|
|
1432
|
+
direction, narrows IN and the PYTHON MATCH MISSES, after which the instant falls
|
|
1433
|
+
through to the two fallback passes.
|
|
1434
|
+
|
|
1435
|
+
SO THE RESIDUAL IS NARROWER THAN A NAIVE STAMP'S EXISTENCE. Every one of the
|
|
1436
|
+
four axis writers takes the block stamp from a `weekly_usage_snapshots` row's
|
|
1437
|
+
own `captured_at_utc` text — the live upsert from `saved["capturedAt"]`, the
|
|
1438
|
+
backfill from its MIN- and MAX-captured rows, the close from the frozen row, the
|
|
1439
|
+
merge migration from the MAX-captured group row — so the pair the capture pass
|
|
1440
|
+
compares always shares a spelling and the pass is correct on a naive store. A
|
|
1441
|
+
mixed pair needs the two texts to diverge after both were written, which is a
|
|
1442
|
+
hand-built or externally-edited store. The degradation is then a fallback rather
|
|
1443
|
+
than a misclassification, which is the one part of the earlier paragraph that
|
|
1444
|
+
held.
|
|
1445
|
+
"""
|
|
1446
|
+
if account_key in cache:
|
|
1447
|
+
return cache[account_key]
|
|
1448
|
+
try:
|
|
1449
|
+
rows = conn.execute(
|
|
1450
|
+
"SELECT captured_at_utc, week_start_date, week_start_at, week_end_at "
|
|
1451
|
+
" FROM weekly_usage_snapshots "
|
|
1452
|
+
" WHERE account_key = ? "
|
|
1453
|
+
" AND unixepoch(captured_at_utc) IN ("
|
|
1454
|
+
" SELECT unixepoch(first_observed_at_utc) "
|
|
1455
|
+
" FROM five_hour_blocks WHERE account_key = ? "
|
|
1456
|
+
" UNION "
|
|
1457
|
+
" SELECT unixepoch(last_observed_at_utc) "
|
|
1458
|
+
" FROM five_hour_blocks WHERE account_key = ?)",
|
|
1459
|
+
(account_key, account_key, account_key),
|
|
1460
|
+
).fetchall()
|
|
1461
|
+
except sqlite3.DatabaseError:
|
|
1462
|
+
rows = []
|
|
1463
|
+
carried: dict = {}
|
|
1464
|
+
for r in rows:
|
|
1465
|
+
try:
|
|
1466
|
+
second = int(parse_iso_datetime(
|
|
1467
|
+
r["captured_at_utc"], "captured_at_utc").timestamp())
|
|
1468
|
+
except (ValueError, TypeError):
|
|
1469
|
+
continue
|
|
1470
|
+
lo, _hi = _parsed_week_bounds(r["week_start_at"], r["week_end_at"])
|
|
1471
|
+
identity = (
|
|
1472
|
+
r["week_start_date"],
|
|
1473
|
+
r["week_start_at"] if lo is not None else None,
|
|
1474
|
+
r["week_end_at"] if lo is not None else None,
|
|
1475
|
+
)
|
|
1476
|
+
# A dict as an ordered set: two rows of one week at one instant are one
|
|
1477
|
+
# identity, and `_credit_retirement_bands` would answer both alike.
|
|
1478
|
+
carried.setdefault(second, {})[identity] = None
|
|
1479
|
+
out = {second: tuple(ids) for second, ids in carried.items()}
|
|
1480
|
+
cache[account_key] = out
|
|
1481
|
+
return out
|
|
1482
|
+
|
|
1483
|
+
|
|
1484
|
+
def _account_axis_bounds_by_start_date(weeks):
|
|
1485
|
+
"""``{week_start_date -> that week's bounds-carrying identities}``, from an
|
|
1486
|
+
already-loaded `_account_axis_weeks` set.
|
|
1487
|
+
|
|
1488
|
+
#834 S1 (#835) Gate A R10. The index the capture pass substitutes from when its
|
|
1489
|
+
matched row carries no bounds of its own. Derived from the account's OWN week
|
|
1490
|
+
set and nothing else, which is the property that separates it from
|
|
1491
|
+
`_get_canonical_boundary_for_date`; built once per account per render, so the
|
|
1492
|
+
substitution adds no query and no per-instant scan.
|
|
1493
|
+
|
|
1494
|
+
Bounds-free weeks are absent by construction: an entry exists only for a
|
|
1495
|
+
`week_start_date` some row of that account spells bounds for, which is exactly
|
|
1496
|
+
the condition under which a substitution is possible.
|
|
1497
|
+
"""
|
|
1498
|
+
out: dict = {}
|
|
1499
|
+
for week in weeks:
|
|
1500
|
+
if week.lo is None:
|
|
1501
|
+
continue
|
|
1502
|
+
out.setdefault(week.week_start_date, {})[
|
|
1503
|
+
(week.week_start_date, week.week_start_at, week.week_end_at)] = None
|
|
1504
|
+
return {date: tuple(ids) for date, ids in out.items()}
|
|
1505
|
+
|
|
1506
|
+
|
|
1507
|
+
def _block_weekly_axis_weeks(conn, *, account_key, instant_iso, cache):
|
|
1508
|
+
"""Every week identity whose credits can have retired a value captured at
|
|
1509
|
+
``instant_iso`` under ``account_key``, as ``(week_start_date, week_start_at,
|
|
1510
|
+
week_end_at)`` triples. Empty when no week names the instant.
|
|
1511
|
+
|
|
1512
|
+
THREE PASSES, IN THIS ORDER: the CAPTURE pass matches the instant against the
|
|
1513
|
+
`captured_at_utc` of the account's own snapshot rows and answers from the week
|
|
1514
|
+
those rows name; the EPOCH pass places the instant inside a week's parsed
|
|
1515
|
+
`[week_start_at, week_end_at)` bounds; the DATE pass asks which bounds-free
|
|
1516
|
+
weeks declare the instant's calendar day. Each runs only when the one before it
|
|
1517
|
+
resolved nothing.
|
|
1518
|
+
|
|
1519
|
+
THE CAPTURE PASS IS EXACT ABOUT WHICH WEEK, AND IT REPLACED AN ARGUMENT THAT
|
|
1520
|
+
WAS WRONG. Gate A R8 had only the other two, and round five defended the
|
|
1521
|
+
over-withholding they cause on the premise that the store keeps no record of
|
|
1522
|
+
which week a capture was observed under. It keeps exactly that record: every
|
|
1523
|
+
axis stamp on a block IS some snapshot row's `captured_at_utc`, and that row
|
|
1524
|
+
carries all four week columns — see `_account_axis_capture_weeks`, which also
|
|
1525
|
+
records why held rows are consulted rather than excluded. So for an instant a
|
|
1526
|
+
capture record carries there is no week to guess, and neither fallback runs.
|
|
1527
|
+
|
|
1528
|
+
BEING EXACT ABOUT THE WEEK IS NOT THE SAME AS RESOLVING BOTH LEGS, which an
|
|
1529
|
+
earlier form of this paragraph left unsaid while the date pass's own paragraphs
|
|
1530
|
+
below state it carefully. `week_start_at` and `week_end_at` are nullable on the
|
|
1531
|
+
matched row too, and `_credit_retirement_bands`' `week_reset_events` leg is
|
|
1532
|
+
scoped purely by containment of `effective_reset_at_utc` in
|
|
1533
|
+
``[week_start_at, week_end_at)`` — that table has no `week_start_date` column at
|
|
1534
|
+
all — so an identity carrying no bounds resolves the `weekly_credit_floors` leg
|
|
1535
|
+
and NOT the automatic one. Through R9 that degradation combined with this pass's
|
|
1536
|
+
precedence into the tranche's own defect class: a bounds-free capture row
|
|
1537
|
+
pre-empted a bounds-carrying placement of the SAME week and the value an
|
|
1538
|
+
automatic credit retired was published. R10 closes it by SUBSTITUTING the
|
|
1539
|
+
account's own bounds (next paragraph), so the per-leg behaviour now is that both
|
|
1540
|
+
legs resolve whenever any row of the named week carries parseable bounds, and
|
|
1541
|
+
the manual leg alone when none does.
|
|
1542
|
+
|
|
1543
|
+
A BOUNDS-FREE CAPTURE IDENTITY IS REPLACED BY THE ACCOUNT'S OWN BOUNDS-CARRYING
|
|
1544
|
+
ROWS OF THE SAME `week_start_date`, and the substitution is lossless. Those rows
|
|
1545
|
+
are already in memory from `_account_axis_weeks`, so no query is added. It is
|
|
1546
|
+
lossless because every substituted identity shares the bounds-free one's
|
|
1547
|
+
`week_start_date`, so each resolves a `weekly_credit_floors` leg identical to the
|
|
1548
|
+
one it replaces and ADDS the automatic leg its bounds can scope; the replaced
|
|
1549
|
+
identity therefore contributes no band the substitutes do not. Several of them
|
|
1550
|
+
are unioned, the way the epoch pass unions overlapping candidates, and that
|
|
1551
|
+
union is here FOR ROBUSTNESS rather than for a reachable shape: no writer is
|
|
1552
|
+
known to put two bound pairs under one `week_start_date`. What pins one pair per
|
|
1553
|
+
date is `_usage_snapshot_columns` (bin/_cctally_record.py:6698-6710), which
|
|
1554
|
+
overrides every later row's derived bounds with `_get_canonical_boundary_for_date`
|
|
1555
|
+
— the FIRST established pair for that date, `ORDER BY captured_at_utc ASC, id ASC
|
|
1556
|
+
LIMIT 1` — so a sub-day re-anchoring is overridden back to the first pair, and a
|
|
1557
|
+
day-scale one lands under a DIFFERENT `week_start_date` because the date derives
|
|
1558
|
+
from the payload's own week start. No code change is owed either way, because
|
|
1559
|
+
`named.update(dict.fromkeys(… or (identity,)))` already handles one through N.
|
|
1560
|
+
Two earlier forms of this sentence cited a producer and both were false. Do NOT
|
|
1561
|
+
cite the timezone fork: it produces the INVERSE shape, identical bounds under two
|
|
1562
|
+
different `week_start_date` values (see the two-identities paragraph below),
|
|
1563
|
+
which puts one identity under each date rather than two under one. Do NOT cite
|
|
1564
|
+
the boundary-shift branch of `_backfill_week_reset_events`
|
|
1565
|
+
(bin/_cctally_weekrefs.py) either: it writes only `week_reset_events` and never
|
|
1566
|
+
the snapshot bounds this substitution reads, and its reset test needs THREE
|
|
1567
|
+
signals rather than the two an earlier form named — the boundary shifted, the
|
|
1568
|
+
prior boundary was still in the future, AND `_is_reset_drop(prior_pct, cur_pct,
|
|
1569
|
+
allow_reset_to_zero=False)`, which with the zero leg off is the >=25pp drop
|
|
1570
|
+
(`bin/_cctally_weekrefs.py:625-637`).
|
|
1571
|
+
|
|
1572
|
+
CONTAINMENT OF THE INSTANT IS DELIBERATELY NOT REQUIRED OF A SUBSTITUTE, and
|
|
1573
|
+
requiring it would reopen the defect on the shape this tranche exists for. A
|
|
1574
|
+
HELD row carries the BASIS week's four columns with the tick's own capture
|
|
1575
|
+
stamp, so a held tick after the week's end names a week whose bounds do not
|
|
1576
|
+
contain its own capture instant. Filtering substitutes by containment would
|
|
1577
|
+
decline to enrich exactly there.
|
|
1578
|
+
|
|
1579
|
+
THAT GENERALITY IS WIDER THAN THE HELD-ROW ARGUMENT NEEDS, and the excess is a
|
|
1580
|
+
stated limit rather than a defended choice. The held-row shape requires only that
|
|
1581
|
+
containment of the CAPTURE instant not be required. Nothing here additionally
|
|
1582
|
+
requires a substitute's bounds to be consistent with the `week_start_date` they
|
|
1583
|
+
are filed under, so a same-date row spelling bounds that start a week early lets
|
|
1584
|
+
the `week_reset_events` leg admit a reset belonging to the PREDECESSOR week, and
|
|
1585
|
+
the value is withheld although no credit of the named week retired it. No writer
|
|
1586
|
+
produces a date/bounds inconsistency of that size, so the shape needs a
|
|
1587
|
+
hand-built or externally-edited store; the narrower rule that would close it is
|
|
1588
|
+
to require the substitute's `week_start_at` UTC date to sit within a day of
|
|
1589
|
+
`week_start_date`, and it is not taken because the reachable benefit is nil and a
|
|
1590
|
+
date/bounds tolerance is one more constant to get wrong.
|
|
1591
|
+
|
|
1592
|
+
NOTHING WIDER THAN THE ACCOUNT'S OWN ROWS IS CONSULTED.
|
|
1593
|
+
`_get_canonical_boundary_for_date` (bin/_cctally_weekrefs.py:51) is the repo's
|
|
1594
|
+
canonical-bounds reader and the obvious candidate, and it carries no
|
|
1595
|
+
`account_key` predicate: it returns the EARLIEST bounds-carrying row for a
|
|
1596
|
+
`week_start_date` across every account. On a multi-account store that does not
|
|
1597
|
+
merely attach the wrong provenance, it fails to fix the defect, because another
|
|
1598
|
+
account's re-anchored bounds cannot scope THIS account's credit — driven and
|
|
1599
|
+
pinned by
|
|
1600
|
+
`tests/test_five_hour_breakdown.py::test_835_r10_the_substitution_takes_the_accounts_own_bounds_not_another_accounts`.
|
|
1601
|
+
|
|
1602
|
+
ONE RESIDUAL SURVIVES, AND IT IS PRE-EXISTING RATHER THAN INTENDED. When NO row
|
|
1603
|
+
of the named week carries parseable bounds there is nothing account-scoped to
|
|
1604
|
+
substitute, so an automatic credit in that week is structurally invisible to
|
|
1605
|
+
this filter and the value it retired is published. Closing it needs a source of
|
|
1606
|
+
a week's bounds outside the account's own snapshot rows, and the paragraph above
|
|
1607
|
+
says why the one that exists is not usable. Pinned, as a gap rather than as
|
|
1608
|
+
correct behaviour, by the second half of
|
|
1609
|
+
`::test_835_r10_a_bounds_free_capture_record_resolves_the_weeks_automatic_credit`.
|
|
1610
|
+
|
|
1611
|
+
A CAPTURE INSTANT CARRIED BY TWO WEEK IDENTITIES RETURNS BOTH. No tie-break is
|
|
1612
|
+
invented for it and the caller's union withholds if either week retired the
|
|
1613
|
+
value, which is this subsystem's established direction and what both fallback
|
|
1614
|
+
passes do as well. The shape is not hypothetical: bin/_cctally_record.py:6601-6614
|
|
1615
|
+
records a host briefly running the wrong timezone forking ONE physical week's
|
|
1616
|
+
`week_start_date` across 18 rows, and `weekly_credit_floors` matches
|
|
1617
|
+
`week_start_date` by equality, so a floor can sit under one fork and not the
|
|
1618
|
+
other.
|
|
1619
|
+
|
|
1620
|
+
#834 S1 (#835) Gate A S1. `five_hour_blocks` carries no week columns, and the
|
|
1621
|
+
retirement bands need all three: the `weekly_credit_floors` leg matches
|
|
1622
|
+
`week_start_date` by equality and the `week_reset_events` leg is scoped by the
|
|
1623
|
+
week's bounds. The week is therefore resolved from the account's own snapshot
|
|
1624
|
+
rows, whose `week_start_at` / `week_end_at` are the same bounds
|
|
1625
|
+
`_credit_retirement_bands` is written against.
|
|
1626
|
+
|
|
1627
|
+
The comparison is on PARSED epochs rather than on text, for the reason every
|
|
1628
|
+
SQL site in this area wraps both sides in ``unixepoch()``: `block_start_at` is
|
|
1629
|
+
host-local while `week_start_at` is `+00:00`, so a lexicographic compare would
|
|
1630
|
+
mis-order them.
|
|
1631
|
+
|
|
1632
|
+
THE TWO LEGS DEGRADE INDEPENDENTLY, the way `_credit_retirement_bands`' own two
|
|
1633
|
+
legs already do (#834 S1 Gate A R8). `week_start_at` and `week_end_at` are both
|
|
1634
|
+
nullable and `_derive_week_from_payload` leaves them ``None`` on three of its
|
|
1635
|
+
four paths, so a week with no bounds is a live shape. Gate A S1 required both
|
|
1636
|
+
bounds and resolved NOTHING without them, which threw away a leg it could still
|
|
1637
|
+
answer: the `weekly_credit_floors` leg matches on `week_start_date` alone, and
|
|
1638
|
+
`week_start_date` / `week_end_date` are `TEXT NOT NULL`. So a bounds-free week
|
|
1639
|
+
is placed by DATE instead and returned with ``None`` bounds, which resolves the
|
|
1640
|
+
manual-credit leg and skips the `week_reset_events` leg — exactly what
|
|
1641
|
+
`_credit_retirement_bands` does with absent bounds.
|
|
1642
|
+
|
|
1643
|
+
A bounds-carrying week outranks a bounds-free one, so the date pass runs only
|
|
1644
|
+
when the epoch pass places the instant nowhere. The
|
|
1645
|
+
date window is INCLUSIVE of `week_end_date`, because the usual bounds-free week
|
|
1646
|
+
carries the week's LAST DAY there: `_derive_week_from_payload`'s third path
|
|
1647
|
+
writes `week_start_date + 6 days` and its fourth derives the same through
|
|
1648
|
+
`compute_week_bounds`. THE TWO READINGS DO MEET, which an earlier form of this
|
|
1649
|
+
docstring denied. `_derive_week_from_payload`'s second path takes both dates
|
|
1650
|
+
from the payload verbatim behind nothing but a `weekEndDate >= weekStartDate`
|
|
1651
|
+
check, and `_account_axis_weeks` sends a week down this pass when its bounds are
|
|
1652
|
+
present but UNPARSEABLE, where `week_end_date` came from the bounds-carrying
|
|
1653
|
+
writer and is the exclusive boundary's date by construction. Either shape puts
|
|
1654
|
+
one calendar day inside two weeks' inclusive ranges.
|
|
1655
|
+
|
|
1656
|
+
SO THE DATE PASS RETURNS EVERY CANDIDATE THAT CONTAINS THE DAY AND PICKS NONE,
|
|
1657
|
+
and the caller unions their bands. Two single-choice rules were tried and each
|
|
1658
|
+
published a retired value on the mirror image of the shape it fixed. Rank order
|
|
1659
|
+
placed an instant in the PREDECESSOR whenever the predecessor's newest snapshot
|
|
1660
|
+
row was the newer of the two. Preferring the greatest `week_start_date` fixed
|
|
1661
|
+
that and broke the case where the floor sits on a LONGER declared week
|
|
1662
|
+
`2026-04-20..2026-05-04` beside an uncredited `2026-04-27..2026-05-04` — the
|
|
1663
|
+
exact shape whose existence is the argument against a seven-day window, so the
|
|
1664
|
+
prohibition and that repair rested on one premise pointing opposite ways. No
|
|
1665
|
+
reading of one week's own columns separates two candidates that both declare the
|
|
1666
|
+
day, so choosing is a guess in either direction and consulting all of them is
|
|
1667
|
+
not.
|
|
1668
|
+
|
|
1669
|
+
NEITHER FALLBACK PASS HAS A TIE-BREAK ANY MORE. Both return every candidate they
|
|
1670
|
+
place the instant in, and the caller unions the bands. Through Gate A R8 the
|
|
1671
|
+
epoch pass returned ONE week — the one whose group held the newest
|
|
1672
|
+
`(captured_at_utc, id)`, which is what the since-removed `rk` aggregate ordered
|
|
1673
|
+
by — justified by the claim that the newest anchoring is the best available
|
|
1674
|
+
evidence of which window was in force at capture. That claim is vacuous on the shape that makes overlapping
|
|
1675
|
+
bounds-carrying anchorings reachable at all: bin/_cctally_record.py:6601-6614
|
|
1676
|
+
records a host running the wrong timezone for seven minutes forking ONE physical
|
|
1677
|
+
week's `week_start_date` across 18 rows, and those two anchorings carry
|
|
1678
|
+
IDENTICAL bounds and differ only in the column `weekly_credit_floors` is keyed
|
|
1679
|
+
on. Both anchorings are the same window, so there is no "which window was in
|
|
1680
|
+
force" to answer, and a newest-anchoring choice decides only which of two
|
|
1681
|
+
spellings of one week's floors gets consulted. It also inverts the claim's own
|
|
1682
|
+
logic: where two anchorings genuinely differ, that tie-break publishes the
|
|
1683
|
+
loser's retired value rather than withholding it.
|
|
1684
|
+
|
|
1685
|
+
The two passes still differ in PRECEDENCE, deliberately. The epoch pass has real
|
|
1686
|
+
half-open boundaries to test an instant against, so every candidate it keeps
|
|
1687
|
+
genuinely contains the instant; the date pass has no boundaries, only two date
|
|
1688
|
+
columns of which a live writer fills one with the EXCLUSIVE boundary's date, so
|
|
1689
|
+
"contains the day" there does not mean "belongs to the week". A bounds-carrying
|
|
1690
|
+
placement is therefore the more precise one and pre-empts the date pass
|
|
1691
|
+
entirely. Pinned by
|
|
1692
|
+
`tests/test_five_hour_breakdown.py::test_835_r9_the_epoch_pass_unions_every_bounds_carrying_week_it_contains`
|
|
1693
|
+
and `::test_835_r8_a_bounds_carrying_week_outranks_a_bounds_free_one`.
|
|
1694
|
+
|
|
1695
|
+
THE DATE PASS'S UNION OVER-WITHHOLDS, AND THAT RESIDUAL IS SMALLER THAN ROUND
|
|
1696
|
+
FIVE'S. When an instant genuinely belongs to an uncredited candidate and an
|
|
1697
|
+
overlapping candidate was credited, the union withholds a value no credit ever
|
|
1698
|
+
retired. Round five paid that on EVERY block observed on a week-boundary day,
|
|
1699
|
+
because `cmd_record_usage` writes `week_end_date = week_start_date + 7 days` —
|
|
1700
|
+
the exclusive boundary's date — while this pass compares inclusively, so every
|
|
1701
|
+
pair of adjacent weeks shares one calendar day. The capture pass now answers
|
|
1702
|
+
every such instant some snapshot row carries, which is the ordinary case, and
|
|
1703
|
+
what is left is an instant NO capture record carries on a store whose weeks have
|
|
1704
|
+
no parseable bounds.
|
|
1705
|
+
|
|
1706
|
+
The direction is the conservative one — an unavailable marker in preference to a
|
|
1707
|
+
number a credit retired — and both fallback passes now take it. It is still not
|
|
1708
|
+
the direction EVERY fork in this change takes: an empty result resolves no bands
|
|
1709
|
+
and publishes the raw value, which is the pre-#835 disposition kept on purpose.
|
|
1710
|
+
`::test_835_r8_the_union_withholds_an_uncredited_weeks_own_value` pins the trade
|
|
1711
|
+
rather than leaving a later reader to find it and read it as a defect, and
|
|
1712
|
+
`::test_835_r9_a_capture_record_resolves_the_week_a_shared_boundary_day_withheld`
|
|
1713
|
+
pins the boundary-day case that no longer pays it.
|
|
1714
|
+
|
|
1715
|
+
A SEVEN-DAY WINDOW OFF `week_start_date` IS STILL NOT AN ALTERNATIVE. It assumes
|
|
1716
|
+
every bounds-free week is seven days long, and the second payload path can
|
|
1717
|
+
declare a longer one, whose tail a seven-day window would place NOWHERE —
|
|
1718
|
+
withdrawing the manual-credit leg this pass exists to resolve. It also buys
|
|
1719
|
+
nothing on the weeks whose `week_end_date` is computed rather than declared: the
|
|
1720
|
+
third path and `compute_week_bounds` both write `week_start_date + 6 days`, and
|
|
1721
|
+
`cmd_record_usage` writes `week_end_date` exactly `week_start_date + 7 days`, so
|
|
1722
|
+
over those rows a seven-day exclusive window and this inclusive one name the
|
|
1723
|
+
same days and neither separates two overlapping candidates.
|
|
1724
|
+
|
|
1725
|
+
ONE DIRECTION IS BOUNDED BY `cmd_record_credit`, AND THE OTHER IS NOT. That
|
|
1726
|
+
command refuses without a canonical weekly boundary and always writes a
|
|
1727
|
+
bounds-carrying synthetic snapshot of its own, so a manually credited week
|
|
1728
|
+
normally has at least one row with bounds — which bounds the direction where a
|
|
1729
|
+
credited week is reachable ONLY through the date pass. It does not bound the
|
|
1730
|
+
opposite direction, the one round five introduced: that same credited week's
|
|
1731
|
+
OTHER rows still form a bounds-free candidate group, because `week_end_date`
|
|
1732
|
+
joins the grouping, so an instant in an adjacent bounds-free week still picks up
|
|
1733
|
+
the credited week's floor and over-withholds. Do not read the synthetic snapshot
|
|
1734
|
+
as bounding both.
|
|
1735
|
+
|
|
1736
|
+
EMPTY when no week names the instant on any pass. That resolves no bands and
|
|
1737
|
+
therefore withholds nothing, which is the pre-#835 behaviour.
|
|
1738
|
+
|
|
1739
|
+
``cache`` IS THE OUTER RENDER CACHE, NOT A WEEK MEMO, AND IT CHANGED SHAPE
|
|
1740
|
+
SILENTLY AT GATE A R8. Gate A S1 passed a dict this function used directly as
|
|
1741
|
+
``{(account_key, instant_iso) -> weeks}``. It now carves its own sub-dicts out
|
|
1742
|
+
of the dict the caller owns, and that dict is the same one
|
|
1743
|
+
`_credit_aware_block_weekly_axes` carves `__bands` out of, so FOUR reserved
|
|
1744
|
+
keys share one namespace:
|
|
1745
|
+
|
|
1746
|
+
``__weeks`` `{account_key -> [_AxisWeek]}`, `_account_axis_weeks`' memo
|
|
1747
|
+
``__stamps`` `{account_key -> {second -> identities}}`, the capture records
|
|
1748
|
+
``__by_date`` `{account_key -> {week_start_date -> identities}}` (R10)
|
|
1749
|
+
``__bands`` `{(week, account_key) -> bands}`, owned by the caller
|
|
1750
|
+
|
|
1751
|
+
A caller passing the OLD shape is not detected and does not raise: every
|
|
1752
|
+
`setdefault` above finds no reserved key, creates a fresh sub-dict, and the
|
|
1753
|
+
pre-existing entries are simply never read. The failure is therefore a silent
|
|
1754
|
+
loss of memoization — one extra pair of queries per account per render, the cost
|
|
1755
|
+
the per-account load exists to remove — and never a wrong answer, because
|
|
1756
|
+
nothing reads a key it did not write. The leading double underscore is what
|
|
1757
|
+
keeps the two populations apart: an `account_key` is either a 32-character
|
|
1758
|
+
lowercase sha256 prefix (`digest.hexdigest()[:32]`, bin/_lib_accounts.py:10,
|
|
1759
|
+
:63 and :70) or the literal `unattributed` (bin/_lib_accounts.py:41), so none of
|
|
1760
|
+
the four names can collide with one.
|
|
1761
|
+
|
|
1762
|
+
EVERY PASS IS GATED ON `_account_axis_weeks` SUCCEEDING, the capture pass
|
|
1763
|
+
included, and that coupling is deliberate. It looks gratuitous, because NEITHER
|
|
1764
|
+
QUERY'S COLUMN DEPENDENCY CONTAINS THE OTHER'S, so in principle a store that
|
|
1765
|
+
fails one could answer the other. The week query reads `week_start_date`,
|
|
1766
|
+
`week_end_date`, `week_start_at` and `week_end_at`; the capture query reads
|
|
1767
|
+
`captured_at_utc`, `week_start_date`, `week_start_at` and `week_end_at`, plus
|
|
1768
|
+
`first_observed_at_utc` and `last_observed_at_utc` from a SECOND table,
|
|
1769
|
+
`five_hour_blocks`. So the divergence has two directions — a store carrying the
|
|
1770
|
+
bounds without `week_end_date`, and one carrying `week_end_date` without
|
|
1771
|
+
`captured_at_utc` or without the block table — and the claim that the capture
|
|
1772
|
+
query's dependency was strictly smaller was FALSE WHEN WRITTEN in R10, because
|
|
1773
|
+
the capture query has read a second table since R9. Two earlier statements of
|
|
1774
|
+
this are withdrawn: the claim was authored by `f32554505`, which is later than
|
|
1775
|
+
`df80ca1dd` that removed the week query's aggregate, so it cannot have "stopped
|
|
1776
|
+
being true" in that round; and it was never true, because the capture query has
|
|
1777
|
+
selected `first_observed_at_utc` / `last_observed_at_utc` from `five_hour_blocks`
|
|
1778
|
+
since `9690c0fae`. Three reasons the gate is kept:
|
|
1779
|
+
|
|
1780
|
+
- The divergence is unreachable. `week_end_date` is `TEXT NOT NULL` in the
|
|
1781
|
+
base `CREATE TABLE` (bin/_cctally_core.py:2127) and no migration adds it, so
|
|
1782
|
+
no cctally store has the other four columns without it. The legacy shape
|
|
1783
|
+
`bin/build-doctor-fixtures.py` builds lacks `week_start_at` / `week_end_at`
|
|
1784
|
+
as well, so BOTH queries fail on it together.
|
|
1785
|
+
- An empty week set from the ordinary cause — the account has no snapshot rows
|
|
1786
|
+
— implies an empty capture map, because the capture query reads the same
|
|
1787
|
+
table filtered on the same `account_key`.
|
|
1788
|
+
- Since R10 the capture pass READS `weeks`, to build the substitution index
|
|
1789
|
+
`_account_axis_bounds_by_start_date` returns, so running it without them
|
|
1790
|
+
degrades it to returning bounds-free identities unenriched.
|
|
1791
|
+
|
|
1792
|
+
THAT THIRD REASON IS NOT THAT DECOUPLING WOULD BUY NOTHING. An unenriched
|
|
1793
|
+
bounds-free identity still resolves the `weekly_credit_floors` leg, and the empty
|
|
1794
|
+
tuple resolves neither leg, so on a store where the week query failed and the
|
|
1795
|
+
capture query succeeded, decoupling WOULD buy the manual leg. What makes the gain
|
|
1796
|
+
unreachable is reason 1, not any equivalence between the two dispositions. The
|
|
1797
|
+
gate is kept because the divergence cannot occur on a store any cctally version
|
|
1798
|
+
created, and decoupling would add a second exit path to reason about for a gain
|
|
1799
|
+
no store can realise. If a future store shape makes the divergence real, the fix
|
|
1800
|
+
is to run the capture pass first and gate only the two fallback passes.
|
|
1801
|
+
"""
|
|
1802
|
+
weeks = _account_axis_weeks(
|
|
1803
|
+
conn, account_key=account_key, cache=cache.setdefault("__weeks", {}))
|
|
1804
|
+
if not weeks:
|
|
1805
|
+
# EVERY pass is gated on the week load, the capture pass included. See the
|
|
1806
|
+
# docstring: the coupling is deliberate and, since R10, load-bearing.
|
|
1807
|
+
return ()
|
|
1808
|
+
try:
|
|
1809
|
+
instant = parse_iso_datetime(instant_iso, "axis_instant")
|
|
1810
|
+
except (ValueError, TypeError):
|
|
1811
|
+
return ()
|
|
1812
|
+
moment = instant.timestamp()
|
|
1813
|
+
# PASS 1 — the capture record. Truncated to the second on both sides, matching
|
|
1814
|
+
# the `unixepoch()` the narrowing query uses; see `_account_axis_capture_weeks`.
|
|
1815
|
+
carried = _account_axis_capture_weeks(
|
|
1816
|
+
conn, account_key=account_key,
|
|
1817
|
+
cache=cache.setdefault("__stamps", {})).get(int(moment))
|
|
1818
|
+
if carried:
|
|
1819
|
+
# A bounds-free matched row names the week but cannot scope the
|
|
1820
|
+
# `week_reset_events` leg, and this pass pre-empts the epoch pass, so
|
|
1821
|
+
# substitute the SAME week's bounds as this account's OWN other rows spell
|
|
1822
|
+
# them — see the docstring for why the substitution is lossless, why
|
|
1823
|
+
# containment is not required of a substitute, and what residual survives.
|
|
1824
|
+
named: dict = {}
|
|
1825
|
+
for identity in carried:
|
|
1826
|
+
if identity[1] is not None:
|
|
1827
|
+
named[identity] = None
|
|
1828
|
+
continue
|
|
1829
|
+
by_date = cache.setdefault("__by_date", {})
|
|
1830
|
+
if account_key not in by_date:
|
|
1831
|
+
by_date[account_key] = _account_axis_bounds_by_start_date(weeks)
|
|
1832
|
+
# Built once per account from the already-loaded week set, so the
|
|
1833
|
+
# substitution costs no query and no per-instant scan.
|
|
1834
|
+
named.update(dict.fromkeys(
|
|
1835
|
+
by_date[account_key].get(identity[0]) or (identity,)))
|
|
1836
|
+
return tuple(named)
|
|
1837
|
+
# PASS 2 — containment in a week's parsed half-open bounds. A dict as an
|
|
1838
|
+
# ordered set throughout: the caller's union is order-insensitive, so the
|
|
1839
|
+
# order these are visited in carries no meaning, and membership must not be a
|
|
1840
|
+
# linear scan.
|
|
1841
|
+
found: dict = {}
|
|
1842
|
+
for week in weeks:
|
|
1843
|
+
if week.lo is not None and week.lo <= moment < week.hi:
|
|
1844
|
+
found[(week.week_start_date, week.week_start_at, week.week_end_at)] = (
|
|
1845
|
+
None)
|
|
1846
|
+
if found:
|
|
1847
|
+
return tuple(found)
|
|
1848
|
+
# PASS 3 — the instant's calendar day against the bounds-free weeks' declared
|
|
1849
|
+
# date ranges. `.astimezone(utc)` before `.date()` for
|
|
1850
|
+
# `_derive_week_from_payload`'s own reason: `parse_iso_datetime` ends in
|
|
1851
|
+
# `.astimezone()`, so on a host whose offset puts the moment on another calendar
|
|
1852
|
+
# date a bare `.date()` would place the instant in the wrong week. EVERY
|
|
1853
|
+
# candidate that declares the day, with no preference among them — see the
|
|
1854
|
+
# docstring for why no preference can be grounded here and what the union costs.
|
|
1855
|
+
day = instant.astimezone(dt.timezone.utc).date().isoformat()
|
|
1856
|
+
for week in weeks:
|
|
1857
|
+
if week.lo is not None:
|
|
1858
|
+
continue
|
|
1859
|
+
# Both columns are `TEXT NOT NULL`, so a None here is a store nobody can
|
|
1860
|
+
# place; skip it rather than raise comparing a string to None.
|
|
1861
|
+
if week.week_start_date is None or week.week_end_date is None:
|
|
1862
|
+
continue
|
|
1863
|
+
if week.week_start_date <= day <= week.week_end_date:
|
|
1864
|
+
# `week_end_date` is in `_account_axis_weeks`' DISTINCT projection, so
|
|
1865
|
+
# two rows differing only there are two candidates carrying ONE identity.
|
|
1866
|
+
# The bands would be identical, so the duplicate is dropped rather than
|
|
1867
|
+
# resolved twice.
|
|
1868
|
+
found[(week.week_start_date, None, None)] = None
|
|
1869
|
+
return tuple(found)
|
|
1870
|
+
|
|
1871
|
+
|
|
1872
|
+
def _credit_aware_block_weekly_axes(conn, block, *, now_utc, cache=None):
|
|
1873
|
+
"""A block's two weekly axes, with a credit-RETIRED value replaced by ``None``.
|
|
1874
|
+
|
|
1875
|
+
#834 S1 (#835) Gate A S1. THE ONE READ-TIME FILTER for
|
|
1876
|
+
`five_hour_blocks.seven_day_pct_at_block_start` and
|
|
1877
|
+
`seven_day_pct_at_block_end`, called by `cmd_five_hour_blocks` and
|
|
1878
|
+
`cmd_five_hour_breakdown` so the two cannot diverge.
|
|
1879
|
+
|
|
1880
|
+
WHY READ TIME, AND WHY A WRITE-TIME FILTER CANNOT SUBSTITUTE. A credit can
|
|
1881
|
+
retire a value AFTER the block row was written — that is the ordinary case,
|
|
1882
|
+
because the live upsert writes the block's weekly end on every tick while the
|
|
1883
|
+
block is open and the credit fires later — so no write-time pass can ever be
|
|
1884
|
+
sufficient. FOUR writers store these columns, not three: the live upsert in
|
|
1885
|
+
bin/_cctally_record.py, `five_hour_block_close`'s frozen harvest in
|
|
1886
|
+
bin/_cctally_journal.py, `_backfill_five_hour_blocks` below, and
|
|
1887
|
+
`_migration_merge_5h_block_duplicates_v1` in bin/_cctally_db.py, which writes
|
|
1888
|
+
`seven_day_pct_at_block_end` from the group row whose `last_observed_at_utc` is
|
|
1889
|
+
MAX. The migration was omitted from the original count; the argument is
|
|
1890
|
+
unaffected, because it also stores what was observed, but the number was
|
|
1891
|
+
documented and wrong. All four store what the fold OBSERVED; this one read
|
|
1892
|
+
decides what a credit retired may still be shown. Making one writer
|
|
1893
|
+
credit-aware instead would also break
|
|
1894
|
+
rebuild convergence, because the store would then hold a number where a
|
|
1895
|
+
rebuilt store holds `NULL` and `tests/test_rederive_command.py` compares
|
|
1896
|
+
exactly these two columns across a rebuild.
|
|
1897
|
+
|
|
1898
|
+
EACH AXIS IS COMPARED AGAINST THE INSTANT THAT AXIS'S VALUE WAS CAPTURED AT,
|
|
1899
|
+
never against a boundary of the block: the start against
|
|
1900
|
+
`_block_weekly_start_instant` (`first_observed_at_utc`), the end against
|
|
1901
|
+
`_block_weekly_end_instant` (`last_observed_at_utc`). Gate A S1 used the block's
|
|
1902
|
+
own boundary stamps instead and that was wrong in both directions — see those
|
|
1903
|
+
two helpers. The rule separates the three positions a block's OBSERVATIONS can
|
|
1904
|
+
hold relative to a credit floor. Observations entirely BEFORE the floor render
|
|
1905
|
+
both stored values, because they genuinely were effective while the block ran
|
|
1906
|
+
and withholding them would destroy historical truth. Observations SPANNING the
|
|
1907
|
+
floor render the start and withhold a retired end. Observations entirely AFTER
|
|
1908
|
+
the floor withhold both, which is the shape a post-credit replay at the retired
|
|
1909
|
+
value produces.
|
|
1910
|
+
|
|
1911
|
+
The bands come from the block's OWN week and `account_key` — a credit under
|
|
1912
|
+
one account retires nothing under another, and a crossed-reset block's two
|
|
1913
|
+
axes legitimately resolve two different weeks.
|
|
1914
|
+
|
|
1915
|
+
``cache`` is an optional dict shared across a multi-row render, and it holds
|
|
1916
|
+
FOUR reserved keys in ONE namespace: this function's own ``__bands``, resolved
|
|
1917
|
+
once per `(week, account_key)`, plus the ``__weeks``, ``__stamps`` and
|
|
1918
|
+
``__by_date`` that `_block_weekly_axis_weeks` carves out of the same dict. That
|
|
1919
|
+
function's docstring states each one's shape and what happens to a caller who
|
|
1920
|
+
passes the pre-R8 shape, which is a silent loss of memoization rather than a
|
|
1921
|
+
wrong answer. Together they hold the account's whole week set, the capture
|
|
1922
|
+
records of that account's block axis stamps, the per-`week_start_date` index of
|
|
1923
|
+
bounds-carrying identities, and the bands. Each axis instant is then resolved
|
|
1924
|
+
against the loaded sets in memory, so the number of queries does not grow with
|
|
1925
|
+
the number of rendered blocks. The caller owns the dict and it must not outlive
|
|
1926
|
+
the connection.
|
|
1927
|
+
"""
|
|
1928
|
+
_c = _cctally()
|
|
1929
|
+
p_start = block.get("seven_day_pct_at_block_start")
|
|
1930
|
+
p_end = block.get("seven_day_pct_at_block_end")
|
|
1931
|
+
if p_start is None and p_end is None:
|
|
1932
|
+
return None, None
|
|
1933
|
+
if cache is None:
|
|
1934
|
+
cache = {}
|
|
1935
|
+
bands = cache.setdefault("__bands", {})
|
|
1936
|
+
account_key = block.get("account_key") or _lib_accounts.UNATTRIBUTED
|
|
1937
|
+
out = []
|
|
1938
|
+
for value, instant in (
|
|
1939
|
+
(p_start, _block_weekly_start_instant(block)),
|
|
1940
|
+
(p_end, _block_weekly_end_instant(block, now_utc=now_utc)),
|
|
1941
|
+
):
|
|
1942
|
+
if value is None or not instant:
|
|
1943
|
+
out.append(value)
|
|
1944
|
+
continue
|
|
1945
|
+
week_ids = _block_weekly_axis_weeks(
|
|
1946
|
+
conn, account_key=account_key, instant_iso=instant, cache=cache)
|
|
1947
|
+
if not week_ids:
|
|
1948
|
+
out.append(value)
|
|
1949
|
+
continue
|
|
1950
|
+
# THE UNION OF EVERY CANDIDATE WEEK'S BANDS, so a value any one of them
|
|
1951
|
+
# retired is withheld. One bounds-carrying week is one candidate; several
|
|
1952
|
+
# bounds-free ones are several, because nothing in their own columns says
|
|
1953
|
+
# which of them an instant belongs to. Each is still resolved by
|
|
1954
|
+
# `_credit_retirement_bands` from its OWN triple, so the per-leg
|
|
1955
|
+
# degradation is unchanged: a bounds-free candidate contributes its
|
|
1956
|
+
# `weekly_credit_floors` leg and no `week_reset_events` leg, which cannot
|
|
1957
|
+
# be scoped without bounds. The memo stays per `(week, account_key)`, so
|
|
1958
|
+
# overlapping candidates shared across blocks resolve once each.
|
|
1959
|
+
merged: list = []
|
|
1960
|
+
for week in week_ids:
|
|
1961
|
+
bk = (week, account_key)
|
|
1962
|
+
if bk not in bands:
|
|
1963
|
+
bands[bk] = _c._credit_retirement_bands(
|
|
1964
|
+
conn, week_start_date=week[0], week_start_at=week[1],
|
|
1965
|
+
week_end_at=week[2], account_key=account_key)
|
|
1966
|
+
merged.extend(bands[bk])
|
|
1967
|
+
out.append(
|
|
1968
|
+
None if _c._weekly_value_is_retired_replica(
|
|
1969
|
+
merged, captured_at_utc=instant, weekly_percent=value)
|
|
1970
|
+
else value)
|
|
1971
|
+
return out[0], out[1]
|
|
1972
|
+
|
|
1973
|
+
|
|
1974
|
+
#: The six `five_hour_milestones` columns every surface already published, plus
|
|
1975
|
+
#: the joined EFFECTIVE weekly value. Ordering is `captured_at_utc, id` — spec
|
|
1976
|
+
#: §5.2's merged-stream order, which post-credit threshold repeats depend on.
|
|
1977
|
+
#: The five `s.*` columns after the joined weekly value are INTERNAL. They exist
|
|
1978
|
+
#: only so the credit retirement bands can be resolved against the JOINED ROW's
|
|
1979
|
+
#: own account and week, and they are aliased because `captured_at_utc` exists on
|
|
1980
|
+
#: both sides of the join. `_load_five_hour_milestones` builds its result dicts
|
|
1981
|
+
#: key by key, so none of them can reach a published shape.
|
|
1982
|
+
_FIVE_HOUR_MILESTONE_SQL = """
|
|
1983
|
+
SELECT m.percent_threshold, m.captured_at_utc, m.block_cost_usd,
|
|
1984
|
+
m.marginal_cost_usd, m.seven_day_pct_at_crossing, m.reset_event_id,
|
|
1985
|
+
s.weekly_percent AS effective_seven_day_pct_at_crossing,
|
|
1986
|
+
s.captured_at_utc AS snapshot_captured_at_utc,
|
|
1987
|
+
s.week_start_date AS snapshot_week_start_date,
|
|
1988
|
+
s.week_start_at AS snapshot_week_start_at,
|
|
1989
|
+
s.week_end_at AS snapshot_week_end_at,
|
|
1990
|
+
s.account_key AS snapshot_account_key
|
|
1991
|
+
FROM five_hour_milestones m
|
|
1992
|
+
LEFT JOIN weekly_usage_snapshots s ON s.id = m.usage_snapshot_id
|
|
1993
|
+
WHERE {selector}
|
|
1994
|
+
ORDER BY m.captured_at_utc ASC, m.id ASC
|
|
1995
|
+
"""
|
|
1996
|
+
|
|
1997
|
+
|
|
1998
|
+
def _load_five_hour_milestones(
|
|
1999
|
+
conn: sqlite3.Connection, *,
|
|
2000
|
+
block_id: "int | None" = None,
|
|
2001
|
+
account_key: "str | None" = None,
|
|
2002
|
+
five_hour_window_key: "int | None" = None,
|
|
2003
|
+
) -> list[dict]:
|
|
2004
|
+
"""The ONE read of a block's five-hour milestones (#834 S1, #836).
|
|
2005
|
+
|
|
2006
|
+
Three human surfaces used to carry their own copy of this query — the CLI
|
|
2007
|
+
breakdown, the dashboard's live current-week route and the dashboard's
|
|
2008
|
+
historical-detail route — and two of them shared a defect: they filtered on
|
|
2009
|
+
`five_hour_window_key` alone while block uniqueness is
|
|
2010
|
+
`(account_key, five_hour_window_key)`, so one physical window's per-account
|
|
2011
|
+
blocks all showed every account's milestones.
|
|
2012
|
+
|
|
2013
|
+
Returns snake_case dicts carrying the six columns every surface already
|
|
2014
|
+
published PLUS `effective_seven_day_pct_at_crossing`, the weekly value a
|
|
2015
|
+
reader actually saw. `seven_day_pct_at_crossing` stays the RAW stored value,
|
|
2016
|
+
unchanged and frozen; the effective value comes from a `LEFT JOIN` to the
|
|
2017
|
+
`weekly_usage_snapshots` row the milestone already names. Since #824 that row
|
|
2018
|
+
carries the effective weekly value on a clamped tick, so no recomputation is
|
|
2019
|
+
needed. The join is row-count-safe because it joins on a primary key, and the
|
|
2020
|
+
existing `ORDER BY captured_at_utc, id` is retained because the merged
|
|
2021
|
+
credit/milestone stream depends on it.
|
|
2022
|
+
|
|
2023
|
+
When the referenced snapshot row is ABSENT — which the non-held stale
|
|
2024
|
+
replicas the credit DELETE still removes can cause — the `LEFT JOIN` retains
|
|
2025
|
+
the milestone and the effective value is None. Every surface must then render
|
|
2026
|
+
an unavailable marker. It must NEVER fall back to the raw value: that would
|
|
2027
|
+
put the number this change exists to stop showing back on the screen,
|
|
2028
|
+
silently and only in the failure case.
|
|
2029
|
+
|
|
2030
|
+
**The joined value is itself CREDIT-AWARE** (#834 S1 Gate A R2). A plain
|
|
2031
|
+
`LEFT JOIN` is not enough: `maybe_record_milestone` sets `usage_snapshot_id`
|
|
2032
|
+
to the row the tick just wrote, and the FIVE-HOUR milestone is not gated off
|
|
2033
|
+
on a weekly-clamped tick — only the weekly one is — so in a credited week the
|
|
2034
|
+
milestone can name a HELD row carrying a pre-credit weekly value the credit
|
|
2035
|
+
retired. Publishing that as the "effective" value would render the exact
|
|
2036
|
+
number #836 exists to remove, on all three human surfaces, on a new read.
|
|
2037
|
+
|
|
2038
|
+
So the retirement bands are resolved against the JOINED ROW's own account and
|
|
2039
|
+
week — never the reader's, because a credit under one account retires nothing
|
|
2040
|
+
under another — memoized per `(week, account)` across the rows of one call, and
|
|
2041
|
+
a retired joined value is published as None. The surfaces already render a None
|
|
2042
|
+
as the unavailable marker, so no renderer changes, and the rule above still
|
|
2043
|
+
holds: never a fallback to the raw value. (The memoization is this function's
|
|
2044
|
+
own: it walks many rows spanning potentially several `(week, account)` pairs.
|
|
2045
|
+
`_latest_seven_day_and_window` does NOT memoize, because its walk resolves the
|
|
2046
|
+
scope, the bands and the floor once from the newest row and may not leave that
|
|
2047
|
+
scope, so there is nothing to memoize per candidate. An earlier version of this
|
|
2048
|
+
paragraph said it did.)
|
|
2049
|
+
|
|
2050
|
+
THE BAND IS APPLIED HERE WITHOUT R1's FLOOR BOUND, deliberately. That bound
|
|
2051
|
+
exists because a WALK can descend past every post-floor row onto a pre-floor
|
|
2052
|
+
one and return the pre-credit reading the band exists to suppress. This read
|
|
2053
|
+
performs no walk: each milestone names exactly ONE snapshot row through
|
|
2054
|
+
`usage_snapshot_id`, and a pre-floor row's weekly value genuinely WAS the
|
|
2055
|
+
effective percentage at that crossing instant. Blanking it would delete
|
|
2056
|
+
historical truth rather than protect the reader, which is the same reason the
|
|
2057
|
+
block's pre-floor weekly axes render.
|
|
2058
|
+
|
|
2059
|
+
SELECTOR. Exactly one of `block_id` or `five_hour_window_key` must be given.
|
|
2060
|
+
`block_id` is the precise form — `five_hour_milestones.block_id` names one
|
|
2061
|
+
block outright. The pair form filters on the window key AND the account,
|
|
2062
|
+
which is the same uniqueness expressed as its two parts, and exists for a
|
|
2063
|
+
caller that holds a window rather than an id. Passing neither, both, or
|
|
2064
|
+
`account_key` alongside `block_id` raises `ValueError` rather than silently
|
|
2065
|
+
ignoring an argument, because a silently ignored account predicate is exactly
|
|
2066
|
+
the defect this function replaces.
|
|
2067
|
+
"""
|
|
2068
|
+
if (block_id is None) == (five_hour_window_key is None):
|
|
2069
|
+
raise ValueError(
|
|
2070
|
+
"_load_five_hour_milestones: pass exactly one of block_id or "
|
|
2071
|
+
"five_hour_window_key")
|
|
2072
|
+
if block_id is not None and account_key is not None:
|
|
2073
|
+
raise ValueError(
|
|
2074
|
+
"_load_five_hour_milestones: account_key is meaningful only with "
|
|
2075
|
+
"five_hour_window_key; block_id already names one account's block")
|
|
2076
|
+
if block_id is not None:
|
|
2077
|
+
selector = "m.block_id = ?"
|
|
2078
|
+
params: tuple = (int(block_id),)
|
|
2079
|
+
else:
|
|
2080
|
+
selector = "m.five_hour_window_key = ?"
|
|
2081
|
+
params = (int(five_hour_window_key),)
|
|
2082
|
+
if account_key is not None:
|
|
2083
|
+
selector += " AND m.account_key = ?"
|
|
2084
|
+
params += (account_key,)
|
|
2085
|
+
rows = conn.execute(
|
|
2086
|
+
_FIVE_HOUR_MILESTONE_SQL.format(selector=selector), params,
|
|
2087
|
+
).fetchall()
|
|
2088
|
+
_c = _cctally()
|
|
2089
|
+
bands_cache: dict = {}
|
|
2090
|
+
out: list[dict] = []
|
|
2091
|
+
for r in rows:
|
|
2092
|
+
eff = r["effective_seven_day_pct_at_crossing"]
|
|
2093
|
+
raw = r["seven_day_pct_at_crossing"]
|
|
2094
|
+
if eff is not None:
|
|
2095
|
+
band_key = (r["snapshot_week_start_date"],
|
|
2096
|
+
r["snapshot_week_start_at"],
|
|
2097
|
+
r["snapshot_week_end_at"],
|
|
2098
|
+
r["snapshot_account_key"])
|
|
2099
|
+
if band_key not in bands_cache:
|
|
2100
|
+
bands_cache[band_key] = _c._credit_retirement_bands(
|
|
2101
|
+
conn,
|
|
2102
|
+
week_start_date=r["snapshot_week_start_date"],
|
|
2103
|
+
week_start_at=r["snapshot_week_start_at"],
|
|
2104
|
+
week_end_at=r["snapshot_week_end_at"],
|
|
2105
|
+
account_key=r["snapshot_account_key"])
|
|
2106
|
+
if _c._weekly_value_is_retired_replica(
|
|
2107
|
+
bands_cache[band_key],
|
|
2108
|
+
captured_at_utc=r["snapshot_captured_at_utc"],
|
|
2109
|
+
weekly_percent=eff):
|
|
2110
|
+
eff = None
|
|
2111
|
+
out.append({
|
|
2112
|
+
"percent_threshold": int(r["percent_threshold"]),
|
|
2113
|
+
"captured_at_utc": r["captured_at_utc"],
|
|
2114
|
+
"block_cost_usd": float(r["block_cost_usd"]),
|
|
2115
|
+
"marginal_cost_usd": (
|
|
2116
|
+
None if r["marginal_cost_usd"] is None
|
|
2117
|
+
else float(r["marginal_cost_usd"])
|
|
2118
|
+
),
|
|
2119
|
+
"seven_day_pct_at_crossing": (
|
|
2120
|
+
None if raw is None else float(raw)
|
|
2121
|
+
),
|
|
2122
|
+
"effective_seven_day_pct_at_crossing": (
|
|
2123
|
+
None if eff is None else float(eff)
|
|
2124
|
+
),
|
|
2125
|
+
"reset_event_id": int(r["reset_event_id"] or 0),
|
|
2126
|
+
})
|
|
2127
|
+
return out
|
|
2128
|
+
|
|
2129
|
+
|
|
958
2130
|
def _parse_date_filter(value: str, flag_name: str) -> str:
|
|
959
2131
|
"""Parse ``YYYY-MM-DD`` or ``YYYYMMDD`` into an ISO date for SQL ``WHERE`` clauses.
|
|
960
2132
|
|
|
@@ -1156,11 +2328,31 @@ def cmd_five_hour_blocks(args: argparse.Namespace) -> int:
|
|
|
1156
2328
|
})
|
|
1157
2329
|
|
|
1158
2330
|
# Build per-block dicts with the active-flag side-channel.
|
|
2331
|
+
# #834 S1 (#835) Gate A R8: one cache shared across the render. The week
|
|
2332
|
+
# SET is loaded ONCE PER ACCOUNT and each axis instant is then placed in
|
|
2333
|
+
# memory; the bands are resolved once per (week, account). The S1 form
|
|
2334
|
+
# memoized on `(account_key, instant_iso)` and claimed the lookup was paid
|
|
2335
|
+
# "once per week rather than once per block" — it was not, because the two
|
|
2336
|
+
# instants of one block are two distinct keys, so the render paid one full
|
|
2337
|
+
# table scan and one full sort per axis. Measured on a 25,053-row store
|
|
2338
|
+
# with 50 rendered blocks: 100 lookups, 49 hits, and 157.8-162.0 ms for
|
|
2339
|
+
# `five-hour-blocks --json` against 7.0-7.8 ms before S1. It must not
|
|
2340
|
+
# outlive `conn`.
|
|
2341
|
+
_axes_cache: dict = {}
|
|
1159
2342
|
block_dicts: list[dict] = []
|
|
1160
2343
|
for r in rows:
|
|
1161
2344
|
d = dict(r)
|
|
1162
2345
|
is_active = _block_is_active(d, latest_window_key, now_utc)
|
|
1163
2346
|
d["__is_active"] = is_active
|
|
2347
|
+
# #834 S1 (#835) Gate A S1: BOTH weekly axes are filtered here, for
|
|
2348
|
+
# every block, closed or active. The stored end is the retired number
|
|
2349
|
+
# by construction on a weekly-clamped tick, and the stored start had
|
|
2350
|
+
# no filter and no override at all. The live `latest_7d` below still
|
|
2351
|
+
# wins on the active row whenever it is not None.
|
|
2352
|
+
(d["seven_day_pct_at_block_start"],
|
|
2353
|
+
d["seven_day_pct_at_block_end"]) = (
|
|
2354
|
+
_c._credit_aware_block_weekly_axes(
|
|
2355
|
+
conn, d, now_utc=now_utc, cache=_axes_cache))
|
|
1164
2356
|
if is_active and latest_7d is not None:
|
|
1165
2357
|
d["seven_day_pct_at_block_end"] = latest_7d
|
|
1166
2358
|
# Side-channel (parallel to __is_active): list of credit
|
|
@@ -1304,23 +2496,14 @@ def cmd_five_hour_breakdown(args: argparse.Namespace) -> int:
|
|
|
1304
2496
|
)
|
|
1305
2497
|
return 2
|
|
1306
2498
|
|
|
1307
|
-
#
|
|
1308
|
-
#
|
|
1309
|
-
#
|
|
1310
|
-
#
|
|
1311
|
-
#
|
|
1312
|
-
#
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
SELECT percent_threshold, captured_at_utc,
|
|
1316
|
-
block_cost_usd, marginal_cost_usd,
|
|
1317
|
-
seven_day_pct_at_crossing, reset_event_id
|
|
1318
|
-
FROM five_hour_milestones
|
|
1319
|
-
WHERE block_id = ?
|
|
1320
|
-
ORDER BY captured_at_utc ASC, id ASC
|
|
1321
|
-
""",
|
|
1322
|
-
(block["id"],),
|
|
1323
|
-
).fetchall()
|
|
2499
|
+
# #834 S1 (#836): the ONE milestone read, shared with both dashboard
|
|
2500
|
+
# routes. It keeps spec §5.2's `ORDER BY captured_at_utc ASC` (NOT
|
|
2501
|
+
# `percent_threshold`) so post-credit segments interleave with pre-credit
|
|
2502
|
+
# ones in time-order — the same human threshold number can appear twice,
|
|
2503
|
+
# once per `reset_event_id` segment, and must render in the order it
|
|
2504
|
+
# crossed. Bucket B per §3.2: ALL segments, no `reset_event_id` filter.
|
|
2505
|
+
# `block_id` is the precise selector, so no account predicate is needed.
|
|
2506
|
+
milestones = _load_five_hour_milestones(conn, block_id=block["id"])
|
|
1324
2507
|
|
|
1325
2508
|
# Spec §5.2 — load in-place credit events for this block's
|
|
1326
2509
|
# window, ascending by effective_reset_at_utc, so the text
|
|
@@ -1349,8 +2532,11 @@ def cmd_five_hour_breakdown(args: argparse.Namespace) -> int:
|
|
|
1349
2532
|
]
|
|
1350
2533
|
|
|
1351
2534
|
crossed = bool(block.get("crossed_seven_day_reset"))
|
|
1352
|
-
|
|
1353
|
-
|
|
2535
|
+
# #834 S1 (#835) Gate A S1: the same ONE read-time filter the blocks
|
|
2536
|
+
# command applies, over both axes. `--block-start` and `--ago` reach
|
|
2537
|
+
# closed blocks, which carry no override at all.
|
|
2538
|
+
p_start, p_end = _c._credit_aware_block_weekly_axes(
|
|
2539
|
+
conn, block, now_utc=now_utc)
|
|
1354
2540
|
|
|
1355
2541
|
# Live 7d_end on active row.
|
|
1356
2542
|
latest_7d, latest_window_key = _latest_seven_day_and_window(
|
|
@@ -1389,6 +2575,10 @@ def cmd_five_hour_breakdown(args: argparse.Namespace) -> int:
|
|
|
1389
2575
|
# consumers can disambiguate post-credit threshold repeats from
|
|
1390
2576
|
# pre-credit ones. ``0`` is the pre-credit/no-credit sentinel
|
|
1391
2577
|
# (matches the schema default).
|
|
2578
|
+
# #834 S1 (#836): `sevenDayPctAtCrossing` keeps its type and meaning —
|
|
2579
|
+
# the RAW stored value — and the nullable
|
|
2580
|
+
# `effectiveSevenDayPctAtCrossing` is added beside it, so the addition is
|
|
2581
|
+
# purely additive and `schemaVersion` does not move.
|
|
1392
2582
|
ms_out = [
|
|
1393
2583
|
{
|
|
1394
2584
|
"percentThreshold": m["percent_threshold"],
|
|
@@ -1399,6 +2589,9 @@ def cmd_five_hour_breakdown(args: argparse.Namespace) -> int:
|
|
|
1399
2589
|
else round(m["marginal_cost_usd"], 9)
|
|
1400
2590
|
),
|
|
1401
2591
|
"sevenDayPctAtCrossing": m["seven_day_pct_at_crossing"],
|
|
2592
|
+
"effectiveSevenDayPctAtCrossing": (
|
|
2593
|
+
m["effective_seven_day_pct_at_crossing"]
|
|
2594
|
+
),
|
|
1402
2595
|
"resetEventId": int(m["reset_event_id"] or 0),
|
|
1403
2596
|
}
|
|
1404
2597
|
for m in milestones
|
|
@@ -1501,9 +2694,15 @@ def cmd_five_hour_breakdown(args: argparse.Namespace) -> int:
|
|
|
1501
2694
|
"n/a" if m["marginalCostUSD"] is None
|
|
1502
2695
|
else f"${m['marginalCostUSD']:.6f}"
|
|
1503
2696
|
)
|
|
2697
|
+
# #834 S1 (#836): the EFFECTIVE weekly value, never the raw one. A
|
|
2698
|
+
# weekly-clamped tick stored a reading no reader ever saw, and this
|
|
2699
|
+
# column is what a human reads. When the joined snapshot row is
|
|
2700
|
+
# absent the value is unavailable and the marker says so — falling
|
|
2701
|
+
# back to `sevenDayPctAtCrossing` would put the raw number back on
|
|
2702
|
+
# the screen exactly where nobody would look for it.
|
|
1504
2703
|
p7d = (
|
|
1505
|
-
"—" if m["
|
|
1506
|
-
else f"{m['
|
|
2704
|
+
"—" if m["effectiveSevenDayPctAtCrossing"] is None
|
|
2705
|
+
else f"{m['effectiveSevenDayPctAtCrossing']:.0f}%"
|
|
1507
2706
|
)
|
|
1508
2707
|
rows.append(
|
|
1509
2708
|
[str(idx), f"{m['percentThreshold']}%", cum, marg, p7d]
|
|
@@ -1583,6 +2782,61 @@ def _backfill_five_hour_blocks(
|
|
|
1583
2782
|
now_iso = now_utc_iso()
|
|
1584
2783
|
now_dt = parse_iso_datetime(now_iso, "now")
|
|
1585
2784
|
|
|
2785
|
+
# #751a ownership context. Each window this backfill materializes
|
|
2786
|
+
# competes with the windows already in `five_hour_blocks` AND with
|
|
2787
|
+
# the other windows this same pass is about to insert, so both are
|
|
2788
|
+
# collected before any total is computed. Without the second half a
|
|
2789
|
+
# from-empty backfill has no competitors at all and every adjacent
|
|
2790
|
+
# pair re-prices its overlap twice.
|
|
2791
|
+
ownership: dict[Any, list] = {}
|
|
2792
|
+
seen_keys: set = set()
|
|
2793
|
+
for row in conn.execute(
|
|
2794
|
+
"SELECT five_hour_window_key, block_start_at, "
|
|
2795
|
+
" five_hour_resets_at, account_key FROM five_hour_blocks"
|
|
2796
|
+
).fetchall():
|
|
2797
|
+
try:
|
|
2798
|
+
w_start = parse_iso_datetime(
|
|
2799
|
+
row["block_start_at"], "five_hour_blocks.block_start_at",
|
|
2800
|
+
).astimezone(dt.timezone.utc)
|
|
2801
|
+
w_reset = parse_iso_datetime(
|
|
2802
|
+
row["five_hour_resets_at"],
|
|
2803
|
+
"five_hour_blocks.five_hour_resets_at",
|
|
2804
|
+
).astimezone(dt.timezone.utc)
|
|
2805
|
+
except ValueError:
|
|
2806
|
+
continue
|
|
2807
|
+
w_key = int(row["five_hour_window_key"])
|
|
2808
|
+
seen_keys.add((row["account_key"], w_key))
|
|
2809
|
+
ownership.setdefault(row["account_key"], []).append(
|
|
2810
|
+
_c.OwnedWindow(key=w_key, start=w_start, reset=w_reset)
|
|
2811
|
+
)
|
|
2812
|
+
for key, acct in keys:
|
|
2813
|
+
if (acct, key) in seen_keys:
|
|
2814
|
+
continue
|
|
2815
|
+
# Same MIN-captured pick the insert loop makes below, so the
|
|
2816
|
+
# ownership interval and the inserted row cannot disagree.
|
|
2817
|
+
anchor_row = conn.execute(
|
|
2818
|
+
"""
|
|
2819
|
+
SELECT five_hour_resets_at
|
|
2820
|
+
FROM weekly_usage_snapshots
|
|
2821
|
+
WHERE five_hour_window_key = ? AND account_key = ?
|
|
2822
|
+
AND five_hour_percent IS NOT NULL
|
|
2823
|
+
ORDER BY captured_at_utc ASC, id ASC LIMIT 1
|
|
2824
|
+
""",
|
|
2825
|
+
(key, acct),
|
|
2826
|
+
).fetchone()
|
|
2827
|
+
if anchor_row is None:
|
|
2828
|
+
continue
|
|
2829
|
+
try:
|
|
2830
|
+
w_reset = parse_iso_datetime(
|
|
2831
|
+
anchor_row["five_hour_resets_at"], "five_hour_resets_at",
|
|
2832
|
+
).astimezone(dt.timezone.utc)
|
|
2833
|
+
except ValueError:
|
|
2834
|
+
continue
|
|
2835
|
+
seen_keys.add((acct, key))
|
|
2836
|
+
ownership.setdefault(acct, []).append(_c.OwnedWindow(
|
|
2837
|
+
key=key, start=w_reset - dt.timedelta(hours=5), reset=w_reset,
|
|
2838
|
+
))
|
|
2839
|
+
|
|
1586
2840
|
# BEGIN IMMEDIATE (not deferred): this transaction's first DML is a
|
|
1587
2841
|
# READ (min_row/max_row below), so a plain deferred BEGIN takes a read
|
|
1588
2842
|
# snapshot and only tries to upgrade to the write lock at the first
|
|
@@ -1689,6 +2943,7 @@ def _backfill_five_hour_blocks(
|
|
|
1689
2943
|
# cascade here would reopen cache.db recursively.
|
|
1690
2944
|
totals = _c._compute_block_totals(
|
|
1691
2945
|
block_start_dt, last_obs_dt, skip_sync=True,
|
|
2946
|
+
owner_key=key, windows=ownership.get(acct, []),
|
|
1692
2947
|
)
|
|
1693
2948
|
|
|
1694
2949
|
cur = conn.execute(
|