cctally 1.115.0 → 1.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/bin/_cctally_cache.py +85 -4
- package/bin/_cctally_completion.py +80 -12
- package/bin/_cctally_core.py +106 -9
- package/bin/_cctally_dashboard.py +6 -9
- package/bin/_cctally_dashboard_envelope.py +26 -32
- package/bin/_cctally_db.py +85 -4
- package/bin/_cctally_doctor.py +38 -5
- package/bin/_cctally_five_hour.py +83 -1017
- package/bin/_cctally_journal.py +1307 -180
- package/bin/_cctally_journal_repair.py +9 -7
- package/bin/_cctally_oauth_samples.py +280 -0
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_quarantine.py +14 -4
- package/bin/_cctally_quota.py +70 -8
- package/bin/_cctally_quota_model.py +35 -4
- package/bin/_cctally_record.py +414 -123
- package/bin/_cctally_rederive.py +632 -235
- package/bin/_cctally_refresh.py +29 -0
- package/bin/_cctally_statusline.py +984 -47
- package/bin/_cctally_store.py +972 -58
- package/bin/_cctally_tui.py +62 -7
- package/bin/_cctally_update.py +25 -0
- package/bin/_lib_account_protocol.py +201 -18
- package/bin/_lib_clone_copy.py +46 -11
- package/bin/_lib_dashboard_sources.py +57 -15
- package/bin/_lib_detector_access_registry.py +273 -52
- package/bin/_lib_doctor.py +302 -1
- package/bin/_lib_journal.py +145 -25
- package/bin/_lib_journal_router.py +166 -48
- package/bin/_lib_oauth_samples.py +238 -0
- package/bin/_lib_pricing_check.py +0 -12
- package/bin/_lib_provider_activity.py +770 -0
- package/bin/_lib_quarantine.py +3 -2
- package/bin/_lib_quota_model.py +96 -6
- package/bin/_lib_rebuild_progress.py +146 -0
- package/bin/_lib_record.py +22 -0
- package/bin/_lib_rederive.py +299 -3
- package/bin/_lib_snapshot_cache.py +43 -1
- package/bin/_lib_stale_proof.py +708 -0
- package/bin/_lib_statusline_candidates.py +65 -5
- package/bin/_lib_worker_launches.py +315 -0
- package/bin/cctally +6 -11
- package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
- package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
- package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
- package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
- package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
- package/dashboard/static/dashboard.html +1 -1
- package/package.json +7 -2
- package/bin/_lib_merge_gate.py +0 -452
- package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
|
@@ -38,13 +38,13 @@ Spec: docs/superpowers/specs/2026-05-30-extract-five-hour-statusline-cmd-design.
|
|
|
38
38
|
from __future__ import annotations
|
|
39
39
|
|
|
40
40
|
import argparse
|
|
41
|
-
import collections
|
|
42
41
|
import datetime as dt
|
|
43
42
|
import json
|
|
44
43
|
import sqlite3
|
|
45
44
|
import sys
|
|
46
45
|
|
|
47
46
|
import _lib_accounts # pure stdlib kernel; UNATTRIBUTED sentinel (#341)
|
|
47
|
+
import _lib_rebuild_progress # #997 D9 progress ticks (stdlib only)
|
|
48
48
|
from _lib_five_hour import latest_five_hour_reading
|
|
49
49
|
from _cctally_core import _command_as_of, eprint, now_utc_iso, open_db, parse_iso_datetime
|
|
50
50
|
|
|
@@ -989,69 +989,36 @@ def _latest_seven_day_and_window(
|
|
|
989
989
|
retired replica or a weekly-only reading can be the newest row, and its NULL
|
|
990
990
|
key would report the live block closed and leave its weekly end stale.
|
|
991
991
|
|
|
992
|
-
**The weekly value is
|
|
993
|
-
|
|
994
|
-
and
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
which is exactly the value a retirement band exists to suppress. A sub-one-point
|
|
1018
|
-
credit reaches it immediately, because such a credit is legal
|
|
1019
|
-
(`_doomed_snapshot_rows`) and places the post-credit synthetic inside a band
|
|
1020
|
-
centred on ``from_pct``, leaving every post-floor row in the band. An
|
|
1021
|
-
unparseable capture stamp inside a credited week is treated as pre-floor for
|
|
1022
|
-
the same reason: the row cannot be placed relative to the floor, so it cannot
|
|
1023
|
-
be certified as able to state the current value.
|
|
1024
|
-
|
|
1025
|
-
*Retirement.* Within the scope and at or after the floor, a retired replica is
|
|
1026
|
-
skipped, and a retired value is therefore never returned.
|
|
1027
|
-
|
|
1028
|
-
The floor bound applies to the NEWEST row as well as to the candidates below
|
|
1029
|
-
it, so a newest row captured before the latest floor answers ``None`` where the
|
|
1030
|
-
pre-#835 code answered its value. That state is unreachable through either
|
|
1031
|
-
production credit path: `record-credit` writes a post-credit synthetic snapshot
|
|
1032
|
-
at its own effective instant, and the automatic path's floor is derived from a
|
|
1033
|
-
capture that is itself in the store, so in both cases at least one row exists at
|
|
1034
|
-
or after the floor and it is the newest.
|
|
1035
|
-
|
|
1036
|
-
An UNCREDITED week resolves no bands, so there is no floor, nothing is skipped,
|
|
1037
|
-
and the read returns the newest row's value exactly as it did before #835. A
|
|
1038
|
-
``NULL`` weekly value is not a retired replica, so it still terminates the walk
|
|
1039
|
-
and still answers ``None``. Both contracts are byte-stable against today.
|
|
1040
|
-
|
|
1041
|
-
**The residual, deliberately unfixed and filed as #840.** A genuine later climb
|
|
1042
|
-
back INTO the band is still classified as a replay, so the weekly end reads the
|
|
1043
|
-
newest out-of-band value until the true value passes the band's upper edge.
|
|
1044
|
-
That is preferred over returning a value a credit retired, and no sound
|
|
1045
|
-
discriminator is available: `record-credit`'s own synthetic post-credit
|
|
1046
|
-
snapshot is written before any later replay, so "a below-band row has been
|
|
1047
|
-
observed" would close the band immediately and let every replay through.
|
|
1048
|
-
`test_840_characterization_a_genuine_climb_back_reads_the_older_value`
|
|
1049
|
-
characterizes it.
|
|
992
|
+
**The weekly value is the newest row's STORED value** (#840, spec section
|
|
993
|
+
4.8). Held-inclusive, because a held row stores the effective weekly value
|
|
994
|
+
and that is what a block's weekly end means. #835 filtered it through a
|
|
995
|
+
retirement band (a value within a point of one a credit retired was skipped
|
|
996
|
+
as a replay), which also withheld every genuine climb back to that value;
|
|
997
|
+
the band is removed. Stale contributors are refused before publication
|
|
998
|
+
(spec section 4.4), and a value already poisoned in a store is corrected by
|
|
999
|
+
`db rederive`'s reviewed `retract`, never by a percentage heuristic.
|
|
1000
|
+
|
|
1001
|
+
*Scope.* The value is the NEWEST row's, within its own
|
|
1002
|
+
``(week_start_date, account_key)``; a read never reports another week's or
|
|
1003
|
+
another account's percentage.
|
|
1004
|
+
|
|
1005
|
+
*Floor.* The read answers ``None`` when that row was captured BEFORE the
|
|
1006
|
+
latest credit floor in its scope, or carries an unparseable capture stamp
|
|
1007
|
+
inside a credited week: such a row cannot state the week's current
|
|
1008
|
+
effective weekly value, and its value is by construction the pre-credit
|
|
1009
|
+
reading. That state is unreachable through either production credit path:
|
|
1010
|
+
`record-credit` writes a post-credit synthetic snapshot at its own effective
|
|
1011
|
+
instant, and the automatic path's floor is derived from a capture that is
|
|
1012
|
+
itself in the store, so in both cases at least one row exists at or after
|
|
1013
|
+
the floor and it is the newest.
|
|
1014
|
+
|
|
1015
|
+
An UNCREDITED week resolves no floor, and the read returns the newest row's
|
|
1016
|
+
value. A ``NULL`` weekly value answers ``None``.
|
|
1050
1017
|
|
|
1051
1018
|
``account_key`` (#341, spec §3): ``None`` = merged / byte-stable; a real key /
|
|
1052
|
-
``unattributed`` scopes both selections to that account. The
|
|
1053
|
-
|
|
1054
|
-
|
|
1019
|
+
``unattributed`` scopes both selections to that account. The credit floor is
|
|
1020
|
+
resolved against the NEWEST ROW's own account and week, because a credit
|
|
1021
|
+
under one account bounds nothing under another.
|
|
1055
1022
|
"""
|
|
1056
1023
|
_c = _cctally()
|
|
1057
1024
|
acct_pred = "" if account_key is None else " WHERE account_key = ?"
|
|
@@ -1076,43 +1043,23 @@ def _latest_seven_day_and_window(
|
|
|
1076
1043
|
# with this account's weekly value.
|
|
1077
1044
|
reading = latest_five_hour_reading(conn, account_key=row["account_key"])
|
|
1078
1045
|
key = None if reading is None else reading.window_key
|
|
1079
|
-
# Axis 2: the weekly value
|
|
1080
|
-
#
|
|
1081
|
-
# leave that scope — so there is nothing to memoize per candidate.
|
|
1082
|
-
#
|
|
1083
|
-
# Dropping `LIMIT 1` does cost something at the storage layer: SQLite
|
|
1084
|
-
# materializes the full sort before yielding the first row, measured at
|
|
1085
|
-
# 9.0ms against 3.4ms on a 25,053-row store. The cost is immaterial
|
|
1086
|
-
# because this runs once per invocation, but the earlier claim that the
|
|
1087
|
-
# ordinary case "reads exactly one row, which is the work the single-row
|
|
1088
|
-
# query did" described only the rows the cursor yields, not the work the
|
|
1089
|
-
# query does, and was wrong.
|
|
1090
|
-
scope = (row["week_start_date"], row["account_key"])
|
|
1046
|
+
# Axis 2: the weekly value, the newest row's stored value, bounded by
|
|
1047
|
+
# the latest credit floor of its own week and account (#840).
|
|
1091
1048
|
bands = _c._credit_retirement_bands(
|
|
1092
1049
|
conn, week_start_date=row["week_start_date"],
|
|
1093
1050
|
week_start_at=row["week_start_at"],
|
|
1094
1051
|
week_end_at=row["week_end_at"],
|
|
1095
1052
|
account_key=row["account_key"])
|
|
1096
1053
|
floor = _c._latest_credit_floor_instant(bands)
|
|
1097
|
-
pct =
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
captured = None
|
|
1107
|
-
if captured is None or captured < floor:
|
|
1108
|
-
break
|
|
1109
|
-
if not _c._weekly_value_is_retired_replica(
|
|
1110
|
-
bands,
|
|
1111
|
-
captured_at_utc=row["captured_at_utc"],
|
|
1112
|
-
weekly_percent=row["weekly_percent"]):
|
|
1113
|
-
pct = row["weekly_percent"]
|
|
1114
|
-
break
|
|
1115
|
-
row = cur.fetchone()
|
|
1054
|
+
pct = row["weekly_percent"]
|
|
1055
|
+
if floor is not None:
|
|
1056
|
+
try:
|
|
1057
|
+
captured = parse_iso_datetime(
|
|
1058
|
+
row["captured_at_utc"], "captured_at").timestamp()
|
|
1059
|
+
except (ValueError, TypeError):
|
|
1060
|
+
captured = None
|
|
1061
|
+
if captured is None or captured < floor:
|
|
1062
|
+
pct = None
|
|
1116
1063
|
except sqlite3.DatabaseError:
|
|
1117
1064
|
return None, None
|
|
1118
1065
|
return (
|
|
@@ -1121,882 +1068,30 @@ def _latest_seven_day_and_window(
|
|
|
1121
1068
|
)
|
|
1122
1069
|
|
|
1123
1070
|
|
|
1124
|
-
def _parsed_block_instant(block, column):
|
|
1125
|
-
"""``block[column]`` when it is present and parses as an instant, else ``None``.
|
|
1126
|
-
|
|
1127
|
-
#834 S1 (#835) Gate A R8. Both observation stamps are declared `TEXT NOT NULL`,
|
|
1128
|
-
so absence is not a live shape — but a fixture, a hand-built store or a legacy
|
|
1129
|
-
projection can still carry an unparseable one, and the two instant helpers
|
|
1130
|
-
below must degrade rather than raise.
|
|
1131
|
-
"""
|
|
1132
|
-
value = block.get(column)
|
|
1133
|
-
if not value:
|
|
1134
|
-
return None
|
|
1135
|
-
try:
|
|
1136
|
-
parse_iso_datetime(value, column)
|
|
1137
|
-
except (ValueError, TypeError):
|
|
1138
|
-
return None
|
|
1139
|
-
return value
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
def _block_weekly_start_instant(block):
|
|
1143
|
-
"""The instant a block's stored weekly START was captured at, as ISO text.
|
|
1144
|
-
|
|
1145
|
-
#834 S1 (#835) Gate A R8. `first_observed_at_utc`, because that is the stamp
|
|
1146
|
-
of the tick that wrote `seven_day_pct_at_block_start`. The live upsert in
|
|
1147
|
-
bin/_cctally_record.py sets both from one fold dict on the INSERT (both are
|
|
1148
|
-
omitted from its `DO UPDATE`, so the first writer owns the pair), and
|
|
1149
|
-
`_backfill_five_hour_blocks` takes `first_obs` and `pct_start_7d` from its
|
|
1150
|
-
single MIN-captured snapshot row.
|
|
1151
|
-
|
|
1152
|
-
NOT `block_start_at`, which is the window's NOMINAL start — `five_hour_resets_at`
|
|
1153
|
-
minus five hours — and therefore at or before the first tick. The gap is not
|
|
1154
|
-
cosmetic: a block whose nominal start precedes a credit floor but whose first
|
|
1155
|
-
tick landed after it would have the floor's retired value compared against a
|
|
1156
|
-
PRE-floor instant and rendered. It falls back to `block_start_at` only when the
|
|
1157
|
-
observation stamp is absent or unparseable, which keeps the pre-R8 disposition
|
|
1158
|
-
rather than blanking the axis.
|
|
1159
|
-
"""
|
|
1160
|
-
return (_parsed_block_instant(block, "first_observed_at_utc")
|
|
1161
|
-
or block.get("block_start_at"))
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
def _block_weekly_end_instant(block, *, now_utc):
|
|
1165
|
-
"""The instant a block's stored weekly END was captured at, as an ISO text.
|
|
1166
|
-
|
|
1167
|
-
#834 S1 (#835) Gate A R8. `last_observed_at_utc`, because that is the stamp of
|
|
1168
|
-
the tick that wrote `seven_day_pct_at_block_end`. All four writers of the pair
|
|
1169
|
-
move the two together: the live upsert assigns
|
|
1170
|
-
`last_observed_at_utc = excluded.last_observed_at_utc` and
|
|
1171
|
-
`seven_day_pct_at_block_end = excluded.seven_day_pct_at_block_end` in one
|
|
1172
|
-
`DO UPDATE SET` from one fold dict, `_backfill_five_hour_blocks` takes
|
|
1173
|
-
`last_obs` and `pct_end_7d` from its single MAX-captured snapshot row,
|
|
1174
|
-
`five_hour_block_close` freezes the whole row into one evt, and
|
|
1175
|
-
`_migration_merge_5h_block_duplicates_v1` (bin/_cctally_db.py) copies both
|
|
1176
|
-
from the group row whose `last_observed_at_utc` is MAX.
|
|
1177
|
-
|
|
1178
|
-
Gate A S1 used `five_hour_resets_at` instead, falling back to the current
|
|
1179
|
-
instant while that reset was still in the future. That is the block's END, not
|
|
1180
|
-
a capture instant, and it was wrong in BOTH directions. A crossed-reset block's
|
|
1181
|
-
reset falls in the SUCCESSOR week, so `_block_weekly_axis_weeks` resolved the
|
|
1182
|
-
successor's bounds and the credited week's bands were never consulted. A closed
|
|
1183
|
-
block whose last tick predates the floor had its genuinely effective end
|
|
1184
|
-
withheld, which contradicts the historical-truth rule this filter rests on.
|
|
1185
|
-
|
|
1186
|
-
The reset-or-now form is KEPT as the fallback for an absent or unparseable
|
|
1187
|
-
stamp, so a store that cannot supply a capture instant behaves as it did
|
|
1188
|
-
before R8 rather than losing the axis.
|
|
1189
|
-
"""
|
|
1190
|
-
_c = _cctally()
|
|
1191
|
-
observed = _parsed_block_instant(block, "last_observed_at_utc")
|
|
1192
|
-
if observed is not None:
|
|
1193
|
-
return observed
|
|
1194
|
-
now_iso = _c._iso_z(now_utc)
|
|
1195
|
-
resets = block.get("five_hour_resets_at")
|
|
1196
|
-
if not resets:
|
|
1197
|
-
return now_iso
|
|
1198
|
-
try:
|
|
1199
|
-
moment = parse_iso_datetime(resets, "five_hour_resets_at").timestamp()
|
|
1200
|
-
except (ValueError, TypeError):
|
|
1201
|
-
return now_iso
|
|
1202
|
-
return resets if moment <= now_utc.timestamp() else now_iso
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
#: One candidate week for `_block_weekly_axis_weeks`, as loaded once per account
|
|
1206
|
-
#: per render. ``lo`` / ``hi`` are the parsed `[week_start_at, week_end_at)`
|
|
1207
|
-
#: epochs, or ``None`` when the week carries no usable bounds.
|
|
1208
|
-
_AxisWeek = collections.namedtuple(
|
|
1209
|
-
"_AxisWeek",
|
|
1210
|
-
"week_start_date week_end_date week_start_at week_end_at lo hi")
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
def _parsed_week_bounds(week_start_at, week_end_at):
|
|
1214
|
-
"""``(lo, hi)`` epochs for a week's bounds, or ``(None, None)``.
|
|
1215
|
-
|
|
1216
|
-
#834 S1 (#835) Gate A R9. Both loaders need the same verdict on the same two
|
|
1217
|
-
nullable columns — `_account_axis_weeks` to decide whether a week is placed by
|
|
1218
|
-
epoch or by date, `_account_axis_capture_weeks` to decide whether the identity
|
|
1219
|
-
it returns carries bounds for the `week_reset_events` leg — so the parse lives
|
|
1220
|
-
in one place and the two cannot disagree about which weeks have usable bounds.
|
|
1221
|
-
Absent or unparseable on either side yields no bounds, which is what
|
|
1222
|
-
`_credit_retirement_bands` already treats as a leg it cannot scope.
|
|
1223
|
-
"""
|
|
1224
|
-
if week_start_at is None or week_end_at is None:
|
|
1225
|
-
return None, None
|
|
1226
|
-
try:
|
|
1227
|
-
return (parse_iso_datetime(week_start_at, "week_start_at").timestamp(),
|
|
1228
|
-
parse_iso_datetime(week_end_at, "week_end_at").timestamp())
|
|
1229
|
-
except (ValueError, TypeError):
|
|
1230
|
-
return None, None
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
def _account_axis_weeks(conn, *, account_key, cache):
|
|
1234
|
-
"""Every distinct week ``account_key``'s snapshot rows name, in no order.
|
|
1235
|
-
|
|
1236
|
-
#834 S1 (#835) Gate A R8. ONE query per account per render, replacing one query
|
|
1237
|
-
per axis instant. The per-instant form was measured at 22x the whole render's
|
|
1238
|
-
prior cost on a 25,000-row store with 50 blocks, because its
|
|
1239
|
-
``unixepoch(week_start_at)`` predicate makes `idx_usage_week_start_at_time`
|
|
1240
|
-
unusable — `EXPLAIN QUERY PLAN` reported `SCAN weekly_usage_snapshots` plus
|
|
1241
|
-
`USE TEMP B-TREE FOR ORDER BY`, and the 50-block render performed about 51 full
|
|
1242
|
-
scans and 51 full sorts. An instant-keyed memo could not collapse them: two
|
|
1243
|
-
instants per block are two distinct keys.
|
|
1244
|
-
|
|
1245
|
-
THE WIN IS THE COLLAPSE TO ONE QUERY, NOT RESTORED INDEX USE. This query is
|
|
1246
|
-
equally unindexed: `EXPLAIN QUERY PLAN` reports `SCAN weekly_usage_snapshots`
|
|
1247
|
-
plus a temp B-tree. About 51 unindexed scans became one unindexed scan, which is
|
|
1248
|
-
the whole of the improvement; do not read `idx_usage_week_start_at_time` as being
|
|
1249
|
-
back in play. The scan verdict was measured on an empty store, on a 25,500-row
|
|
1250
|
-
store and again after `ANALYZE` while this query was still a `GROUP BY`; R10
|
|
1251
|
-
replaced that clause with `SELECT DISTINCT`, which changed the B-tree's stated
|
|
1252
|
-
purpose and not the scan (see the R10 paragraph below for the measured plan).
|
|
1253
|
-
|
|
1254
|
-
THE RESIDUAL IS NOT IRREDUCIBLE, and it is accepted as shipped rather than
|
|
1255
|
-
defended as minimal. `idx_usage_week_time` leads on `week_start_date`, so
|
|
1256
|
-
narrowing the projection to that column alone does use it — `SELECT DISTINCT
|
|
1257
|
-
week_start_date` reports `SCAN weekly_usage_snapshots USING INDEX
|
|
1258
|
-
idx_usage_week_time`, measured on the same two stores, and an earlier form said
|
|
1259
|
-
"grouping on that column alone", which is stale terminology since R10 replaced
|
|
1260
|
-
the `GROUP BY` with `SELECT DISTINCT` — and a render only
|
|
1261
|
-
needs the weeks overlapping its blocks' instant range rather than the account's
|
|
1262
|
-
whole history. Both are real reductions and neither is taken, because the
|
|
1263
|
-
property that matters is that the cost is CONSTANT in the number of rendered
|
|
1264
|
-
blocks, which it already is. `week_end_date` and the two bounds are in the
|
|
1265
|
-
projection because the date pass needs all four, which is exactly what a
|
|
1266
|
-
`week_start_date`-only projection would give up.
|
|
1267
|
-
|
|
1268
|
-
ONE COMPARISON SEMANTIC CHANGED WITH THE FORM, unreachably. The per-instant
|
|
1269
|
-
predicate compared SQLite `unixepoch()` integers, which truncate a fractional
|
|
1270
|
-
second; this form compares `parse_iso_datetime(...).timestamp()` floats, which
|
|
1271
|
-
keep it. The two disagree for a week bound carrying a fractional second: a
|
|
1272
|
-
`week_start_at` of `…T00:00:00.500000+00:00` against a probe at
|
|
1273
|
-
`…T00:00:00+00:00` is INSIDE the week under the integer compare and outside it
|
|
1274
|
-
under the float one. No writer can produce that bound.
|
|
1275
|
-
`_canonicalize_optional_iso` returns `isoformat(timespec="seconds")` over a
|
|
1276
|
-
`_normalize_week_boundary_dt` result, which zeroes second and microsecond and
|
|
1277
|
-
snaps to the hour, and `cmd_record_usage`'s pipeline derives both bounds from
|
|
1278
|
-
that same normalizer and writes them with the same `timespec="seconds"`.
|
|
1279
|
-
|
|
1280
|
-
THE SET IS UNORDERED, AND THE QUERY IS A PLAIN `SELECT DISTINCT` (Gate A R10).
|
|
1281
|
-
Through R9 it carried a `MAX(captured_at_utc || char(1) || printf('%020d', id))`
|
|
1282
|
-
aggregate whose SQL alias was `rk` and whose `_AxisWeek` field was `rank` — call
|
|
1283
|
-
it `rk`, because that is the alias, and earlier text here called the column
|
|
1284
|
-
`rank` — introduced to reproduce the per-instant query's
|
|
1285
|
-
``ORDER BY captured_at_utc DESC, id DESC`` tie-break exactly so that which week
|
|
1286
|
-
won an overlap did not change with the query's form, and kept as documentary
|
|
1287
|
-
after R9 deleted the tie-break. Nothing read it: this function has one caller,
|
|
1288
|
-
every pass in `_block_weekly_axis_weeks` builds an order-insensitive set, and
|
|
1289
|
-
the caller ORs a predicate over the result. The grouping was there only to carry
|
|
1290
|
-
the aggregate, so both went.
|
|
1291
|
-
|
|
1292
|
-
NOTHING PINS THE ORDER NOW. There was never a SQL `ORDER BY` on this query: the
|
|
1293
|
-
order came from a Python `weeks.sort(key=lambda w: w.rank, reverse=True)` over the
|
|
1294
|
-
aggregate. Through R8 a consumer tie-break in the epoch pass read it; R9 deleted
|
|
1295
|
-
that tie-break and left the aggregate documentary; R10 deleted the aggregate and
|
|
1296
|
-
the sort together. So the returned order is now whatever SQLite produces, and no
|
|
1297
|
-
test asserts order-insensitivity of the consumer, so a future consumer that cares
|
|
1298
|
-
about order has nothing to fail against. It is safe today because the one consumer
|
|
1299
|
-
ORs a monotone band predicate over the whole set. A consumer that needs an order
|
|
1300
|
-
must establish it itself rather than assume this one.
|
|
1301
|
-
|
|
1302
|
-
The statement-count test cannot see R10's REMOVAL of the aggregate and the
|
|
1303
|
-
grouping, because the statement count is unchanged by it; the removal was
|
|
1304
|
-
measured instead, best of five on a 25,194-row 13-week 50-block store, with the
|
|
1305
|
-
order of the three forms reversed to rule out page-cache bias and the three
|
|
1306
|
-
result sets asserted identical: 13.57 ms with the aggregate, 9.36 ms for the
|
|
1307
|
-
bare `GROUP BY`, 2.34 ms for `SELECT DISTINCT` — so removing both took about 83%
|
|
1308
|
-
off the statement, and the plan changed from `SCAN weekly_usage_snapshots` +
|
|
1309
|
-
`USE TEMP B-TREE FOR GROUP BY` to the same scan + `USE TEMP B-TREE FOR DISTINCT`.
|
|
1310
|
-
It is still an unindexed scan; the `idx_usage_week_time` reduction named above is
|
|
1311
|
-
still not taken, for the reason given there.
|
|
1312
|
-
|
|
1313
|
-
DO NOT QUOTE A SPLIT BETWEEN THE TWO REMOVALS. On those figures the aggregate is
|
|
1314
|
-
31% of the statement and the grouping a further 52%, but the review that checked
|
|
1315
|
-
this measured 12.87-13.09 / 6.30-6.77 / 2.02-2.23 ms on its own 25,194-row store,
|
|
1316
|
-
which makes the aggregate about 50% and the grouping about 34% — the same 83%
|
|
1317
|
-
total, with the two components in the opposite order. The total reproduces and
|
|
1318
|
-
the ranking does not, so treat the split as machine- and data-dependent.
|
|
1319
|
-
|
|
1320
|
-
`week_start_at` / `week_end_at` are both nullable and
|
|
1321
|
-
`_derive_week_from_payload` leaves them `None` on three of its four paths, so
|
|
1322
|
-
a week with no bounds is a live shape; it is retained here with ``lo``/``hi``
|
|
1323
|
-
of ``None`` and placed by date instead (see `_block_weekly_axis_weeks`).
|
|
1324
|
-
"""
|
|
1325
|
-
if account_key in cache:
|
|
1326
|
-
return cache[account_key]
|
|
1327
|
-
try:
|
|
1328
|
-
rows = conn.execute(
|
|
1329
|
-
"SELECT DISTINCT week_start_date, week_end_date, week_start_at, "
|
|
1330
|
-
" week_end_at "
|
|
1331
|
-
" FROM weekly_usage_snapshots "
|
|
1332
|
-
" WHERE account_key = ?",
|
|
1333
|
-
(account_key,),
|
|
1334
|
-
).fetchall()
|
|
1335
|
-
except sqlite3.DatabaseError:
|
|
1336
|
-
rows = []
|
|
1337
|
-
weeks: list = []
|
|
1338
|
-
for r in rows:
|
|
1339
|
-
lo, hi = _parsed_week_bounds(r["week_start_at"], r["week_end_at"])
|
|
1340
|
-
weeks.append(_AxisWeek(
|
|
1341
|
-
week_start_date=r["week_start_date"],
|
|
1342
|
-
week_end_date=r["week_end_date"],
|
|
1343
|
-
week_start_at=r["week_start_at"] if lo is not None else None,
|
|
1344
|
-
week_end_at=r["week_end_at"] if hi is not None else None,
|
|
1345
|
-
lo=lo, hi=hi))
|
|
1346
|
-
cache[account_key] = weeks
|
|
1347
|
-
return weeks
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
def _account_axis_capture_weeks(conn, *, account_key, cache):
|
|
1351
|
-
"""``{capture second -> the week identities a snapshot row at that instant
|
|
1352
|
-
names}``, for the capture instants ``account_key``'s blocks actually carry.
|
|
1353
|
-
|
|
1354
|
-
#834 S1 (#835) Gate A R9. THE STORE KEEPS A RECORD OF WHICH WEEK A CAPTURE WAS
|
|
1355
|
-
OBSERVED UNDER, and this is the read of it. Round five documented its
|
|
1356
|
-
over-withholding on the premise that it does not. All four writers of
|
|
1357
|
-
`five_hour_blocks.seven_day_pct_at_block_start` / `_at_block_end` take the
|
|
1358
|
-
matching axis STAMP from a `weekly_usage_snapshots` row's `captured_at_utc`:
|
|
1359
|
-
the live upsert passes `captured_at = saved["capturedAt"]` into
|
|
1360
|
-
`first_observed_at_utc` and `last_observed_at_utc` in one `INSERT … DO UPDATE`
|
|
1361
|
-
and that value IS the row's `captured_at_utc` (bin/_cctally_record.py:2351,
|
|
1362
|
-
:2633-2634, :6828), `_backfill_five_hour_blocks` takes them from its MIN- and
|
|
1363
|
-
MAX-captured rows, `five_hour_block_close` freezes the row, and
|
|
1364
|
-
`_migration_merge_5h_block_duplicates_v1` copies from the MAX-captured group
|
|
1365
|
-
row. Every snapshot row carries all four week columns. So a row whose
|
|
1366
|
-
`captured_at_utc` equals an axis instant NAMES the week in force at that
|
|
1367
|
-
capture, with no date-range guess involved.
|
|
1368
|
-
|
|
1369
|
-
HELD ROWS ARE NOT EXCLUDED, which is the one judgement call in this function.
|
|
1370
|
-
The rest of this tranche excludes `weekly_observation_held = 1` from weekly
|
|
1371
|
-
VALUE reads, because a held row carries a reading no fresh observation
|
|
1372
|
-
confirmed. This function reads no value: it reads which week the value on the
|
|
1373
|
-
block's axis came from, and on a held tick that is the BASIS week by
|
|
1374
|
-
construction — the held write copies the basis's four week columns verbatim
|
|
1375
|
-
while keeping the tick's own `capture_at` (bin/_cctally_record.py:7190-7203),
|
|
1376
|
-
and the block's axis takes `fold.weekly.effective_pct`, the same
|
|
1377
|
-
carried-forward value, stamped with that same `capture_at`. The held row is
|
|
1378
|
-
therefore the provenance record of the number on the axis, and the credits
|
|
1379
|
-
that retired that number are the basis week's.
|
|
1380
|
-
|
|
1381
|
-
Two consequences follow. A held row is exactly how a pre-credit weekly value
|
|
1382
|
-
reaches a post-credit capture instant, which is the case this whole tranche
|
|
1383
|
-
exists to handle, and excluding it would drop the only exact evidence of which
|
|
1384
|
-
week's floor applies. And because no predicate names
|
|
1385
|
-
`weekly_observation_held`, a store that stops short of epoch 1015 is served
|
|
1386
|
-
unchanged and `weekly_held_exclusion` is not needed here. Pinned by
|
|
1387
|
-
`tests/test_five_hour_breakdown.py::test_835_r9_a_held_capture_record_names_the_basis_week_it_carried`
|
|
1388
|
-
and `::test_835_r9_a_pre_column_store_resolves_the_capture_record`.
|
|
1389
|
-
|
|
1390
|
-
THE SET IS BOUNDED BY THE ACCOUNT'S BLOCK COUNT, NOT BY ITS SNAPSHOT HISTORY,
|
|
1391
|
-
and the form was chosen on measurement rather than on shape. Re-measured at Gate
|
|
1392
|
-
A R10, because R10 replaced `_account_axis_weeks`' grouped query with a plain
|
|
1393
|
-
`SELECT DISTINCT` and the earlier round's figures for that statement no longer
|
|
1394
|
-
described what ships. All three were timed in ONE session on a 25,194-row,
|
|
1395
|
-
13-week, 50-block store — the store `_account_axis_weeks` documents its own
|
|
1396
|
-
numbers against — best of five each, with the order reversed to rule out
|
|
1397
|
-
page-cache bias:
|
|
1398
|
-
|
|
1399
|
-
- this narrowed query: 2.10-2.27 ms, 50 rows, `SCAN weekly_usage_snapshots`
|
|
1400
|
-
plus a `LIST SUBQUERY` that SEARCHes `five_hour_blocks` on its own index;
|
|
1401
|
-
- `_account_axis_weeks`' `SELECT DISTINCT`: 2.06-2.13 ms, 13 rows;
|
|
1402
|
-
- one ungrouped full-row scan serving BOTH passes from a single statement:
|
|
1403
|
-
14.18-14.38 ms, 25,194 rows, a bare `SCAN weekly_usage_snapshots`.
|
|
1404
|
-
|
|
1405
|
-
So the two statements together cost 4.40 ms against the single statement's
|
|
1406
|
-
14.18 ms and retain 63 rows against 25,194 — a 400th. Two statements per account
|
|
1407
|
-
per render is the cheaper form on both axes, and it keeps the property that
|
|
1408
|
-
matters: the cost is CONSTANT in the number of rendered blocks. Bounded by the
|
|
1409
|
-
account's own `five_hour_blocks` rows rather than by which of them this render
|
|
1410
|
-
shows, which is why no instant range has to be threaded down from the caller.
|
|
1411
|
-
|
|
1412
|
-
TWO EARLIER ROUNDS DISAGREED ABOUT THE SINGLE-STATEMENT FIGURE — this docstring
|
|
1413
|
-
said 22.03 ms and `docs/five-hour-gotchas.md` said 21.97 ms for the same query on
|
|
1414
|
-
the same store — and neither number survives, because both were taken against the
|
|
1415
|
-
superseded grouped form of the week query. The figures above replace both rather
|
|
1416
|
-
than reconciling them.
|
|
1417
|
-
|
|
1418
|
-
THE NARROWING AND THE MATCH USE DIFFERENT READERS, deliberately. SQLite's
|
|
1419
|
-
`unixepoch()` selects the rows; `parse_iso_datetime(...).timestamp()` keys
|
|
1420
|
-
them and keys the probe, so the match itself is Python's and is consistent
|
|
1421
|
-
between map and probe. Both readers truncate to the second, which is how two
|
|
1422
|
-
ISO spellings of one moment — `Z` against `+00:00` — compare equal.
|
|
1423
|
-
|
|
1424
|
-
A NAIVE STAMP IS A LIVE SHAPE, and the two readings of it differ. An earlier
|
|
1425
|
-
form of this paragraph said no writer produces one, citing `now_utc_iso()`.
|
|
1426
|
-
That is wrong: `_coerce_payload_captured_at` (bin/_cctally_record.py:6576-6585)
|
|
1427
|
-
returns the PAYLOAD's `capturedAt` verbatim whenever `parse_iso_datetime`
|
|
1428
|
-
accepts it, `parse_iso_datetime` accepts a naive string and reads it host-local,
|
|
1429
|
-
and `insert_usage_snapshot` then stores that text. Driven with
|
|
1430
|
-
`TZ=America/Los_Angeles`: a payload `capturedAt` of `2026-04-30T13:00:00` is
|
|
1431
|
-
stored as `captured_at_utc = '2026-04-30T13:00:00'`, and `unixepoch()` reads it
|
|
1432
|
-
1777554000 against Python's 1777579200.
|
|
1433
|
-
|
|
1434
|
-
THE FAILURE IS A MIXED PAIR, NOT A NAIVE STAMP, AND IT IS THE MATCH THAT MISSES
|
|
1435
|
-
RATHER THAN THE NARROWING THAT EXCLUDES. The same earlier paragraph stated it
|
|
1436
|
-
the other way round. Each reader is self-consistent across the two sides it
|
|
1437
|
-
compares, so the spelling only matters when the snapshot stamp and the block
|
|
1438
|
-
stamp are spelled DIFFERENTLY. All four pairings, driven on that same non-UTC
|
|
1439
|
-
host: equal naive text narrows IN and the Python match HITS; `Z` against
|
|
1440
|
-
`+00:00` narrows IN and HITS; naive against offset-carrying, in either
|
|
1441
|
-
direction, narrows IN and the PYTHON MATCH MISSES, after which the instant falls
|
|
1442
|
-
through to the two fallback passes.
|
|
1443
|
-
|
|
1444
|
-
SO THE RESIDUAL IS NARROWER THAN A NAIVE STAMP'S EXISTENCE. Every one of the
|
|
1445
|
-
four axis writers takes the block stamp from a `weekly_usage_snapshots` row's
|
|
1446
|
-
own `captured_at_utc` text — the live upsert from `saved["capturedAt"]`, the
|
|
1447
|
-
backfill from its MIN- and MAX-captured rows, the close from the frozen row, the
|
|
1448
|
-
merge migration from the MAX-captured group row — so the pair the capture pass
|
|
1449
|
-
compares always shares a spelling and the pass is correct on a naive store. A
|
|
1450
|
-
mixed pair needs the two texts to diverge after both were written, which is a
|
|
1451
|
-
hand-built or externally-edited store. The degradation is then a fallback rather
|
|
1452
|
-
than a misclassification, which is the one part of the earlier paragraph that
|
|
1453
|
-
held.
|
|
1454
|
-
"""
|
|
1455
|
-
if account_key in cache:
|
|
1456
|
-
return cache[account_key]
|
|
1457
|
-
try:
|
|
1458
|
-
rows = conn.execute(
|
|
1459
|
-
"SELECT captured_at_utc, week_start_date, week_start_at, week_end_at "
|
|
1460
|
-
" FROM weekly_usage_snapshots "
|
|
1461
|
-
" WHERE account_key = ? "
|
|
1462
|
-
" AND unixepoch(captured_at_utc) IN ("
|
|
1463
|
-
" SELECT unixepoch(first_observed_at_utc) "
|
|
1464
|
-
" FROM five_hour_blocks WHERE account_key = ? "
|
|
1465
|
-
" UNION "
|
|
1466
|
-
" SELECT unixepoch(last_observed_at_utc) "
|
|
1467
|
-
" FROM five_hour_blocks WHERE account_key = ?)",
|
|
1468
|
-
(account_key, account_key, account_key),
|
|
1469
|
-
).fetchall()
|
|
1470
|
-
except sqlite3.DatabaseError:
|
|
1471
|
-
rows = []
|
|
1472
|
-
carried: dict = {}
|
|
1473
|
-
for r in rows:
|
|
1474
|
-
try:
|
|
1475
|
-
second = int(parse_iso_datetime(
|
|
1476
|
-
r["captured_at_utc"], "captured_at_utc").timestamp())
|
|
1477
|
-
except (ValueError, TypeError):
|
|
1478
|
-
continue
|
|
1479
|
-
lo, _hi = _parsed_week_bounds(r["week_start_at"], r["week_end_at"])
|
|
1480
|
-
identity = (
|
|
1481
|
-
r["week_start_date"],
|
|
1482
|
-
r["week_start_at"] if lo is not None else None,
|
|
1483
|
-
r["week_end_at"] if lo is not None else None,
|
|
1484
|
-
)
|
|
1485
|
-
# A dict as an ordered set: two rows of one week at one instant are one
|
|
1486
|
-
# identity, and `_credit_retirement_bands` would answer both alike.
|
|
1487
|
-
carried.setdefault(second, {})[identity] = None
|
|
1488
|
-
out = {second: tuple(ids) for second, ids in carried.items()}
|
|
1489
|
-
cache[account_key] = out
|
|
1490
|
-
return out
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
def _account_axis_bounds_by_start_date(weeks):
|
|
1494
|
-
"""``{week_start_date -> that week's bounds-carrying identities}``, from an
|
|
1495
|
-
already-loaded `_account_axis_weeks` set.
|
|
1496
|
-
|
|
1497
|
-
#834 S1 (#835) Gate A R10. The index the capture pass substitutes from when its
|
|
1498
|
-
matched row carries no bounds of its own. Derived from the account's OWN week
|
|
1499
|
-
set and nothing else, which is the property that separates it from
|
|
1500
|
-
`_get_canonical_boundary_for_date`; built once per account per render, so the
|
|
1501
|
-
substitution adds no query and no per-instant scan.
|
|
1502
|
-
|
|
1503
|
-
Bounds-free weeks are absent by construction: an entry exists only for a
|
|
1504
|
-
`week_start_date` some row of that account spells bounds for, which is exactly
|
|
1505
|
-
the condition under which a substitution is possible.
|
|
1506
|
-
"""
|
|
1507
|
-
out: dict = {}
|
|
1508
|
-
for week in weeks:
|
|
1509
|
-
if week.lo is None:
|
|
1510
|
-
continue
|
|
1511
|
-
out.setdefault(week.week_start_date, {})[
|
|
1512
|
-
(week.week_start_date, week.week_start_at, week.week_end_at)] = None
|
|
1513
|
-
return {date: tuple(ids) for date, ids in out.items()}
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
def _block_weekly_axis_weeks(conn, *, account_key, instant_iso, cache):
|
|
1517
|
-
"""Every week identity whose credits can have retired a value captured at
|
|
1518
|
-
``instant_iso`` under ``account_key``, as ``(week_start_date, week_start_at,
|
|
1519
|
-
week_end_at)`` triples. Empty when no week names the instant.
|
|
1520
|
-
|
|
1521
|
-
THREE PASSES, IN THIS ORDER: the CAPTURE pass matches the instant against the
|
|
1522
|
-
`captured_at_utc` of the account's own snapshot rows and answers from the week
|
|
1523
|
-
those rows name; the EPOCH pass places the instant inside a week's parsed
|
|
1524
|
-
`[week_start_at, week_end_at)` bounds; the DATE pass asks which bounds-free
|
|
1525
|
-
weeks declare the instant's calendar day. Each runs only when the one before it
|
|
1526
|
-
resolved nothing.
|
|
1527
|
-
|
|
1528
|
-
THE CAPTURE PASS IS EXACT ABOUT WHICH WEEK, AND IT REPLACED AN ARGUMENT THAT
|
|
1529
|
-
WAS WRONG. Gate A R8 had only the other two, and round five defended the
|
|
1530
|
-
over-withholding they cause on the premise that the store keeps no record of
|
|
1531
|
-
which week a capture was observed under. It keeps exactly that record: every
|
|
1532
|
-
axis stamp on a block IS some snapshot row's `captured_at_utc`, and that row
|
|
1533
|
-
carries all four week columns — see `_account_axis_capture_weeks`, which also
|
|
1534
|
-
records why held rows are consulted rather than excluded. So for an instant a
|
|
1535
|
-
capture record carries there is no week to guess, and neither fallback runs.
|
|
1536
|
-
|
|
1537
|
-
BEING EXACT ABOUT THE WEEK IS NOT THE SAME AS RESOLVING BOTH LEGS, which an
|
|
1538
|
-
earlier form of this paragraph left unsaid while the date pass's own paragraphs
|
|
1539
|
-
below state it carefully. `week_start_at` and `week_end_at` are nullable on the
|
|
1540
|
-
matched row too, and `_credit_retirement_bands`' `week_reset_events` leg is
|
|
1541
|
-
scoped purely by containment of `effective_reset_at_utc` in
|
|
1542
|
-
``[week_start_at, week_end_at)`` — that table has no `week_start_date` column at
|
|
1543
|
-
all — so an identity carrying no bounds resolves the `weekly_credit_floors` leg
|
|
1544
|
-
and NOT the automatic one. Through R9 that degradation combined with this pass's
|
|
1545
|
-
precedence into the tranche's own defect class: a bounds-free capture row
|
|
1546
|
-
pre-empted a bounds-carrying placement of the SAME week and the value an
|
|
1547
|
-
automatic credit retired was published. R10 closes it by SUBSTITUTING the
|
|
1548
|
-
account's own bounds (next paragraph), so the per-leg behaviour now is that both
|
|
1549
|
-
legs resolve whenever any row of the named week carries parseable bounds, and
|
|
1550
|
-
the manual leg alone when none does.
|
|
1551
|
-
|
|
1552
|
-
A BOUNDS-FREE CAPTURE IDENTITY IS REPLACED BY THE ACCOUNT'S OWN BOUNDS-CARRYING
|
|
1553
|
-
ROWS OF THE SAME `week_start_date`, and the substitution is lossless. Those rows
|
|
1554
|
-
are already in memory from `_account_axis_weeks`, so no query is added. It is
|
|
1555
|
-
lossless because every substituted identity shares the bounds-free one's
|
|
1556
|
-
`week_start_date`, so each resolves a `weekly_credit_floors` leg identical to the
|
|
1557
|
-
one it replaces and ADDS the automatic leg its bounds can scope; the replaced
|
|
1558
|
-
identity therefore contributes no band the substitutes do not. Several of them
|
|
1559
|
-
are unioned, the way the epoch pass unions overlapping candidates, and that
|
|
1560
|
-
union is here FOR ROBUSTNESS rather than for a reachable shape: no writer is
|
|
1561
|
-
known to put two bound pairs under one `week_start_date`. What pins one pair per
|
|
1562
|
-
date is `_usage_snapshot_columns` (bin/_cctally_record.py:6698-6710), which
|
|
1563
|
-
overrides every later row's derived bounds with `_get_canonical_boundary_for_date`
|
|
1564
|
-
— the FIRST established pair for that date, `ORDER BY captured_at_utc ASC, id ASC
|
|
1565
|
-
LIMIT 1` — so a sub-day re-anchoring is overridden back to the first pair, and a
|
|
1566
|
-
day-scale one lands under a DIFFERENT `week_start_date` because the date derives
|
|
1567
|
-
from the payload's own week start. No code change is owed either way, because
|
|
1568
|
-
`named.update(dict.fromkeys(… or (identity,)))` already handles one through N.
|
|
1569
|
-
Two earlier forms of this sentence cited a producer and both were false. Do NOT
|
|
1570
|
-
cite the timezone fork: it produces the INVERSE shape, identical bounds under two
|
|
1571
|
-
different `week_start_date` values (see the two-identities paragraph below),
|
|
1572
|
-
which puts one identity under each date rather than two under one. Do NOT cite
|
|
1573
|
-
the boundary-shift branch of `_backfill_week_reset_events`
|
|
1574
|
-
(bin/_cctally_weekrefs.py) either: it writes only `week_reset_events` and never
|
|
1575
|
-
the snapshot bounds this substitution reads, and its reset test needs THREE
|
|
1576
|
-
signals rather than the two an earlier form named — the boundary shifted, the
|
|
1577
|
-
prior boundary was still in the future, AND `_is_reset_drop(prior_pct, cur_pct,
|
|
1578
|
-
allow_reset_to_zero=False)`, which with the zero leg off is the >=25pp drop
|
|
1579
|
-
(`bin/_cctally_weekrefs.py:625-637`).
|
|
1580
|
-
|
|
1581
|
-
CONTAINMENT OF THE INSTANT IS DELIBERATELY NOT REQUIRED OF A SUBSTITUTE, and
|
|
1582
|
-
requiring it would reopen the defect on the shape this tranche exists for. A
|
|
1583
|
-
HELD row carries the BASIS week's four columns with the tick's own capture
|
|
1584
|
-
stamp, so a held tick after the week's end names a week whose bounds do not
|
|
1585
|
-
contain its own capture instant. Filtering substitutes by containment would
|
|
1586
|
-
decline to enrich exactly there.
|
|
1587
|
-
|
|
1588
|
-
THAT GENERALITY IS WIDER THAN THE HELD-ROW ARGUMENT NEEDS, and the excess is a
|
|
1589
|
-
stated limit rather than a defended choice. The held-row shape requires only that
|
|
1590
|
-
containment of the CAPTURE instant not be required. Nothing here additionally
|
|
1591
|
-
requires a substitute's bounds to be consistent with the `week_start_date` they
|
|
1592
|
-
are filed under, so a same-date row spelling bounds that start a week early lets
|
|
1593
|
-
the `week_reset_events` leg admit a reset belonging to the PREDECESSOR week, and
|
|
1594
|
-
the value is withheld although no credit of the named week retired it. No writer
|
|
1595
|
-
produces a date/bounds inconsistency of that size, so the shape needs a
|
|
1596
|
-
hand-built or externally-edited store; the narrower rule that would close it is
|
|
1597
|
-
to require the substitute's `week_start_at` UTC date to sit within a day of
|
|
1598
|
-
`week_start_date`, and it is not taken because the reachable benefit is nil and a
|
|
1599
|
-
date/bounds tolerance is one more constant to get wrong.
|
|
1600
|
-
|
|
1601
|
-
NOTHING WIDER THAN THE ACCOUNT'S OWN ROWS IS CONSULTED.
|
|
1602
|
-
`_get_canonical_boundary_for_date` (bin/_cctally_weekrefs.py:51) is the repo's
|
|
1603
|
-
canonical-bounds reader and the obvious candidate, and it carries no
|
|
1604
|
-
`account_key` predicate: it returns the EARLIEST bounds-carrying row for a
|
|
1605
|
-
`week_start_date` across every account. On a multi-account store that does not
|
|
1606
|
-
merely attach the wrong provenance, it fails to fix the defect, because another
|
|
1607
|
-
account's re-anchored bounds cannot scope THIS account's credit — driven and
|
|
1608
|
-
pinned by
|
|
1609
|
-
`tests/test_five_hour_breakdown.py::test_835_r10_the_substitution_takes_the_accounts_own_bounds_not_another_accounts`.
|
|
1610
|
-
|
|
1611
|
-
ONE RESIDUAL SURVIVES, AND IT IS PRE-EXISTING RATHER THAN INTENDED. When NO row
|
|
1612
|
-
of the named week carries parseable bounds there is nothing account-scoped to
|
|
1613
|
-
substitute, so an automatic credit in that week is structurally invisible to
|
|
1614
|
-
this filter and the value it retired is published. Closing it needs a source of
|
|
1615
|
-
a week's bounds outside the account's own snapshot rows, and the paragraph above
|
|
1616
|
-
says why the one that exists is not usable. Pinned, as a gap rather than as
|
|
1617
|
-
correct behaviour, by the second half of
|
|
1618
|
-
`::test_835_r10_a_bounds_free_capture_record_resolves_the_weeks_automatic_credit`.
|
|
1619
|
-
|
|
1620
|
-
A CAPTURE INSTANT CARRIED BY TWO WEEK IDENTITIES RETURNS BOTH. No tie-break is
|
|
1621
|
-
invented for it and the caller's union withholds if either week retired the
|
|
1622
|
-
value, which is this subsystem's established direction and what both fallback
|
|
1623
|
-
passes do as well. The shape is not hypothetical: bin/_cctally_record.py:6601-6614
|
|
1624
|
-
records a host briefly running the wrong timezone forking ONE physical week's
|
|
1625
|
-
`week_start_date` across 18 rows, and `weekly_credit_floors` matches
|
|
1626
|
-
`week_start_date` by equality, so a floor can sit under one fork and not the
|
|
1627
|
-
other.
|
|
1628
|
-
|
|
1629
|
-
#834 S1 (#835) Gate A S1. `five_hour_blocks` carries no week columns, and the
|
|
1630
|
-
retirement bands need all three: the `weekly_credit_floors` leg matches
|
|
1631
|
-
`week_start_date` by equality and the `week_reset_events` leg is scoped by the
|
|
1632
|
-
week's bounds. The week is therefore resolved from the account's own snapshot
|
|
1633
|
-
rows, whose `week_start_at` / `week_end_at` are the same bounds
|
|
1634
|
-
`_credit_retirement_bands` is written against.
|
|
1635
|
-
|
|
1636
|
-
The comparison is on PARSED epochs rather than on text, for the reason every
|
|
1637
|
-
SQL site in this area wraps both sides in ``unixepoch()``: `block_start_at` is
|
|
1638
|
-
host-local while `week_start_at` is `+00:00`, so a lexicographic compare would
|
|
1639
|
-
mis-order them.
|
|
1640
|
-
|
|
1641
|
-
THE TWO LEGS DEGRADE INDEPENDENTLY, the way `_credit_retirement_bands`' own two
|
|
1642
|
-
legs already do (#834 S1 Gate A R8). `week_start_at` and `week_end_at` are both
|
|
1643
|
-
nullable and `_derive_week_from_payload` leaves them ``None`` on three of its
|
|
1644
|
-
four paths, so a week with no bounds is a live shape. Gate A S1 required both
|
|
1645
|
-
bounds and resolved NOTHING without them, which threw away a leg it could still
|
|
1646
|
-
answer: the `weekly_credit_floors` leg matches on `week_start_date` alone, and
|
|
1647
|
-
`week_start_date` / `week_end_date` are `TEXT NOT NULL`. So a bounds-free week
|
|
1648
|
-
is placed by DATE instead and returned with ``None`` bounds, which resolves the
|
|
1649
|
-
manual-credit leg and skips the `week_reset_events` leg — exactly what
|
|
1650
|
-
`_credit_retirement_bands` does with absent bounds.
|
|
1651
|
-
|
|
1652
|
-
A bounds-carrying week outranks a bounds-free one, so the date pass runs only
|
|
1653
|
-
when the epoch pass places the instant nowhere. The
|
|
1654
|
-
date window is INCLUSIVE of `week_end_date`, because the usual bounds-free week
|
|
1655
|
-
carries the week's LAST DAY there: `_derive_week_from_payload`'s third path
|
|
1656
|
-
writes `week_start_date + 6 days` and its fourth derives the same through
|
|
1657
|
-
`compute_week_bounds`. THE TWO READINGS DO MEET, which an earlier form of this
|
|
1658
|
-
docstring denied. `_derive_week_from_payload`'s second path takes both dates
|
|
1659
|
-
from the payload verbatim behind nothing but a `weekEndDate >= weekStartDate`
|
|
1660
|
-
check, and `_account_axis_weeks` sends a week down this pass when its bounds are
|
|
1661
|
-
present but UNPARSEABLE, where `week_end_date` came from the bounds-carrying
|
|
1662
|
-
writer and is the exclusive boundary's date by construction. Either shape puts
|
|
1663
|
-
one calendar day inside two weeks' inclusive ranges.
|
|
1664
|
-
|
|
1665
|
-
SO THE DATE PASS RETURNS EVERY CANDIDATE THAT CONTAINS THE DAY AND PICKS NONE,
|
|
1666
|
-
and the caller unions their bands. Two single-choice rules were tried and each
|
|
1667
|
-
published a retired value on the mirror image of the shape it fixed. Rank order
|
|
1668
|
-
placed an instant in the PREDECESSOR whenever the predecessor's newest snapshot
|
|
1669
|
-
row was the newer of the two. Preferring the greatest `week_start_date` fixed
|
|
1670
|
-
that and broke the case where the floor sits on a LONGER declared week
|
|
1671
|
-
`2026-04-20..2026-05-04` beside an uncredited `2026-04-27..2026-05-04` — the
|
|
1672
|
-
exact shape whose existence is the argument against a seven-day window, so the
|
|
1673
|
-
prohibition and that repair rested on one premise pointing opposite ways. No
|
|
1674
|
-
reading of one week's own columns separates two candidates that both declare the
|
|
1675
|
-
day, so choosing is a guess in either direction and consulting all of them is
|
|
1676
|
-
not.
|
|
1677
|
-
|
|
1678
|
-
NEITHER FALLBACK PASS HAS A TIE-BREAK ANY MORE. Both return every candidate they
|
|
1679
|
-
place the instant in, and the caller unions the bands. Through Gate A R8 the
|
|
1680
|
-
epoch pass returned ONE week — the one whose group held the newest
|
|
1681
|
-
`(captured_at_utc, id)`, which is what the since-removed `rk` aggregate ordered
|
|
1682
|
-
by — justified by the claim that the newest anchoring is the best available
|
|
1683
|
-
evidence of which window was in force at capture. That claim is vacuous on the shape that makes overlapping
|
|
1684
|
-
bounds-carrying anchorings reachable at all: bin/_cctally_record.py:6601-6614
|
|
1685
|
-
records a host running the wrong timezone for seven minutes forking ONE physical
|
|
1686
|
-
week's `week_start_date` across 18 rows, and those two anchorings carry
|
|
1687
|
-
IDENTICAL bounds and differ only in the column `weekly_credit_floors` is keyed
|
|
1688
|
-
on. Both anchorings are the same window, so there is no "which window was in
|
|
1689
|
-
force" to answer, and a newest-anchoring choice decides only which of two
|
|
1690
|
-
spellings of one week's floors gets consulted. It also inverts the claim's own
|
|
1691
|
-
logic: where two anchorings genuinely differ, that tie-break publishes the
|
|
1692
|
-
loser's retired value rather than withholding it.
|
|
1693
|
-
|
|
1694
|
-
The two passes still differ in PRECEDENCE, deliberately. The epoch pass has real
|
|
1695
|
-
half-open boundaries to test an instant against, so every candidate it keeps
|
|
1696
|
-
genuinely contains the instant; the date pass has no boundaries, only two date
|
|
1697
|
-
columns of which a live writer fills one with the EXCLUSIVE boundary's date, so
|
|
1698
|
-
"contains the day" there does not mean "belongs to the week". A bounds-carrying
|
|
1699
|
-
placement is therefore the more precise one and pre-empts the date pass
|
|
1700
|
-
entirely. Pinned by
|
|
1701
|
-
`tests/test_five_hour_breakdown.py::test_835_r9_the_epoch_pass_unions_every_bounds_carrying_week_it_contains`
|
|
1702
|
-
and `::test_835_r8_a_bounds_carrying_week_outranks_a_bounds_free_one`.
|
|
1703
|
-
|
|
1704
|
-
THE DATE PASS'S UNION OVER-WITHHOLDS, AND THAT RESIDUAL IS SMALLER THAN ROUND
|
|
1705
|
-
FIVE'S. When an instant genuinely belongs to an uncredited candidate and an
|
|
1706
|
-
overlapping candidate was credited, the union withholds a value no credit ever
|
|
1707
|
-
retired. Round five paid that on EVERY block observed on a week-boundary day,
|
|
1708
|
-
because `cmd_record_usage` writes `week_end_date = week_start_date + 7 days` —
|
|
1709
|
-
the exclusive boundary's date — while this pass compares inclusively, so every
|
|
1710
|
-
pair of adjacent weeks shares one calendar day. The capture pass now answers
|
|
1711
|
-
every such instant some snapshot row carries, which is the ordinary case, and
|
|
1712
|
-
what is left is an instant NO capture record carries on a store whose weeks have
|
|
1713
|
-
no parseable bounds.
|
|
1714
|
-
|
|
1715
|
-
The direction is the conservative one — an unavailable marker in preference to a
|
|
1716
|
-
number a credit retired — and both fallback passes now take it. It is still not
|
|
1717
|
-
the direction EVERY fork in this change takes: an empty result resolves no bands
|
|
1718
|
-
and publishes the raw value, which is the pre-#835 disposition kept on purpose.
|
|
1719
|
-
`::test_835_r8_the_union_withholds_an_uncredited_weeks_own_value` pins the trade
|
|
1720
|
-
rather than leaving a later reader to find it and read it as a defect, and
|
|
1721
|
-
`::test_835_r9_a_capture_record_resolves_the_week_a_shared_boundary_day_withheld`
|
|
1722
|
-
pins the boundary-day case that no longer pays it.
|
|
1723
|
-
|
|
1724
|
-
A SEVEN-DAY WINDOW OFF `week_start_date` IS STILL NOT AN ALTERNATIVE. It assumes
|
|
1725
|
-
every bounds-free week is seven days long, and the second payload path can
|
|
1726
|
-
declare a longer one, whose tail a seven-day window would place NOWHERE —
|
|
1727
|
-
withdrawing the manual-credit leg this pass exists to resolve. It also buys
|
|
1728
|
-
nothing on the weeks whose `week_end_date` is computed rather than declared: the
|
|
1729
|
-
third path and `compute_week_bounds` both write `week_start_date + 6 days`, and
|
|
1730
|
-
`cmd_record_usage` writes `week_end_date` exactly `week_start_date + 7 days`, so
|
|
1731
|
-
over those rows a seven-day exclusive window and this inclusive one name the
|
|
1732
|
-
same days and neither separates two overlapping candidates.
|
|
1733
|
-
|
|
1734
|
-
ONE DIRECTION IS BOUNDED BY `cmd_record_credit`, AND THE OTHER IS NOT. That
|
|
1735
|
-
command refuses without a canonical weekly boundary and always writes a
|
|
1736
|
-
bounds-carrying synthetic snapshot of its own, so a manually credited week
|
|
1737
|
-
normally has at least one row with bounds — which bounds the direction where a
|
|
1738
|
-
credited week is reachable ONLY through the date pass. It does not bound the
|
|
1739
|
-
opposite direction, the one round five introduced: that same credited week's
|
|
1740
|
-
OTHER rows still form a bounds-free candidate group, because `week_end_date`
|
|
1741
|
-
joins the grouping, so an instant in an adjacent bounds-free week still picks up
|
|
1742
|
-
the credited week's floor and over-withholds. Do not read the synthetic snapshot
|
|
1743
|
-
as bounding both.
|
|
1744
|
-
|
|
1745
|
-
EMPTY when no week names the instant on any pass. That resolves no bands and
|
|
1746
|
-
therefore withholds nothing, which is the pre-#835 behaviour.
|
|
1747
|
-
|
|
1748
|
-
``cache`` IS THE OUTER RENDER CACHE, NOT A WEEK MEMO, AND IT CHANGED SHAPE
|
|
1749
|
-
SILENTLY AT GATE A R8. Gate A S1 passed a dict this function used directly as
|
|
1750
|
-
``{(account_key, instant_iso) -> weeks}``. It now carves its own sub-dicts out
|
|
1751
|
-
of the dict the caller owns, and that dict is the same one
|
|
1752
|
-
`_credit_aware_block_weekly_axes` carves `__bands` out of, so FOUR reserved
|
|
1753
|
-
keys share one namespace:
|
|
1754
|
-
|
|
1755
|
-
``__weeks`` `{account_key -> [_AxisWeek]}`, `_account_axis_weeks`' memo
|
|
1756
|
-
``__stamps`` `{account_key -> {second -> identities}}`, the capture records
|
|
1757
|
-
``__by_date`` `{account_key -> {week_start_date -> identities}}` (R10)
|
|
1758
|
-
``__bands`` `{(week, account_key) -> bands}`, owned by the caller
|
|
1759
|
-
|
|
1760
|
-
A caller passing the OLD shape is not detected and does not raise: every
|
|
1761
|
-
`setdefault` above finds no reserved key, creates a fresh sub-dict, and the
|
|
1762
|
-
pre-existing entries are simply never read. The failure is therefore a silent
|
|
1763
|
-
loss of memoization — one extra pair of queries per account per render, the cost
|
|
1764
|
-
the per-account load exists to remove — and never a wrong answer, because
|
|
1765
|
-
nothing reads a key it did not write. The leading double underscore is what
|
|
1766
|
-
keeps the two populations apart: an `account_key` is either a 32-character
|
|
1767
|
-
lowercase sha256 prefix (`digest.hexdigest()[:32]`, bin/_lib_accounts.py:10,
|
|
1768
|
-
:63 and :70) or the literal `unattributed` (bin/_lib_accounts.py:41), so none of
|
|
1769
|
-
the four names can collide with one.
|
|
1770
|
-
|
|
1771
|
-
EVERY PASS IS GATED ON `_account_axis_weeks` SUCCEEDING, the capture pass
|
|
1772
|
-
included, and that coupling is deliberate. It looks gratuitous, because NEITHER
|
|
1773
|
-
QUERY'S COLUMN DEPENDENCY CONTAINS THE OTHER'S, so in principle a store that
|
|
1774
|
-
fails one could answer the other. The week query reads `week_start_date`,
|
|
1775
|
-
`week_end_date`, `week_start_at` and `week_end_at`; the capture query reads
|
|
1776
|
-
`captured_at_utc`, `week_start_date`, `week_start_at` and `week_end_at`, plus
|
|
1777
|
-
`first_observed_at_utc` and `last_observed_at_utc` from a SECOND table,
|
|
1778
|
-
`five_hour_blocks`. So the divergence has two directions — a store carrying the
|
|
1779
|
-
bounds without `week_end_date`, and one carrying `week_end_date` without
|
|
1780
|
-
`captured_at_utc` or without the block table — and the claim that the capture
|
|
1781
|
-
query's dependency was strictly smaller was FALSE WHEN WRITTEN in R10, because
|
|
1782
|
-
the capture query has read a second table since R9. Two earlier statements of
|
|
1783
|
-
this are withdrawn: the claim was authored by `f32554505`, which is later than
|
|
1784
|
-
`df80ca1dd` that removed the week query's aggregate, so it cannot have "stopped
|
|
1785
|
-
being true" in that round; and it was never true, because the capture query has
|
|
1786
|
-
selected `first_observed_at_utc` / `last_observed_at_utc` from `five_hour_blocks`
|
|
1787
|
-
since `9690c0fae`. Three reasons the gate is kept:
|
|
1788
|
-
|
|
1789
|
-
- The divergence is unreachable. `week_end_date` is `TEXT NOT NULL` in the
|
|
1790
|
-
base `CREATE TABLE` (bin/_cctally_core.py:2127) and no migration adds it, so
|
|
1791
|
-
no cctally store has the other four columns without it. The legacy shape
|
|
1792
|
-
`bin/build-doctor-fixtures.py` builds lacks `week_start_at` / `week_end_at`
|
|
1793
|
-
as well, so BOTH queries fail on it together.
|
|
1794
|
-
- An empty week set from the ordinary cause — the account has no snapshot rows
|
|
1795
|
-
— implies an empty capture map, because the capture query reads the same
|
|
1796
|
-
table filtered on the same `account_key`.
|
|
1797
|
-
- Since R10 the capture pass READS `weeks`, to build the substitution index
|
|
1798
|
-
`_account_axis_bounds_by_start_date` returns, so running it without them
|
|
1799
|
-
degrades it to returning bounds-free identities unenriched.
|
|
1800
|
-
|
|
1801
|
-
THAT THIRD REASON IS NOT THAT DECOUPLING WOULD BUY NOTHING. An unenriched
|
|
1802
|
-
bounds-free identity still resolves the `weekly_credit_floors` leg, and the empty
|
|
1803
|
-
tuple resolves neither leg, so on a store where the week query failed and the
|
|
1804
|
-
capture query succeeded, decoupling WOULD buy the manual leg. What makes the gain
|
|
1805
|
-
unreachable is reason 1, not any equivalence between the two dispositions. The
|
|
1806
|
-
gate is kept because the divergence cannot occur on a store any cctally version
|
|
1807
|
-
created, and decoupling would add a second exit path to reason about for a gain
|
|
1808
|
-
no store can realise. If a future store shape makes the divergence real, the fix
|
|
1809
|
-
is to run the capture pass first and gate only the two fallback passes.
|
|
1810
|
-
"""
|
|
1811
|
-
weeks = _account_axis_weeks(
|
|
1812
|
-
conn, account_key=account_key, cache=cache.setdefault("__weeks", {}))
|
|
1813
|
-
if not weeks:
|
|
1814
|
-
# EVERY pass is gated on the week load, the capture pass included. See the
|
|
1815
|
-
# docstring: the coupling is deliberate and, since R10, load-bearing.
|
|
1816
|
-
return ()
|
|
1817
|
-
try:
|
|
1818
|
-
instant = parse_iso_datetime(instant_iso, "axis_instant")
|
|
1819
|
-
except (ValueError, TypeError):
|
|
1820
|
-
return ()
|
|
1821
|
-
moment = instant.timestamp()
|
|
1822
|
-
# PASS 1 — the capture record. Truncated to the second on both sides, matching
|
|
1823
|
-
# the `unixepoch()` the narrowing query uses; see `_account_axis_capture_weeks`.
|
|
1824
|
-
carried = _account_axis_capture_weeks(
|
|
1825
|
-
conn, account_key=account_key,
|
|
1826
|
-
cache=cache.setdefault("__stamps", {})).get(int(moment))
|
|
1827
|
-
if carried:
|
|
1828
|
-
# A bounds-free matched row names the week but cannot scope the
|
|
1829
|
-
# `week_reset_events` leg, and this pass pre-empts the epoch pass, so
|
|
1830
|
-
# substitute the SAME week's bounds as this account's OWN other rows spell
|
|
1831
|
-
# them — see the docstring for why the substitution is lossless, why
|
|
1832
|
-
# containment is not required of a substitute, and what residual survives.
|
|
1833
|
-
named: dict = {}
|
|
1834
|
-
for identity in carried:
|
|
1835
|
-
if identity[1] is not None:
|
|
1836
|
-
named[identity] = None
|
|
1837
|
-
continue
|
|
1838
|
-
by_date = cache.setdefault("__by_date", {})
|
|
1839
|
-
if account_key not in by_date:
|
|
1840
|
-
by_date[account_key] = _account_axis_bounds_by_start_date(weeks)
|
|
1841
|
-
# Built once per account from the already-loaded week set, so the
|
|
1842
|
-
# substitution costs no query and no per-instant scan.
|
|
1843
|
-
named.update(dict.fromkeys(
|
|
1844
|
-
by_date[account_key].get(identity[0]) or (identity,)))
|
|
1845
|
-
return tuple(named)
|
|
1846
|
-
# PASS 2 — containment in a week's parsed half-open bounds. A dict as an
|
|
1847
|
-
# ordered set throughout: the caller's union is order-insensitive, so the
|
|
1848
|
-
# order these are visited in carries no meaning, and membership must not be a
|
|
1849
|
-
# linear scan.
|
|
1850
|
-
found: dict = {}
|
|
1851
|
-
for week in weeks:
|
|
1852
|
-
if week.lo is not None and week.lo <= moment < week.hi:
|
|
1853
|
-
found[(week.week_start_date, week.week_start_at, week.week_end_at)] = (
|
|
1854
|
-
None)
|
|
1855
|
-
if found:
|
|
1856
|
-
return tuple(found)
|
|
1857
|
-
# PASS 3 — the instant's calendar day against the bounds-free weeks' declared
|
|
1858
|
-
# date ranges. `.astimezone(utc)` before `.date()` for
|
|
1859
|
-
# `_derive_week_from_payload`'s own reason: `parse_iso_datetime` ends in
|
|
1860
|
-
# `.astimezone()`, so on a host whose offset puts the moment on another calendar
|
|
1861
|
-
# date a bare `.date()` would place the instant in the wrong week. EVERY
|
|
1862
|
-
# candidate that declares the day, with no preference among them — see the
|
|
1863
|
-
# docstring for why no preference can be grounded here and what the union costs.
|
|
1864
|
-
day = instant.astimezone(dt.timezone.utc).date().isoformat()
|
|
1865
|
-
for week in weeks:
|
|
1866
|
-
if week.lo is not None:
|
|
1867
|
-
continue
|
|
1868
|
-
# Both columns are `TEXT NOT NULL`, so a None here is a store nobody can
|
|
1869
|
-
# place; skip it rather than raise comparing a string to None.
|
|
1870
|
-
if week.week_start_date is None or week.week_end_date is None:
|
|
1871
|
-
continue
|
|
1872
|
-
if week.week_start_date <= day <= week.week_end_date:
|
|
1873
|
-
# `week_end_date` is in `_account_axis_weeks`' DISTINCT projection, so
|
|
1874
|
-
# two rows differing only there are two candidates carrying ONE identity.
|
|
1875
|
-
# The bands would be identical, so the duplicate is dropped rather than
|
|
1876
|
-
# resolved twice.
|
|
1877
|
-
found[(week.week_start_date, None, None)] = None
|
|
1878
|
-
return tuple(found)
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
1071
|
def _credit_aware_block_weekly_axes(conn, block, *, now_utc, cache=None):
|
|
1882
|
-
"""A block's two weekly axes,
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
block
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
`_migration_merge_5h_block_duplicates_v1` in bin/_cctally_db.py, which writes
|
|
1897
|
-
`seven_day_pct_at_block_end` from the group row whose `last_observed_at_utc` is
|
|
1898
|
-
MAX. The migration was omitted from the original count; the argument is
|
|
1899
|
-
unaffected, because it also stores what was observed, but the number was
|
|
1900
|
-
documented and wrong. All four store what the fold OBSERVED; this one read
|
|
1901
|
-
decides what a credit retired may still be shown. Making one writer
|
|
1902
|
-
credit-aware instead would also break
|
|
1903
|
-
rebuild convergence, because the store would then hold a number where a
|
|
1904
|
-
rebuilt store holds `NULL` and `tests/test_rederive_command.py` compares
|
|
1905
|
-
exactly these two columns across a rebuild.
|
|
1906
|
-
|
|
1907
|
-
EACH AXIS IS COMPARED AGAINST THE INSTANT THAT AXIS'S VALUE WAS CAPTURED AT,
|
|
1908
|
-
never against a boundary of the block: the start against
|
|
1909
|
-
`_block_weekly_start_instant` (`first_observed_at_utc`), the end against
|
|
1910
|
-
`_block_weekly_end_instant` (`last_observed_at_utc`). Gate A S1 used the block's
|
|
1911
|
-
own boundary stamps instead and that was wrong in both directions — see those
|
|
1912
|
-
two helpers. The rule separates the three positions a block's OBSERVATIONS can
|
|
1913
|
-
hold relative to a credit floor. Observations entirely BEFORE the floor render
|
|
1914
|
-
both stored values, because they genuinely were effective while the block ran
|
|
1915
|
-
and withholding them would destroy historical truth. Observations SPANNING the
|
|
1916
|
-
floor render the start and withhold a retired end. Observations entirely AFTER
|
|
1917
|
-
the floor withhold both, which is the shape a post-credit replay at the retired
|
|
1918
|
-
value produces.
|
|
1919
|
-
|
|
1920
|
-
The bands come from the block's OWN week and `account_key` — a credit under
|
|
1921
|
-
one account retires nothing under another, and a crossed-reset block's two
|
|
1922
|
-
axes legitimately resolve two different weeks.
|
|
1923
|
-
|
|
1924
|
-
``cache`` is an optional dict shared across a multi-row render, and it holds
|
|
1925
|
-
FOUR reserved keys in ONE namespace: this function's own ``__bands``, resolved
|
|
1926
|
-
once per `(week, account_key)`, plus the ``__weeks``, ``__stamps`` and
|
|
1927
|
-
``__by_date`` that `_block_weekly_axis_weeks` carves out of the same dict. That
|
|
1928
|
-
function's docstring states each one's shape and what happens to a caller who
|
|
1929
|
-
passes the pre-R8 shape, which is a silent loss of memoization rather than a
|
|
1930
|
-
wrong answer. Together they hold the account's whole week set, the capture
|
|
1931
|
-
records of that account's block axis stamps, the per-`week_start_date` index of
|
|
1932
|
-
bounds-carrying identities, and the bands. Each axis instant is then resolved
|
|
1933
|
-
against the loaded sets in memory, so the number of queries does not grow with
|
|
1934
|
-
the number of rendered blocks. The caller owns the dict and it must not outlive
|
|
1935
|
-
the connection.
|
|
1936
|
-
"""
|
|
1937
|
-
_c = _cctally()
|
|
1938
|
-
p_start = block.get("seven_day_pct_at_block_start")
|
|
1939
|
-
p_end = block.get("seven_day_pct_at_block_end")
|
|
1940
|
-
if p_start is None and p_end is None:
|
|
1941
|
-
return None, None
|
|
1942
|
-
if cache is None:
|
|
1943
|
-
cache = {}
|
|
1944
|
-
bands = cache.setdefault("__bands", {})
|
|
1945
|
-
account_key = block.get("account_key") or _lib_accounts.UNATTRIBUTED
|
|
1946
|
-
out = []
|
|
1947
|
-
for value, instant in (
|
|
1948
|
-
(p_start, _block_weekly_start_instant(block)),
|
|
1949
|
-
(p_end, _block_weekly_end_instant(block, now_utc=now_utc)),
|
|
1950
|
-
):
|
|
1951
|
-
if value is None or not instant:
|
|
1952
|
-
out.append(value)
|
|
1953
|
-
continue
|
|
1954
|
-
week_ids = _block_weekly_axis_weeks(
|
|
1955
|
-
conn, account_key=account_key, instant_iso=instant, cache=cache)
|
|
1956
|
-
if not week_ids:
|
|
1957
|
-
out.append(value)
|
|
1958
|
-
continue
|
|
1959
|
-
# THE UNION OF EVERY CANDIDATE WEEK'S BANDS, so a value any one of them
|
|
1960
|
-
# retired is withheld. One bounds-carrying week is one candidate; several
|
|
1961
|
-
# bounds-free ones are several, because nothing in their own columns says
|
|
1962
|
-
# which of them an instant belongs to. Each is still resolved by
|
|
1963
|
-
# `_credit_retirement_bands` from its OWN triple, so the per-leg
|
|
1964
|
-
# degradation is unchanged: a bounds-free candidate contributes its
|
|
1965
|
-
# `weekly_credit_floors` leg and no `week_reset_events` leg, which cannot
|
|
1966
|
-
# be scoped without bounds. The memo stays per `(week, account_key)`, so
|
|
1967
|
-
# overlapping candidates shared across blocks resolve once each.
|
|
1968
|
-
merged: list = []
|
|
1969
|
-
for week in week_ids:
|
|
1970
|
-
bk = (week, account_key)
|
|
1971
|
-
if bk not in bands:
|
|
1972
|
-
bands[bk] = _c._credit_retirement_bands(
|
|
1973
|
-
conn, week_start_date=week[0], week_start_at=week[1],
|
|
1974
|
-
week_end_at=week[2], account_key=account_key)
|
|
1975
|
-
merged.extend(bands[bk])
|
|
1976
|
-
out.append(
|
|
1977
|
-
None if _c._weekly_value_is_retired_replica(
|
|
1978
|
-
merged, captured_at_utc=instant, weekly_percent=value)
|
|
1979
|
-
else value)
|
|
1980
|
-
return out[0], out[1]
|
|
1072
|
+
"""A block's two stored weekly axes,
|
|
1073
|
+
``(seven_day_pct_at_block_start, seven_day_pct_at_block_end)``.
|
|
1074
|
+
|
|
1075
|
+
#840 (spec section 4.8). #835 replaced a value within a point of one a
|
|
1076
|
+
credit retired with ``None`` here, which also withheld every genuine climb
|
|
1077
|
+
back to that value; the retirement band is removed. Stale contributors are
|
|
1078
|
+
refused before publication (spec section 4.4), and a value already
|
|
1079
|
+
poisoned in a store is corrected by `db rederive`'s reviewed `retract`,
|
|
1080
|
+
never by a percentage heuristic. The name, the signature and the callers
|
|
1081
|
+
(`cmd_five_hour_blocks`, `cmd_five_hour_breakdown` and the dashboard's
|
|
1082
|
+
block envelope) are unchanged; ``conn``, ``now_utc`` and ``cache`` are no
|
|
1083
|
+
longer read."""
|
|
1084
|
+
return (block.get("seven_day_pct_at_block_start"),
|
|
1085
|
+
block.get("seven_day_pct_at_block_end"))
|
|
1981
1086
|
|
|
1982
1087
|
|
|
1983
1088
|
#: The six `five_hour_milestones` columns every surface already published, plus
|
|
1984
1089
|
#: the joined EFFECTIVE weekly value. Ordering is `captured_at_utc, id` — spec
|
|
1985
1090
|
#: §5.2's merged-stream order, which post-credit threshold repeats depend on.
|
|
1986
|
-
#: The five `s.*` columns after the joined weekly value are INTERNAL. They exist
|
|
1987
|
-
#: only so the credit retirement bands can be resolved against the JOINED ROW's
|
|
1988
|
-
#: own account and week, and they are aliased because `captured_at_utc` exists on
|
|
1989
|
-
#: both sides of the join. `_load_five_hour_milestones` builds its result dicts
|
|
1990
|
-
#: key by key, so none of them can reach a published shape.
|
|
1991
1091
|
_FIVE_HOUR_MILESTONE_SQL = """
|
|
1992
1092
|
SELECT m.percent_threshold, m.captured_at_utc, m.block_cost_usd,
|
|
1993
1093
|
m.marginal_cost_usd, m.seven_day_pct_at_crossing, m.reset_event_id,
|
|
1994
|
-
s.weekly_percent AS effective_seven_day_pct_at_crossing
|
|
1995
|
-
s.captured_at_utc AS snapshot_captured_at_utc,
|
|
1996
|
-
s.week_start_date AS snapshot_week_start_date,
|
|
1997
|
-
s.week_start_at AS snapshot_week_start_at,
|
|
1998
|
-
s.week_end_at AS snapshot_week_end_at,
|
|
1999
|
-
s.account_key AS snapshot_account_key
|
|
1094
|
+
s.weekly_percent AS effective_seven_day_pct_at_crossing
|
|
2000
1095
|
FROM five_hour_milestones m
|
|
2001
1096
|
LEFT JOIN weekly_usage_snapshots s ON s.id = m.usage_snapshot_id
|
|
2002
1097
|
WHERE {selector}
|
|
@@ -2036,34 +1131,12 @@ def _load_five_hour_milestones(
|
|
|
2036
1131
|
put the number this change exists to stop showing back on the screen,
|
|
2037
1132
|
silently and only in the failure case.
|
|
2038
1133
|
|
|
2039
|
-
**The joined value is
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
number #836 exists to remove, on all three human surfaces, on a new read.
|
|
2046
|
-
|
|
2047
|
-
So the retirement bands are resolved against the JOINED ROW's own account and
|
|
2048
|
-
week — never the reader's, because a credit under one account retires nothing
|
|
2049
|
-
under another — memoized per `(week, account)` across the rows of one call, and
|
|
2050
|
-
a retired joined value is published as None. The surfaces already render a None
|
|
2051
|
-
as the unavailable marker, so no renderer changes, and the rule above still
|
|
2052
|
-
holds: never a fallback to the raw value. (The memoization is this function's
|
|
2053
|
-
own: it walks many rows spanning potentially several `(week, account)` pairs.
|
|
2054
|
-
`_latest_seven_day_and_window` does NOT memoize, because its walk resolves the
|
|
2055
|
-
scope, the bands and the floor once from the newest row and may not leave that
|
|
2056
|
-
scope, so there is nothing to memoize per candidate. An earlier version of this
|
|
2057
|
-
paragraph said it did.)
|
|
2058
|
-
|
|
2059
|
-
THE BAND IS APPLIED HERE WITHOUT R1's FLOOR BOUND, deliberately. That bound
|
|
2060
|
-
exists because a WALK can descend past every post-floor row onto a pre-floor
|
|
2061
|
-
one and return the pre-credit reading the band exists to suppress. This read
|
|
2062
|
-
performs no walk: each milestone names exactly ONE snapshot row through
|
|
2063
|
-
`usage_snapshot_id`, and a pre-floor row's weekly value genuinely WAS the
|
|
2064
|
-
effective percentage at that crossing instant. Blanking it would delete
|
|
2065
|
-
historical truth rather than protect the reader, which is the same reason the
|
|
2066
|
-
block's pre-floor weekly axes render.
|
|
1134
|
+
**The joined value is the snapshot's STORED value** (#840, spec section
|
|
1135
|
+
4.8). #834 S1 Gate A R2 replaced a joined value within a point of one a
|
|
1136
|
+
credit retired with None, which also withheld genuine readings at that
|
|
1137
|
+
value; the retirement band is removed, and a value already poisoned in a
|
|
1138
|
+
store is corrected by `db rederive`'s reviewed `retract`. A missing joined
|
|
1139
|
+
snapshot still publishes None and never falls back to the raw value.
|
|
2067
1140
|
|
|
2068
1141
|
SELECTOR. Exactly one of `block_id` or `five_hour_window_key` must be given.
|
|
2069
1142
|
`block_id` is the precise form — `five_hour_milestones.block_id` names one
|
|
@@ -2094,29 +1167,10 @@ def _load_five_hour_milestones(
|
|
|
2094
1167
|
rows = conn.execute(
|
|
2095
1168
|
_FIVE_HOUR_MILESTONE_SQL.format(selector=selector), params,
|
|
2096
1169
|
).fetchall()
|
|
2097
|
-
_c = _cctally()
|
|
2098
|
-
bands_cache: dict = {}
|
|
2099
1170
|
out: list[dict] = []
|
|
2100
1171
|
for r in rows:
|
|
2101
1172
|
eff = r["effective_seven_day_pct_at_crossing"]
|
|
2102
1173
|
raw = r["seven_day_pct_at_crossing"]
|
|
2103
|
-
if eff is not None:
|
|
2104
|
-
band_key = (r["snapshot_week_start_date"],
|
|
2105
|
-
r["snapshot_week_start_at"],
|
|
2106
|
-
r["snapshot_week_end_at"],
|
|
2107
|
-
r["snapshot_account_key"])
|
|
2108
|
-
if band_key not in bands_cache:
|
|
2109
|
-
bands_cache[band_key] = _c._credit_retirement_bands(
|
|
2110
|
-
conn,
|
|
2111
|
-
week_start_date=r["snapshot_week_start_date"],
|
|
2112
|
-
week_start_at=r["snapshot_week_start_at"],
|
|
2113
|
-
week_end_at=r["snapshot_week_end_at"],
|
|
2114
|
-
account_key=r["snapshot_account_key"])
|
|
2115
|
-
if _c._weekly_value_is_retired_replica(
|
|
2116
|
-
bands_cache[band_key],
|
|
2117
|
-
captured_at_utc=r["snapshot_captured_at_utc"],
|
|
2118
|
-
weekly_percent=eff):
|
|
2119
|
-
eff = None
|
|
2120
1174
|
out.append({
|
|
2121
1175
|
"percent_threshold": int(r["percent_threshold"]),
|
|
2122
1176
|
"captured_at_utc": r["captured_at_utc"],
|
|
@@ -2818,16 +1872,24 @@ def _backfill_five_hour_blocks(
|
|
|
2818
1872
|
AND snapshots.five_hour_percent IS NOT NULL
|
|
2819
1873
|
"""
|
|
2820
1874
|
if only_missing:
|
|
1875
|
+
# #997 D19: each account's latest qualifying snapshot comes from
|
|
1876
|
+
# one ranking pass over the qualifying rows (O(n log n)). A
|
|
1877
|
+
# correlated subquery here re-scanned and re-sorted the table per
|
|
1878
|
+
# qualifying row (quadratic: 68.4 s on a 27,521-row store, past
|
|
1879
|
+
# the rebuild's progress-gap bound). The partition groups NULL
|
|
1880
|
+
# account keys as `IS` did; the ORDER BY is unchanged.
|
|
2821
1881
|
keys_sql += (
|
|
2822
|
-
" AND snapshots.id
|
|
2823
|
-
" SELECT
|
|
2824
|
-
"
|
|
2825
|
-
"
|
|
2826
|
-
"
|
|
2827
|
-
"
|
|
2828
|
-
"
|
|
2829
|
-
"
|
|
2830
|
-
"
|
|
1882
|
+
" AND snapshots.id IN ("
|
|
1883
|
+
" SELECT id FROM ("
|
|
1884
|
+
" SELECT id, ROW_NUMBER() OVER ("
|
|
1885
|
+
" PARTITION BY account_key"
|
|
1886
|
+
" ORDER BY unixepoch(captured_at_utc) DESC,"
|
|
1887
|
+
" id DESC"
|
|
1888
|
+
" ) AS rn"
|
|
1889
|
+
" FROM weekly_usage_snapshots"
|
|
1890
|
+
" WHERE five_hour_window_key IS NOT NULL"
|
|
1891
|
+
" AND five_hour_percent IS NOT NULL"
|
|
1892
|
+
" ) WHERE rn = 1"
|
|
2831
1893
|
" )"
|
|
2832
1894
|
" AND NOT EXISTS ("
|
|
2833
1895
|
" SELECT 1 FROM five_hour_blocks AS blocks"
|
|
@@ -2898,7 +1960,11 @@ def _backfill_five_hour_blocks(
|
|
|
2898
1960
|
# See cctally-dev#87.
|
|
2899
1961
|
conn.execute("BEGIN IMMEDIATE")
|
|
2900
1962
|
try:
|
|
2901
|
-
for key, acct in keys:
|
|
1963
|
+
for block_index, (key, acct) in enumerate(keys, 1):
|
|
1964
|
+
if not block_index & 1023:
|
|
1965
|
+
# #997 D9: a rebuild's open-block projection reports
|
|
1966
|
+
# progress (throttled; a no-op without a progress sink).
|
|
1967
|
+
_lib_rebuild_progress.tick(blocks=block_index)
|
|
2902
1968
|
# The start row (the first writer, else the MIN-captured
|
|
2903
1969
|
# survivor) defines the immutable block boundary values
|
|
2904
1970
|
# (deterministic — picking "any in-window row" would be
|