cctally 1.115.0 → 1.116.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/bin/_cctally_cache.py +85 -4
  3. package/bin/_cctally_completion.py +80 -12
  4. package/bin/_cctally_core.py +106 -9
  5. package/bin/_cctally_dashboard.py +6 -9
  6. package/bin/_cctally_dashboard_envelope.py +26 -32
  7. package/bin/_cctally_db.py +85 -4
  8. package/bin/_cctally_doctor.py +38 -5
  9. package/bin/_cctally_five_hour.py +83 -1017
  10. package/bin/_cctally_journal.py +1307 -180
  11. package/bin/_cctally_journal_repair.py +9 -7
  12. package/bin/_cctally_oauth_samples.py +280 -0
  13. package/bin/_cctally_parser.py +5 -0
  14. package/bin/_cctally_quarantine.py +14 -4
  15. package/bin/_cctally_quota.py +70 -8
  16. package/bin/_cctally_quota_model.py +35 -4
  17. package/bin/_cctally_record.py +414 -123
  18. package/bin/_cctally_rederive.py +632 -235
  19. package/bin/_cctally_refresh.py +29 -0
  20. package/bin/_cctally_statusline.py +984 -47
  21. package/bin/_cctally_store.py +972 -58
  22. package/bin/_cctally_tui.py +62 -7
  23. package/bin/_cctally_update.py +25 -0
  24. package/bin/_lib_account_protocol.py +201 -18
  25. package/bin/_lib_clone_copy.py +46 -11
  26. package/bin/_lib_dashboard_sources.py +57 -15
  27. package/bin/_lib_detector_access_registry.py +273 -52
  28. package/bin/_lib_doctor.py +302 -1
  29. package/bin/_lib_journal.py +145 -25
  30. package/bin/_lib_journal_router.py +166 -48
  31. package/bin/_lib_oauth_samples.py +238 -0
  32. package/bin/_lib_pricing_check.py +0 -12
  33. package/bin/_lib_provider_activity.py +770 -0
  34. package/bin/_lib_quarantine.py +3 -2
  35. package/bin/_lib_quota_model.py +96 -6
  36. package/bin/_lib_rebuild_progress.py +146 -0
  37. package/bin/_lib_record.py +22 -0
  38. package/bin/_lib_rederive.py +299 -3
  39. package/bin/_lib_snapshot_cache.py +43 -1
  40. package/bin/_lib_stale_proof.py +708 -0
  41. package/bin/_lib_statusline_candidates.py +65 -5
  42. package/bin/_lib_worker_launches.py +315 -0
  43. package/bin/cctally +6 -11
  44. package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
  45. package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
  46. package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
  47. package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
  48. package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
  49. package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
  50. package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
  51. package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
  52. package/dashboard/static/dashboard.html +1 -1
  53. package/package.json +7 -2
  54. package/bin/_lib_merge_gate.py +0 -452
  55. package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
@@ -38,13 +38,13 @@ Spec: docs/superpowers/specs/2026-05-30-extract-five-hour-statusline-cmd-design.
38
38
  from __future__ import annotations
39
39
 
40
40
  import argparse
41
- import collections
42
41
  import datetime as dt
43
42
  import json
44
43
  import sqlite3
45
44
  import sys
46
45
 
47
46
  import _lib_accounts # pure stdlib kernel; UNATTRIBUTED sentinel (#341)
47
+ import _lib_rebuild_progress # #997 D9 progress ticks (stdlib only)
48
48
  from _lib_five_hour import latest_five_hour_reading
49
49
  from _cctally_core import _command_as_of, eprint, now_utc_iso, open_db, parse_iso_datetime
50
50
 
@@ -989,69 +989,36 @@ def _latest_seven_day_and_window(
989
989
  retired replica or a weekly-only reading can be the newest row, and its NULL
990
990
  key would report the live block closed and leave its weekly end stale.
991
991
 
992
- **The weekly value is CREDIT-AWARE**, which is the #835 change. It comes from
993
- the most recent row that is not a post-floor stale replica — the same band
994
- and floor predicate the stale-replica DELETE applies, applied as a read
995
- filter. Held-inclusive is still right in principle here, because a held row
996
- stores the EFFECTIVE weekly value and that is what a block's weekly end
997
- means. What is not right is reporting a value a credit RETIRED, and #835
998
- made that reachable by preserving held rows: before it, the DELETE removed
999
- every post-floor replica, so this read could not see one. Excluding held rows
1000
- outright would be the wrong repair — it would report the block end from an
1001
- older tick, which is the mistake #824's comment here warned against.
1002
-
1003
- **The weekly walk is BOUNDED three ways, and each bound closes a hole tranche
1004
- A left open (#834 S1 Gate A R1).**
1005
-
1006
- *Scope.* The walk descends only within the NEWEST row's own
1007
- ``(week_start_date, account_key)``. A candidate belonging to another week or
1008
- another account ends the walk with ``None``, because week W-1's percentage is
1009
- not week W's weekly end and account B's is not account A's, under any reading.
1010
- Before #835 this read took exactly one row, so the absent predicates could not
1011
- matter; a walk makes them matter.
1012
-
1013
- *Floor.* The walk also ends with ``None`` at the first row captured BEFORE the
1014
- latest credit floor in that scope. This is the soundness bound: a pre-floor row
1015
- is never classified as retired, so without it the walk terminated on one and
1016
- returned its value — and that value is by construction the PRE-credit reading,
1017
- which is exactly the value a retirement band exists to suppress. A sub-one-point
1018
- credit reaches it immediately, because such a credit is legal
1019
- (`_doomed_snapshot_rows`) and places the post-credit synthetic inside a band
1020
- centred on ``from_pct``, leaving every post-floor row in the band. An
1021
- unparseable capture stamp inside a credited week is treated as pre-floor for
1022
- the same reason: the row cannot be placed relative to the floor, so it cannot
1023
- be certified as able to state the current value.
1024
-
1025
- *Retirement.* Within the scope and at or after the floor, a retired replica is
1026
- skipped, and a retired value is therefore never returned.
1027
-
1028
- The floor bound applies to the NEWEST row as well as to the candidates below
1029
- it, so a newest row captured before the latest floor answers ``None`` where the
1030
- pre-#835 code answered its value. That state is unreachable through either
1031
- production credit path: `record-credit` writes a post-credit synthetic snapshot
1032
- at its own effective instant, and the automatic path's floor is derived from a
1033
- capture that is itself in the store, so in both cases at least one row exists at
1034
- or after the floor and it is the newest.
1035
-
1036
- An UNCREDITED week resolves no bands, so there is no floor, nothing is skipped,
1037
- and the read returns the newest row's value exactly as it did before #835. A
1038
- ``NULL`` weekly value is not a retired replica, so it still terminates the walk
1039
- and still answers ``None``. Both contracts are byte-stable against today.
1040
-
1041
- **The residual, deliberately unfixed and filed as #840.** A genuine later climb
1042
- back INTO the band is still classified as a replay, so the weekly end reads the
1043
- newest out-of-band value until the true value passes the band's upper edge.
1044
- That is preferred over returning a value a credit retired, and no sound
1045
- discriminator is available: `record-credit`'s own synthetic post-credit
1046
- snapshot is written before any later replay, so "a below-band row has been
1047
- observed" would close the band immediately and let every replay through.
1048
- `test_840_characterization_a_genuine_climb_back_reads_the_older_value`
1049
- characterizes it.
992
+ **The weekly value is the newest row's STORED value** (#840, spec section
993
+ 4.8). Held-inclusive, because a held row stores the effective weekly value
994
+ and that is what a block's weekly end means. #835 filtered it through a
995
+ retirement band (a value within a point of one a credit retired was skipped
996
+ as a replay), which also withheld every genuine climb back to that value;
997
+ the band is removed. Stale contributors are refused before publication
998
+ (spec section 4.4), and a value already poisoned in a store is corrected by
999
+ `db rederive`'s reviewed `retract`, never by a percentage heuristic.
1000
+
1001
+ *Scope.* The value is the NEWEST row's, within its own
1002
+ ``(week_start_date, account_key)``; a read never reports another week's or
1003
+ another account's percentage.
1004
+
1005
+ *Floor.* The read answers ``None`` when that row was captured BEFORE the
1006
+ latest credit floor in its scope, or carries an unparseable capture stamp
1007
+ inside a credited week: such a row cannot state the week's current
1008
+ effective weekly value, and its value is by construction the pre-credit
1009
+ reading. That state is unreachable through either production credit path:
1010
+ `record-credit` writes a post-credit synthetic snapshot at its own effective
1011
+ instant, and the automatic path's floor is derived from a capture that is
1012
+ itself in the store, so in both cases at least one row exists at or after
1013
+ the floor and it is the newest.
1014
+
1015
+ An UNCREDITED week resolves no floor, and the read returns the newest row's
1016
+ value. A ``NULL`` weekly value answers ``None``.
1050
1017
 
1051
1018
  ``account_key`` (#341, spec §3): ``None`` = merged / byte-stable; a real key /
1052
- ``unattributed`` scopes both selections to that account. The retirement bands
1053
- are resolved against the NEWEST ROW's own account, which is also the walk's
1054
- scope, because a credit under one account retires nothing under another.
1019
+ ``unattributed`` scopes both selections to that account. The credit floor is
1020
+ resolved against the NEWEST ROW's own account and week, because a credit
1021
+ under one account bounds nothing under another.
1055
1022
  """
1056
1023
  _c = _cctally()
1057
1024
  acct_pred = "" if account_key is None else " WHERE account_key = ?"
@@ -1076,43 +1043,23 @@ def _latest_seven_day_and_window(
1076
1043
  # with this account's weekly value.
1077
1044
  reading = latest_five_hour_reading(conn, account_key=row["account_key"])
1078
1045
  key = None if reading is None else reading.window_key
1079
- # Axis 2: the weekly value. The scope, the bands and the floor all come
1080
- # from the NEWEST row and are resolved ONCE, because the walk may not
1081
- # leave that scope — so there is nothing to memoize per candidate.
1082
- #
1083
- # Dropping `LIMIT 1` does cost something at the storage layer: SQLite
1084
- # materializes the full sort before yielding the first row, measured at
1085
- # 9.0ms against 3.4ms on a 25,053-row store. The cost is immaterial
1086
- # because this runs once per invocation, but the earlier claim that the
1087
- # ordinary case "reads exactly one row, which is the work the single-row
1088
- # query did" described only the rows the cursor yields, not the work the
1089
- # query does, and was wrong.
1090
- scope = (row["week_start_date"], row["account_key"])
1046
+ # Axis 2: the weekly value, the newest row's stored value, bounded by
1047
+ # the latest credit floor of its own week and account (#840).
1091
1048
  bands = _c._credit_retirement_bands(
1092
1049
  conn, week_start_date=row["week_start_date"],
1093
1050
  week_start_at=row["week_start_at"],
1094
1051
  week_end_at=row["week_end_at"],
1095
1052
  account_key=row["account_key"])
1096
1053
  floor = _c._latest_credit_floor_instant(bands)
1097
- pct = None
1098
- while row is not None:
1099
- if (row["week_start_date"], row["account_key"]) != scope:
1100
- break
1101
- if floor is not None:
1102
- try:
1103
- captured = parse_iso_datetime(
1104
- row["captured_at_utc"], "captured_at").timestamp()
1105
- except (ValueError, TypeError):
1106
- captured = None
1107
- if captured is None or captured < floor:
1108
- break
1109
- if not _c._weekly_value_is_retired_replica(
1110
- bands,
1111
- captured_at_utc=row["captured_at_utc"],
1112
- weekly_percent=row["weekly_percent"]):
1113
- pct = row["weekly_percent"]
1114
- break
1115
- row = cur.fetchone()
1054
+ pct = row["weekly_percent"]
1055
+ if floor is not None:
1056
+ try:
1057
+ captured = parse_iso_datetime(
1058
+ row["captured_at_utc"], "captured_at").timestamp()
1059
+ except (ValueError, TypeError):
1060
+ captured = None
1061
+ if captured is None or captured < floor:
1062
+ pct = None
1116
1063
  except sqlite3.DatabaseError:
1117
1064
  return None, None
1118
1065
  return (
@@ -1121,882 +1068,30 @@ def _latest_seven_day_and_window(
1121
1068
  )
1122
1069
 
1123
1070
 
1124
- def _parsed_block_instant(block, column):
1125
- """``block[column]`` when it is present and parses as an instant, else ``None``.
1126
-
1127
- #834 S1 (#835) Gate A R8. Both observation stamps are declared `TEXT NOT NULL`,
1128
- so absence is not a live shape — but a fixture, a hand-built store or a legacy
1129
- projection can still carry an unparseable one, and the two instant helpers
1130
- below must degrade rather than raise.
1131
- """
1132
- value = block.get(column)
1133
- if not value:
1134
- return None
1135
- try:
1136
- parse_iso_datetime(value, column)
1137
- except (ValueError, TypeError):
1138
- return None
1139
- return value
1140
-
1141
-
1142
- def _block_weekly_start_instant(block):
1143
- """The instant a block's stored weekly START was captured at, as ISO text.
1144
-
1145
- #834 S1 (#835) Gate A R8. `first_observed_at_utc`, because that is the stamp
1146
- of the tick that wrote `seven_day_pct_at_block_start`. The live upsert in
1147
- bin/_cctally_record.py sets both from one fold dict on the INSERT (both are
1148
- omitted from its `DO UPDATE`, so the first writer owns the pair), and
1149
- `_backfill_five_hour_blocks` takes `first_obs` and `pct_start_7d` from its
1150
- single MIN-captured snapshot row.
1151
-
1152
- NOT `block_start_at`, which is the window's NOMINAL start — `five_hour_resets_at`
1153
- minus five hours — and therefore at or before the first tick. The gap is not
1154
- cosmetic: a block whose nominal start precedes a credit floor but whose first
1155
- tick landed after it would have the floor's retired value compared against a
1156
- PRE-floor instant and rendered. It falls back to `block_start_at` only when the
1157
- observation stamp is absent or unparseable, which keeps the pre-R8 disposition
1158
- rather than blanking the axis.
1159
- """
1160
- return (_parsed_block_instant(block, "first_observed_at_utc")
1161
- or block.get("block_start_at"))
1162
-
1163
-
1164
- def _block_weekly_end_instant(block, *, now_utc):
1165
- """The instant a block's stored weekly END was captured at, as an ISO text.
1166
-
1167
- #834 S1 (#835) Gate A R8. `last_observed_at_utc`, because that is the stamp of
1168
- the tick that wrote `seven_day_pct_at_block_end`. All four writers of the pair
1169
- move the two together: the live upsert assigns
1170
- `last_observed_at_utc = excluded.last_observed_at_utc` and
1171
- `seven_day_pct_at_block_end = excluded.seven_day_pct_at_block_end` in one
1172
- `DO UPDATE SET` from one fold dict, `_backfill_five_hour_blocks` takes
1173
- `last_obs` and `pct_end_7d` from its single MAX-captured snapshot row,
1174
- `five_hour_block_close` freezes the whole row into one evt, and
1175
- `_migration_merge_5h_block_duplicates_v1` (bin/_cctally_db.py) copies both
1176
- from the group row whose `last_observed_at_utc` is MAX.
1177
-
1178
- Gate A S1 used `five_hour_resets_at` instead, falling back to the current
1179
- instant while that reset was still in the future. That is the block's END, not
1180
- a capture instant, and it was wrong in BOTH directions. A crossed-reset block's
1181
- reset falls in the SUCCESSOR week, so `_block_weekly_axis_weeks` resolved the
1182
- successor's bounds and the credited week's bands were never consulted. A closed
1183
- block whose last tick predates the floor had its genuinely effective end
1184
- withheld, which contradicts the historical-truth rule this filter rests on.
1185
-
1186
- The reset-or-now form is KEPT as the fallback for an absent or unparseable
1187
- stamp, so a store that cannot supply a capture instant behaves as it did
1188
- before R8 rather than losing the axis.
1189
- """
1190
- _c = _cctally()
1191
- observed = _parsed_block_instant(block, "last_observed_at_utc")
1192
- if observed is not None:
1193
- return observed
1194
- now_iso = _c._iso_z(now_utc)
1195
- resets = block.get("five_hour_resets_at")
1196
- if not resets:
1197
- return now_iso
1198
- try:
1199
- moment = parse_iso_datetime(resets, "five_hour_resets_at").timestamp()
1200
- except (ValueError, TypeError):
1201
- return now_iso
1202
- return resets if moment <= now_utc.timestamp() else now_iso
1203
-
1204
-
1205
- #: One candidate week for `_block_weekly_axis_weeks`, as loaded once per account
1206
- #: per render. ``lo`` / ``hi`` are the parsed `[week_start_at, week_end_at)`
1207
- #: epochs, or ``None`` when the week carries no usable bounds.
1208
- _AxisWeek = collections.namedtuple(
1209
- "_AxisWeek",
1210
- "week_start_date week_end_date week_start_at week_end_at lo hi")
1211
-
1212
-
1213
- def _parsed_week_bounds(week_start_at, week_end_at):
1214
- """``(lo, hi)`` epochs for a week's bounds, or ``(None, None)``.
1215
-
1216
- #834 S1 (#835) Gate A R9. Both loaders need the same verdict on the same two
1217
- nullable columns — `_account_axis_weeks` to decide whether a week is placed by
1218
- epoch or by date, `_account_axis_capture_weeks` to decide whether the identity
1219
- it returns carries bounds for the `week_reset_events` leg — so the parse lives
1220
- in one place and the two cannot disagree about which weeks have usable bounds.
1221
- Absent or unparseable on either side yields no bounds, which is what
1222
- `_credit_retirement_bands` already treats as a leg it cannot scope.
1223
- """
1224
- if week_start_at is None or week_end_at is None:
1225
- return None, None
1226
- try:
1227
- return (parse_iso_datetime(week_start_at, "week_start_at").timestamp(),
1228
- parse_iso_datetime(week_end_at, "week_end_at").timestamp())
1229
- except (ValueError, TypeError):
1230
- return None, None
1231
-
1232
-
1233
- def _account_axis_weeks(conn, *, account_key, cache):
1234
- """Every distinct week ``account_key``'s snapshot rows name, in no order.
1235
-
1236
- #834 S1 (#835) Gate A R8. ONE query per account per render, replacing one query
1237
- per axis instant. The per-instant form was measured at 22x the whole render's
1238
- prior cost on a 25,000-row store with 50 blocks, because its
1239
- ``unixepoch(week_start_at)`` predicate makes `idx_usage_week_start_at_time`
1240
- unusable — `EXPLAIN QUERY PLAN` reported `SCAN weekly_usage_snapshots` plus
1241
- `USE TEMP B-TREE FOR ORDER BY`, and the 50-block render performed about 51 full
1242
- scans and 51 full sorts. An instant-keyed memo could not collapse them: two
1243
- instants per block are two distinct keys.
1244
-
1245
- THE WIN IS THE COLLAPSE TO ONE QUERY, NOT RESTORED INDEX USE. This query is
1246
- equally unindexed: `EXPLAIN QUERY PLAN` reports `SCAN weekly_usage_snapshots`
1247
- plus a temp B-tree. About 51 unindexed scans became one unindexed scan, which is
1248
- the whole of the improvement; do not read `idx_usage_week_start_at_time` as being
1249
- back in play. The scan verdict was measured on an empty store, on a 25,500-row
1250
- store and again after `ANALYZE` while this query was still a `GROUP BY`; R10
1251
- replaced that clause with `SELECT DISTINCT`, which changed the B-tree's stated
1252
- purpose and not the scan (see the R10 paragraph below for the measured plan).
1253
-
1254
- THE RESIDUAL IS NOT IRREDUCIBLE, and it is accepted as shipped rather than
1255
- defended as minimal. `idx_usage_week_time` leads on `week_start_date`, so
1256
- narrowing the projection to that column alone does use it — `SELECT DISTINCT
1257
- week_start_date` reports `SCAN weekly_usage_snapshots USING INDEX
1258
- idx_usage_week_time`, measured on the same two stores, and an earlier form said
1259
- "grouping on that column alone", which is stale terminology since R10 replaced
1260
- the `GROUP BY` with `SELECT DISTINCT` — and a render only
1261
- needs the weeks overlapping its blocks' instant range rather than the account's
1262
- whole history. Both are real reductions and neither is taken, because the
1263
- property that matters is that the cost is CONSTANT in the number of rendered
1264
- blocks, which it already is. `week_end_date` and the two bounds are in the
1265
- projection because the date pass needs all four, which is exactly what a
1266
- `week_start_date`-only projection would give up.
1267
-
1268
- ONE COMPARISON SEMANTIC CHANGED WITH THE FORM, unreachably. The per-instant
1269
- predicate compared SQLite `unixepoch()` integers, which truncate a fractional
1270
- second; this form compares `parse_iso_datetime(...).timestamp()` floats, which
1271
- keep it. The two disagree for a week bound carrying a fractional second: a
1272
- `week_start_at` of `…T00:00:00.500000+00:00` against a probe at
1273
- `…T00:00:00+00:00` is INSIDE the week under the integer compare and outside it
1274
- under the float one. No writer can produce that bound.
1275
- `_canonicalize_optional_iso` returns `isoformat(timespec="seconds")` over a
1276
- `_normalize_week_boundary_dt` result, which zeroes second and microsecond and
1277
- snaps to the hour, and `cmd_record_usage`'s pipeline derives both bounds from
1278
- that same normalizer and writes them with the same `timespec="seconds"`.
1279
-
1280
- THE SET IS UNORDERED, AND THE QUERY IS A PLAIN `SELECT DISTINCT` (Gate A R10).
1281
- Through R9 it carried a `MAX(captured_at_utc || char(1) || printf('%020d', id))`
1282
- aggregate whose SQL alias was `rk` and whose `_AxisWeek` field was `rank` — call
1283
- it `rk`, because that is the alias, and earlier text here called the column
1284
- `rank` — introduced to reproduce the per-instant query's
1285
- ``ORDER BY captured_at_utc DESC, id DESC`` tie-break exactly so that which week
1286
- won an overlap did not change with the query's form, and kept as documentary
1287
- after R9 deleted the tie-break. Nothing read it: this function has one caller,
1288
- every pass in `_block_weekly_axis_weeks` builds an order-insensitive set, and
1289
- the caller ORs a predicate over the result. The grouping was there only to carry
1290
- the aggregate, so both went.
1291
-
1292
- NOTHING PINS THE ORDER NOW. There was never a SQL `ORDER BY` on this query: the
1293
- order came from a Python `weeks.sort(key=lambda w: w.rank, reverse=True)` over the
1294
- aggregate. Through R8 a consumer tie-break in the epoch pass read it; R9 deleted
1295
- that tie-break and left the aggregate documentary; R10 deleted the aggregate and
1296
- the sort together. So the returned order is now whatever SQLite produces, and no
1297
- test asserts order-insensitivity of the consumer, so a future consumer that cares
1298
- about order has nothing to fail against. It is safe today because the one consumer
1299
- ORs a monotone band predicate over the whole set. A consumer that needs an order
1300
- must establish it itself rather than assume this one.
1301
-
1302
- The statement-count test cannot see R10's REMOVAL of the aggregate and the
1303
- grouping, because the statement count is unchanged by it; the removal was
1304
- measured instead, best of five on a 25,194-row 13-week 50-block store, with the
1305
- order of the three forms reversed to rule out page-cache bias and the three
1306
- result sets asserted identical: 13.57 ms with the aggregate, 9.36 ms for the
1307
- bare `GROUP BY`, 2.34 ms for `SELECT DISTINCT` — so removing both took about 83%
1308
- off the statement, and the plan changed from `SCAN weekly_usage_snapshots` +
1309
- `USE TEMP B-TREE FOR GROUP BY` to the same scan + `USE TEMP B-TREE FOR DISTINCT`.
1310
- It is still an unindexed scan; the `idx_usage_week_time` reduction named above is
1311
- still not taken, for the reason given there.
1312
-
1313
- DO NOT QUOTE A SPLIT BETWEEN THE TWO REMOVALS. On those figures the aggregate is
1314
- 31% of the statement and the grouping a further 52%, but the review that checked
1315
- this measured 12.87-13.09 / 6.30-6.77 / 2.02-2.23 ms on its own 25,194-row store,
1316
- which makes the aggregate about 50% and the grouping about 34% — the same 83%
1317
- total, with the two components in the opposite order. The total reproduces and
1318
- the ranking does not, so treat the split as machine- and data-dependent.
1319
-
1320
- `week_start_at` / `week_end_at` are both nullable and
1321
- `_derive_week_from_payload` leaves them `None` on three of its four paths, so
1322
- a week with no bounds is a live shape; it is retained here with ``lo``/``hi``
1323
- of ``None`` and placed by date instead (see `_block_weekly_axis_weeks`).
1324
- """
1325
- if account_key in cache:
1326
- return cache[account_key]
1327
- try:
1328
- rows = conn.execute(
1329
- "SELECT DISTINCT week_start_date, week_end_date, week_start_at, "
1330
- " week_end_at "
1331
- " FROM weekly_usage_snapshots "
1332
- " WHERE account_key = ?",
1333
- (account_key,),
1334
- ).fetchall()
1335
- except sqlite3.DatabaseError:
1336
- rows = []
1337
- weeks: list = []
1338
- for r in rows:
1339
- lo, hi = _parsed_week_bounds(r["week_start_at"], r["week_end_at"])
1340
- weeks.append(_AxisWeek(
1341
- week_start_date=r["week_start_date"],
1342
- week_end_date=r["week_end_date"],
1343
- week_start_at=r["week_start_at"] if lo is not None else None,
1344
- week_end_at=r["week_end_at"] if hi is not None else None,
1345
- lo=lo, hi=hi))
1346
- cache[account_key] = weeks
1347
- return weeks
1348
-
1349
-
1350
- def _account_axis_capture_weeks(conn, *, account_key, cache):
1351
- """``{capture second -> the week identities a snapshot row at that instant
1352
- names}``, for the capture instants ``account_key``'s blocks actually carry.
1353
-
1354
- #834 S1 (#835) Gate A R9. THE STORE KEEPS A RECORD OF WHICH WEEK A CAPTURE WAS
1355
- OBSERVED UNDER, and this is the read of it. Round five documented its
1356
- over-withholding on the premise that it does not. All four writers of
1357
- `five_hour_blocks.seven_day_pct_at_block_start` / `_at_block_end` take the
1358
- matching axis STAMP from a `weekly_usage_snapshots` row's `captured_at_utc`:
1359
- the live upsert passes `captured_at = saved["capturedAt"]` into
1360
- `first_observed_at_utc` and `last_observed_at_utc` in one `INSERT … DO UPDATE`
1361
- and that value IS the row's `captured_at_utc` (bin/_cctally_record.py:2351,
1362
- :2633-2634, :6828), `_backfill_five_hour_blocks` takes them from its MIN- and
1363
- MAX-captured rows, `five_hour_block_close` freezes the row, and
1364
- `_migration_merge_5h_block_duplicates_v1` copies from the MAX-captured group
1365
- row. Every snapshot row carries all four week columns. So a row whose
1366
- `captured_at_utc` equals an axis instant NAMES the week in force at that
1367
- capture, with no date-range guess involved.
1368
-
1369
- HELD ROWS ARE NOT EXCLUDED, which is the one judgement call in this function.
1370
- The rest of this tranche excludes `weekly_observation_held = 1` from weekly
1371
- VALUE reads, because a held row carries a reading no fresh observation
1372
- confirmed. This function reads no value: it reads which week the value on the
1373
- block's axis came from, and on a held tick that is the BASIS week by
1374
- construction — the held write copies the basis's four week columns verbatim
1375
- while keeping the tick's own `capture_at` (bin/_cctally_record.py:7190-7203),
1376
- and the block's axis takes `fold.weekly.effective_pct`, the same
1377
- carried-forward value, stamped with that same `capture_at`. The held row is
1378
- therefore the provenance record of the number on the axis, and the credits
1379
- that retired that number are the basis week's.
1380
-
1381
- Two consequences follow. A held row is exactly how a pre-credit weekly value
1382
- reaches a post-credit capture instant, which is the case this whole tranche
1383
- exists to handle, and excluding it would drop the only exact evidence of which
1384
- week's floor applies. And because no predicate names
1385
- `weekly_observation_held`, a store that stops short of epoch 1015 is served
1386
- unchanged and `weekly_held_exclusion` is not needed here. Pinned by
1387
- `tests/test_five_hour_breakdown.py::test_835_r9_a_held_capture_record_names_the_basis_week_it_carried`
1388
- and `::test_835_r9_a_pre_column_store_resolves_the_capture_record`.
1389
-
1390
- THE SET IS BOUNDED BY THE ACCOUNT'S BLOCK COUNT, NOT BY ITS SNAPSHOT HISTORY,
1391
- and the form was chosen on measurement rather than on shape. Re-measured at Gate
1392
- A R10, because R10 replaced `_account_axis_weeks`' grouped query with a plain
1393
- `SELECT DISTINCT` and the earlier round's figures for that statement no longer
1394
- described what ships. All three were timed in ONE session on a 25,194-row,
1395
- 13-week, 50-block store — the store `_account_axis_weeks` documents its own
1396
- numbers against — best of five each, with the order reversed to rule out
1397
- page-cache bias:
1398
-
1399
- - this narrowed query: 2.10-2.27 ms, 50 rows, `SCAN weekly_usage_snapshots`
1400
- plus a `LIST SUBQUERY` that SEARCHes `five_hour_blocks` on its own index;
1401
- - `_account_axis_weeks`' `SELECT DISTINCT`: 2.06-2.13 ms, 13 rows;
1402
- - one ungrouped full-row scan serving BOTH passes from a single statement:
1403
- 14.18-14.38 ms, 25,194 rows, a bare `SCAN weekly_usage_snapshots`.
1404
-
1405
- So the two statements together cost 4.40 ms against the single statement's
1406
- 14.18 ms and retain 63 rows against 25,194 — a 400th. Two statements per account
1407
- per render is the cheaper form on both axes, and it keeps the property that
1408
- matters: the cost is CONSTANT in the number of rendered blocks. Bounded by the
1409
- account's own `five_hour_blocks` rows rather than by which of them this render
1410
- shows, which is why no instant range has to be threaded down from the caller.
1411
-
1412
- TWO EARLIER ROUNDS DISAGREED ABOUT THE SINGLE-STATEMENT FIGURE — this docstring
1413
- said 22.03 ms and `docs/five-hour-gotchas.md` said 21.97 ms for the same query on
1414
- the same store — and neither number survives, because both were taken against the
1415
- superseded grouped form of the week query. The figures above replace both rather
1416
- than reconciling them.
1417
-
1418
- THE NARROWING AND THE MATCH USE DIFFERENT READERS, deliberately. SQLite's
1419
- `unixepoch()` selects the rows; `parse_iso_datetime(...).timestamp()` keys
1420
- them and keys the probe, so the match itself is Python's and is consistent
1421
- between map and probe. Both readers truncate to the second, which is how two
1422
- ISO spellings of one moment — `Z` against `+00:00` — compare equal.
1423
-
1424
- A NAIVE STAMP IS A LIVE SHAPE, and the two readings of it differ. An earlier
1425
- form of this paragraph said no writer produces one, citing `now_utc_iso()`.
1426
- That is wrong: `_coerce_payload_captured_at` (bin/_cctally_record.py:6576-6585)
1427
- returns the PAYLOAD's `capturedAt` verbatim whenever `parse_iso_datetime`
1428
- accepts it, `parse_iso_datetime` accepts a naive string and reads it host-local,
1429
- and `insert_usage_snapshot` then stores that text. Driven with
1430
- `TZ=America/Los_Angeles`: a payload `capturedAt` of `2026-04-30T13:00:00` is
1431
- stored as `captured_at_utc = '2026-04-30T13:00:00'`, and `unixepoch()` reads it
1432
- 1777554000 against Python's 1777579200.
1433
-
1434
- THE FAILURE IS A MIXED PAIR, NOT A NAIVE STAMP, AND IT IS THE MATCH THAT MISSES
1435
- RATHER THAN THE NARROWING THAT EXCLUDES. The same earlier paragraph stated it
1436
- the other way round. Each reader is self-consistent across the two sides it
1437
- compares, so the spelling only matters when the snapshot stamp and the block
1438
- stamp are spelled DIFFERENTLY. All four pairings, driven on that same non-UTC
1439
- host: equal naive text narrows IN and the Python match HITS; `Z` against
1440
- `+00:00` narrows IN and HITS; naive against offset-carrying, in either
1441
- direction, narrows IN and the PYTHON MATCH MISSES, after which the instant falls
1442
- through to the two fallback passes.
1443
-
1444
- SO THE RESIDUAL IS NARROWER THAN A NAIVE STAMP'S EXISTENCE. Every one of the
1445
- four axis writers takes the block stamp from a `weekly_usage_snapshots` row's
1446
- own `captured_at_utc` text — the live upsert from `saved["capturedAt"]`, the
1447
- backfill from its MIN- and MAX-captured rows, the close from the frozen row, the
1448
- merge migration from the MAX-captured group row — so the pair the capture pass
1449
- compares always shares a spelling and the pass is correct on a naive store. A
1450
- mixed pair needs the two texts to diverge after both were written, which is a
1451
- hand-built or externally-edited store. The degradation is then a fallback rather
1452
- than a misclassification, which is the one part of the earlier paragraph that
1453
- held.
1454
- """
1455
- if account_key in cache:
1456
- return cache[account_key]
1457
- try:
1458
- rows = conn.execute(
1459
- "SELECT captured_at_utc, week_start_date, week_start_at, week_end_at "
1460
- " FROM weekly_usage_snapshots "
1461
- " WHERE account_key = ? "
1462
- " AND unixepoch(captured_at_utc) IN ("
1463
- " SELECT unixepoch(first_observed_at_utc) "
1464
- " FROM five_hour_blocks WHERE account_key = ? "
1465
- " UNION "
1466
- " SELECT unixepoch(last_observed_at_utc) "
1467
- " FROM five_hour_blocks WHERE account_key = ?)",
1468
- (account_key, account_key, account_key),
1469
- ).fetchall()
1470
- except sqlite3.DatabaseError:
1471
- rows = []
1472
- carried: dict = {}
1473
- for r in rows:
1474
- try:
1475
- second = int(parse_iso_datetime(
1476
- r["captured_at_utc"], "captured_at_utc").timestamp())
1477
- except (ValueError, TypeError):
1478
- continue
1479
- lo, _hi = _parsed_week_bounds(r["week_start_at"], r["week_end_at"])
1480
- identity = (
1481
- r["week_start_date"],
1482
- r["week_start_at"] if lo is not None else None,
1483
- r["week_end_at"] if lo is not None else None,
1484
- )
1485
- # A dict as an ordered set: two rows of one week at one instant are one
1486
- # identity, and `_credit_retirement_bands` would answer both alike.
1487
- carried.setdefault(second, {})[identity] = None
1488
- out = {second: tuple(ids) for second, ids in carried.items()}
1489
- cache[account_key] = out
1490
- return out
1491
-
1492
-
1493
- def _account_axis_bounds_by_start_date(weeks):
1494
- """``{week_start_date -> that week's bounds-carrying identities}``, from an
1495
- already-loaded `_account_axis_weeks` set.
1496
-
1497
- #834 S1 (#835) Gate A R10. The index the capture pass substitutes from when its
1498
- matched row carries no bounds of its own. Derived from the account's OWN week
1499
- set and nothing else, which is the property that separates it from
1500
- `_get_canonical_boundary_for_date`; built once per account per render, so the
1501
- substitution adds no query and no per-instant scan.
1502
-
1503
- Bounds-free weeks are absent by construction: an entry exists only for a
1504
- `week_start_date` some row of that account spells bounds for, which is exactly
1505
- the condition under which a substitution is possible.
1506
- """
1507
- out: dict = {}
1508
- for week in weeks:
1509
- if week.lo is None:
1510
- continue
1511
- out.setdefault(week.week_start_date, {})[
1512
- (week.week_start_date, week.week_start_at, week.week_end_at)] = None
1513
- return {date: tuple(ids) for date, ids in out.items()}
1514
-
1515
-
1516
- def _block_weekly_axis_weeks(conn, *, account_key, instant_iso, cache):
1517
- """Every week identity whose credits can have retired a value captured at
1518
- ``instant_iso`` under ``account_key``, as ``(week_start_date, week_start_at,
1519
- week_end_at)`` triples. Empty when no week names the instant.
1520
-
1521
- THREE PASSES, IN THIS ORDER: the CAPTURE pass matches the instant against the
1522
- `captured_at_utc` of the account's own snapshot rows and answers from the week
1523
- those rows name; the EPOCH pass places the instant inside a week's parsed
1524
- `[week_start_at, week_end_at)` bounds; the DATE pass asks which bounds-free
1525
- weeks declare the instant's calendar day. Each runs only when the one before it
1526
- resolved nothing.
1527
-
1528
- THE CAPTURE PASS IS EXACT ABOUT WHICH WEEK, AND IT REPLACED AN ARGUMENT THAT
1529
- WAS WRONG. Gate A R8 had only the other two, and round five defended the
1530
- over-withholding they cause on the premise that the store keeps no record of
1531
- which week a capture was observed under. It keeps exactly that record: every
1532
- axis stamp on a block IS some snapshot row's `captured_at_utc`, and that row
1533
- carries all four week columns — see `_account_axis_capture_weeks`, which also
1534
- records why held rows are consulted rather than excluded. So for an instant a
1535
- capture record carries there is no week to guess, and neither fallback runs.
1536
-
1537
- BEING EXACT ABOUT THE WEEK IS NOT THE SAME AS RESOLVING BOTH LEGS, which an
1538
- earlier form of this paragraph left unsaid while the date pass's own paragraphs
1539
- below state it carefully. `week_start_at` and `week_end_at` are nullable on the
1540
- matched row too, and `_credit_retirement_bands`' `week_reset_events` leg is
1541
- scoped purely by containment of `effective_reset_at_utc` in
1542
- ``[week_start_at, week_end_at)`` — that table has no `week_start_date` column at
1543
- all — so an identity carrying no bounds resolves the `weekly_credit_floors` leg
1544
- and NOT the automatic one. Through R9 that degradation combined with this pass's
1545
- precedence into the tranche's own defect class: a bounds-free capture row
1546
- pre-empted a bounds-carrying placement of the SAME week and the value an
1547
- automatic credit retired was published. R10 closes it by SUBSTITUTING the
1548
- account's own bounds (next paragraph), so the per-leg behaviour now is that both
1549
- legs resolve whenever any row of the named week carries parseable bounds, and
1550
- the manual leg alone when none does.
1551
-
1552
- A BOUNDS-FREE CAPTURE IDENTITY IS REPLACED BY THE ACCOUNT'S OWN BOUNDS-CARRYING
1553
- ROWS OF THE SAME `week_start_date`, and the substitution is lossless. Those rows
1554
- are already in memory from `_account_axis_weeks`, so no query is added. It is
1555
- lossless because every substituted identity shares the bounds-free one's
1556
- `week_start_date`, so each resolves a `weekly_credit_floors` leg identical to the
1557
- one it replaces and ADDS the automatic leg its bounds can scope; the replaced
1558
- identity therefore contributes no band the substitutes do not. Several of them
1559
- are unioned, the way the epoch pass unions overlapping candidates, and that
1560
- union is here FOR ROBUSTNESS rather than for a reachable shape: no writer is
1561
- known to put two bound pairs under one `week_start_date`. What pins one pair per
1562
- date is `_usage_snapshot_columns` (bin/_cctally_record.py:6698-6710), which
1563
- overrides every later row's derived bounds with `_get_canonical_boundary_for_date`
1564
- — the FIRST established pair for that date, `ORDER BY captured_at_utc ASC, id ASC
1565
- LIMIT 1` — so a sub-day re-anchoring is overridden back to the first pair, and a
1566
- day-scale one lands under a DIFFERENT `week_start_date` because the date derives
1567
- from the payload's own week start. No code change is owed either way, because
1568
- `named.update(dict.fromkeys(… or (identity,)))` already handles one through N.
1569
- Two earlier forms of this sentence cited a producer and both were false. Do NOT
1570
- cite the timezone fork: it produces the INVERSE shape, identical bounds under two
1571
- different `week_start_date` values (see the two-identities paragraph below),
1572
- which puts one identity under each date rather than two under one. Do NOT cite
1573
- the boundary-shift branch of `_backfill_week_reset_events`
1574
- (bin/_cctally_weekrefs.py) either: it writes only `week_reset_events` and never
1575
- the snapshot bounds this substitution reads, and its reset test needs THREE
1576
- signals rather than the two an earlier form named — the boundary shifted, the
1577
- prior boundary was still in the future, AND `_is_reset_drop(prior_pct, cur_pct,
1578
- allow_reset_to_zero=False)`, which with the zero leg off is the >=25pp drop
1579
- (`bin/_cctally_weekrefs.py:625-637`).
1580
-
1581
- CONTAINMENT OF THE INSTANT IS DELIBERATELY NOT REQUIRED OF A SUBSTITUTE, and
1582
- requiring it would reopen the defect on the shape this tranche exists for. A
1583
- HELD row carries the BASIS week's four columns with the tick's own capture
1584
- stamp, so a held tick after the week's end names a week whose bounds do not
1585
- contain its own capture instant. Filtering substitutes by containment would
1586
- decline to enrich exactly there.
1587
-
1588
- THAT GENERALITY IS WIDER THAN THE HELD-ROW ARGUMENT NEEDS, and the excess is a
1589
- stated limit rather than a defended choice. The held-row shape requires only that
1590
- containment of the CAPTURE instant not be required. Nothing here additionally
1591
- requires a substitute's bounds to be consistent with the `week_start_date` they
1592
- are filed under, so a same-date row spelling bounds that start a week early lets
1593
- the `week_reset_events` leg admit a reset belonging to the PREDECESSOR week, and
1594
- the value is withheld although no credit of the named week retired it. No writer
1595
- produces a date/bounds inconsistency of that size, so the shape needs a
1596
- hand-built or externally-edited store; the narrower rule that would close it is
1597
- to require the substitute's `week_start_at` UTC date to sit within a day of
1598
- `week_start_date`, and it is not taken because the reachable benefit is nil and a
1599
- date/bounds tolerance is one more constant to get wrong.
1600
-
1601
- NOTHING WIDER THAN THE ACCOUNT'S OWN ROWS IS CONSULTED.
1602
- `_get_canonical_boundary_for_date` (bin/_cctally_weekrefs.py:51) is the repo's
1603
- canonical-bounds reader and the obvious candidate, and it carries no
1604
- `account_key` predicate: it returns the EARLIEST bounds-carrying row for a
1605
- `week_start_date` across every account. On a multi-account store that does not
1606
- merely attach the wrong provenance, it fails to fix the defect, because another
1607
- account's re-anchored bounds cannot scope THIS account's credit — driven and
1608
- pinned by
1609
- `tests/test_five_hour_breakdown.py::test_835_r10_the_substitution_takes_the_accounts_own_bounds_not_another_accounts`.
1610
-
1611
- ONE RESIDUAL SURVIVES, AND IT IS PRE-EXISTING RATHER THAN INTENDED. When NO row
1612
- of the named week carries parseable bounds there is nothing account-scoped to
1613
- substitute, so an automatic credit in that week is structurally invisible to
1614
- this filter and the value it retired is published. Closing it needs a source of
1615
- a week's bounds outside the account's own snapshot rows, and the paragraph above
1616
- says why the one that exists is not usable. Pinned, as a gap rather than as
1617
- correct behaviour, by the second half of
1618
- `::test_835_r10_a_bounds_free_capture_record_resolves_the_weeks_automatic_credit`.
1619
-
1620
- A CAPTURE INSTANT CARRIED BY TWO WEEK IDENTITIES RETURNS BOTH. No tie-break is
1621
- invented for it and the caller's union withholds if either week retired the
1622
- value, which is this subsystem's established direction and what both fallback
1623
- passes do as well. The shape is not hypothetical: bin/_cctally_record.py:6601-6614
1624
- records a host briefly running the wrong timezone forking ONE physical week's
1625
- `week_start_date` across 18 rows, and `weekly_credit_floors` matches
1626
- `week_start_date` by equality, so a floor can sit under one fork and not the
1627
- other.
1628
-
1629
- #834 S1 (#835) Gate A S1. `five_hour_blocks` carries no week columns, and the
1630
- retirement bands need all three: the `weekly_credit_floors` leg matches
1631
- `week_start_date` by equality and the `week_reset_events` leg is scoped by the
1632
- week's bounds. The week is therefore resolved from the account's own snapshot
1633
- rows, whose `week_start_at` / `week_end_at` are the same bounds
1634
- `_credit_retirement_bands` is written against.
1635
-
1636
- The comparison is on PARSED epochs rather than on text, for the reason every
1637
- SQL site in this area wraps both sides in ``unixepoch()``: `block_start_at` is
1638
- host-local while `week_start_at` is `+00:00`, so a lexicographic compare would
1639
- mis-order them.
1640
-
1641
- THE TWO LEGS DEGRADE INDEPENDENTLY, the way `_credit_retirement_bands`' own two
1642
- legs already do (#834 S1 Gate A R8). `week_start_at` and `week_end_at` are both
1643
- nullable and `_derive_week_from_payload` leaves them ``None`` on three of its
1644
- four paths, so a week with no bounds is a live shape. Gate A S1 required both
1645
- bounds and resolved NOTHING without them, which threw away a leg it could still
1646
- answer: the `weekly_credit_floors` leg matches on `week_start_date` alone, and
1647
- `week_start_date` / `week_end_date` are `TEXT NOT NULL`. So a bounds-free week
1648
- is placed by DATE instead and returned with ``None`` bounds, which resolves the
1649
- manual-credit leg and skips the `week_reset_events` leg — exactly what
1650
- `_credit_retirement_bands` does with absent bounds.
1651
-
1652
- A bounds-carrying week outranks a bounds-free one, so the date pass runs only
1653
- when the epoch pass places the instant nowhere. The
1654
- date window is INCLUSIVE of `week_end_date`, because the usual bounds-free week
1655
- carries the week's LAST DAY there: `_derive_week_from_payload`'s third path
1656
- writes `week_start_date + 6 days` and its fourth derives the same through
1657
- `compute_week_bounds`. THE TWO READINGS DO MEET, which an earlier form of this
1658
- docstring denied. `_derive_week_from_payload`'s second path takes both dates
1659
- from the payload verbatim behind nothing but a `weekEndDate >= weekStartDate`
1660
- check, and `_account_axis_weeks` sends a week down this pass when its bounds are
1661
- present but UNPARSEABLE, where `week_end_date` came from the bounds-carrying
1662
- writer and is the exclusive boundary's date by construction. Either shape puts
1663
- one calendar day inside two weeks' inclusive ranges.
1664
-
1665
- SO THE DATE PASS RETURNS EVERY CANDIDATE THAT CONTAINS THE DAY AND PICKS NONE,
1666
- and the caller unions their bands. Two single-choice rules were tried and each
1667
- published a retired value on the mirror image of the shape it fixed. Rank order
1668
- placed an instant in the PREDECESSOR whenever the predecessor's newest snapshot
1669
- row was the newer of the two. Preferring the greatest `week_start_date` fixed
1670
- that and broke the case where the floor sits on a LONGER declared week
1671
- `2026-04-20..2026-05-04` beside an uncredited `2026-04-27..2026-05-04` — the
1672
- exact shape whose existence is the argument against a seven-day window, so the
1673
- prohibition and that repair rested on one premise pointing opposite ways. No
1674
- reading of one week's own columns separates two candidates that both declare the
1675
- day, so choosing is a guess in either direction and consulting all of them is
1676
- not.
1677
-
1678
- NEITHER FALLBACK PASS HAS A TIE-BREAK ANY MORE. Both return every candidate they
1679
- place the instant in, and the caller unions the bands. Through Gate A R8 the
1680
- epoch pass returned ONE week — the one whose group held the newest
1681
- `(captured_at_utc, id)`, which is what the since-removed `rk` aggregate ordered
1682
- by — justified by the claim that the newest anchoring is the best available
1683
- evidence of which window was in force at capture. That claim is vacuous on the shape that makes overlapping
1684
- bounds-carrying anchorings reachable at all: bin/_cctally_record.py:6601-6614
1685
- records a host running the wrong timezone for seven minutes forking ONE physical
1686
- week's `week_start_date` across 18 rows, and those two anchorings carry
1687
- IDENTICAL bounds and differ only in the column `weekly_credit_floors` is keyed
1688
- on. Both anchorings are the same window, so there is no "which window was in
1689
- force" to answer, and a newest-anchoring choice decides only which of two
1690
- spellings of one week's floors gets consulted. It also inverts the claim's own
1691
- logic: where two anchorings genuinely differ, that tie-break publishes the
1692
- loser's retired value rather than withholding it.
1693
-
1694
- The two passes still differ in PRECEDENCE, deliberately. The epoch pass has real
1695
- half-open boundaries to test an instant against, so every candidate it keeps
1696
- genuinely contains the instant; the date pass has no boundaries, only two date
1697
- columns of which a live writer fills one with the EXCLUSIVE boundary's date, so
1698
- "contains the day" there does not mean "belongs to the week". A bounds-carrying
1699
- placement is therefore the more precise one and pre-empts the date pass
1700
- entirely. Pinned by
1701
- `tests/test_five_hour_breakdown.py::test_835_r9_the_epoch_pass_unions_every_bounds_carrying_week_it_contains`
1702
- and `::test_835_r8_a_bounds_carrying_week_outranks_a_bounds_free_one`.
1703
-
1704
- THE DATE PASS'S UNION OVER-WITHHOLDS, AND THAT RESIDUAL IS SMALLER THAN ROUND
1705
- FIVE'S. When an instant genuinely belongs to an uncredited candidate and an
1706
- overlapping candidate was credited, the union withholds a value no credit ever
1707
- retired. Round five paid that on EVERY block observed on a week-boundary day,
1708
- because `cmd_record_usage` writes `week_end_date = week_start_date + 7 days` —
1709
- the exclusive boundary's date — while this pass compares inclusively, so every
1710
- pair of adjacent weeks shares one calendar day. The capture pass now answers
1711
- every such instant some snapshot row carries, which is the ordinary case, and
1712
- what is left is an instant NO capture record carries on a store whose weeks have
1713
- no parseable bounds.
1714
-
1715
- The direction is the conservative one — an unavailable marker in preference to a
1716
- number a credit retired — and both fallback passes now take it. It is still not
1717
- the direction EVERY fork in this change takes: an empty result resolves no bands
1718
- and publishes the raw value, which is the pre-#835 disposition kept on purpose.
1719
- `::test_835_r8_the_union_withholds_an_uncredited_weeks_own_value` pins the trade
1720
- rather than leaving a later reader to find it and read it as a defect, and
1721
- `::test_835_r9_a_capture_record_resolves_the_week_a_shared_boundary_day_withheld`
1722
- pins the boundary-day case that no longer pays it.
1723
-
1724
- A SEVEN-DAY WINDOW OFF `week_start_date` IS STILL NOT AN ALTERNATIVE. It assumes
1725
- every bounds-free week is seven days long, and the second payload path can
1726
- declare a longer one, whose tail a seven-day window would place NOWHERE —
1727
- withdrawing the manual-credit leg this pass exists to resolve. It also buys
1728
- nothing on the weeks whose `week_end_date` is computed rather than declared: the
1729
- third path and `compute_week_bounds` both write `week_start_date + 6 days`, and
1730
- `cmd_record_usage` writes `week_end_date` exactly `week_start_date + 7 days`, so
1731
- over those rows a seven-day exclusive window and this inclusive one name the
1732
- same days and neither separates two overlapping candidates.
1733
-
1734
- ONE DIRECTION IS BOUNDED BY `cmd_record_credit`, AND THE OTHER IS NOT. That
1735
- command refuses without a canonical weekly boundary and always writes a
1736
- bounds-carrying synthetic snapshot of its own, so a manually credited week
1737
- normally has at least one row with bounds — which bounds the direction where a
1738
- credited week is reachable ONLY through the date pass. It does not bound the
1739
- opposite direction, the one round five introduced: that same credited week's
1740
- OTHER rows still form a bounds-free candidate group, because `week_end_date`
1741
- joins the grouping, so an instant in an adjacent bounds-free week still picks up
1742
- the credited week's floor and over-withholds. Do not read the synthetic snapshot
1743
- as bounding both.
1744
-
1745
- EMPTY when no week names the instant on any pass. That resolves no bands and
1746
- therefore withholds nothing, which is the pre-#835 behaviour.
1747
-
1748
- ``cache`` IS THE OUTER RENDER CACHE, NOT A WEEK MEMO, AND IT CHANGED SHAPE
1749
- SILENTLY AT GATE A R8. Gate A S1 passed a dict this function used directly as
1750
- ``{(account_key, instant_iso) -> weeks}``. It now carves its own sub-dicts out
1751
- of the dict the caller owns, and that dict is the same one
1752
- `_credit_aware_block_weekly_axes` carves `__bands` out of, so FOUR reserved
1753
- keys share one namespace:
1754
-
1755
- ``__weeks`` `{account_key -> [_AxisWeek]}`, `_account_axis_weeks`' memo
1756
- ``__stamps`` `{account_key -> {second -> identities}}`, the capture records
1757
- ``__by_date`` `{account_key -> {week_start_date -> identities}}` (R10)
1758
- ``__bands`` `{(week, account_key) -> bands}`, owned by the caller
1759
-
1760
- A caller passing the OLD shape is not detected and does not raise: every
1761
- `setdefault` above finds no reserved key, creates a fresh sub-dict, and the
1762
- pre-existing entries are simply never read. The failure is therefore a silent
1763
- loss of memoization — one extra pair of queries per account per render, the cost
1764
- the per-account load exists to remove — and never a wrong answer, because
1765
- nothing reads a key it did not write. The leading double underscore is what
1766
- keeps the two populations apart: an `account_key` is either a 32-character
1767
- lowercase sha256 prefix (`digest.hexdigest()[:32]`, bin/_lib_accounts.py:10,
1768
- :63 and :70) or the literal `unattributed` (bin/_lib_accounts.py:41), so none of
1769
- the four names can collide with one.
1770
-
1771
- EVERY PASS IS GATED ON `_account_axis_weeks` SUCCEEDING, the capture pass
1772
- included, and that coupling is deliberate. It looks gratuitous, because NEITHER
1773
- QUERY'S COLUMN DEPENDENCY CONTAINS THE OTHER'S, so in principle a store that
1774
- fails one could answer the other. The week query reads `week_start_date`,
1775
- `week_end_date`, `week_start_at` and `week_end_at`; the capture query reads
1776
- `captured_at_utc`, `week_start_date`, `week_start_at` and `week_end_at`, plus
1777
- `first_observed_at_utc` and `last_observed_at_utc` from a SECOND table,
1778
- `five_hour_blocks`. So the divergence has two directions — a store carrying the
1779
- bounds without `week_end_date`, and one carrying `week_end_date` without
1780
- `captured_at_utc` or without the block table — and the claim that the capture
1781
- query's dependency was strictly smaller was FALSE WHEN WRITTEN in R10, because
1782
- the capture query has read a second table since R9. Two earlier statements of
1783
- this are withdrawn: the claim was authored by `f32554505`, which is later than
1784
- `df80ca1dd` that removed the week query's aggregate, so it cannot have "stopped
1785
- being true" in that round; and it was never true, because the capture query has
1786
- selected `first_observed_at_utc` / `last_observed_at_utc` from `five_hour_blocks`
1787
- since `9690c0fae`. Three reasons the gate is kept:
1788
-
1789
- - The divergence is unreachable. `week_end_date` is `TEXT NOT NULL` in the
1790
- base `CREATE TABLE` (bin/_cctally_core.py:2127) and no migration adds it, so
1791
- no cctally store has the other four columns without it. The legacy shape
1792
- `bin/build-doctor-fixtures.py` builds lacks `week_start_at` / `week_end_at`
1793
- as well, so BOTH queries fail on it together.
1794
- - An empty week set from the ordinary cause — the account has no snapshot rows
1795
- — implies an empty capture map, because the capture query reads the same
1796
- table filtered on the same `account_key`.
1797
- - Since R10 the capture pass READS `weeks`, to build the substitution index
1798
- `_account_axis_bounds_by_start_date` returns, so running it without them
1799
- degrades it to returning bounds-free identities unenriched.
1800
-
1801
- THAT THIRD REASON IS NOT THAT DECOUPLING WOULD BUY NOTHING. An unenriched
1802
- bounds-free identity still resolves the `weekly_credit_floors` leg, and the empty
1803
- tuple resolves neither leg, so on a store where the week query failed and the
1804
- capture query succeeded, decoupling WOULD buy the manual leg. What makes the gain
1805
- unreachable is reason 1, not any equivalence between the two dispositions. The
1806
- gate is kept because the divergence cannot occur on a store any cctally version
1807
- created, and decoupling would add a second exit path to reason about for a gain
1808
- no store can realise. If a future store shape makes the divergence real, the fix
1809
- is to run the capture pass first and gate only the two fallback passes.
1810
- """
1811
- weeks = _account_axis_weeks(
1812
- conn, account_key=account_key, cache=cache.setdefault("__weeks", {}))
1813
- if not weeks:
1814
- # EVERY pass is gated on the week load, the capture pass included. See the
1815
- # docstring: the coupling is deliberate and, since R10, load-bearing.
1816
- return ()
1817
- try:
1818
- instant = parse_iso_datetime(instant_iso, "axis_instant")
1819
- except (ValueError, TypeError):
1820
- return ()
1821
- moment = instant.timestamp()
1822
- # PASS 1 — the capture record. Truncated to the second on both sides, matching
1823
- # the `unixepoch()` the narrowing query uses; see `_account_axis_capture_weeks`.
1824
- carried = _account_axis_capture_weeks(
1825
- conn, account_key=account_key,
1826
- cache=cache.setdefault("__stamps", {})).get(int(moment))
1827
- if carried:
1828
- # A bounds-free matched row names the week but cannot scope the
1829
- # `week_reset_events` leg, and this pass pre-empts the epoch pass, so
1830
- # substitute the SAME week's bounds as this account's OWN other rows spell
1831
- # them — see the docstring for why the substitution is lossless, why
1832
- # containment is not required of a substitute, and what residual survives.
1833
- named: dict = {}
1834
- for identity in carried:
1835
- if identity[1] is not None:
1836
- named[identity] = None
1837
- continue
1838
- by_date = cache.setdefault("__by_date", {})
1839
- if account_key not in by_date:
1840
- by_date[account_key] = _account_axis_bounds_by_start_date(weeks)
1841
- # Built once per account from the already-loaded week set, so the
1842
- # substitution costs no query and no per-instant scan.
1843
- named.update(dict.fromkeys(
1844
- by_date[account_key].get(identity[0]) or (identity,)))
1845
- return tuple(named)
1846
- # PASS 2 — containment in a week's parsed half-open bounds. A dict as an
1847
- # ordered set throughout: the caller's union is order-insensitive, so the
1848
- # order these are visited in carries no meaning, and membership must not be a
1849
- # linear scan.
1850
- found: dict = {}
1851
- for week in weeks:
1852
- if week.lo is not None and week.lo <= moment < week.hi:
1853
- found[(week.week_start_date, week.week_start_at, week.week_end_at)] = (
1854
- None)
1855
- if found:
1856
- return tuple(found)
1857
- # PASS 3 — the instant's calendar day against the bounds-free weeks' declared
1858
- # date ranges. `.astimezone(utc)` before `.date()` for
1859
- # `_derive_week_from_payload`'s own reason: `parse_iso_datetime` ends in
1860
- # `.astimezone()`, so on a host whose offset puts the moment on another calendar
1861
- # date a bare `.date()` would place the instant in the wrong week. EVERY
1862
- # candidate that declares the day, with no preference among them — see the
1863
- # docstring for why no preference can be grounded here and what the union costs.
1864
- day = instant.astimezone(dt.timezone.utc).date().isoformat()
1865
- for week in weeks:
1866
- if week.lo is not None:
1867
- continue
1868
- # Both columns are `TEXT NOT NULL`, so a None here is a store nobody can
1869
- # place; skip it rather than raise comparing a string to None.
1870
- if week.week_start_date is None or week.week_end_date is None:
1871
- continue
1872
- if week.week_start_date <= day <= week.week_end_date:
1873
- # `week_end_date` is in `_account_axis_weeks`' DISTINCT projection, so
1874
- # two rows differing only there are two candidates carrying ONE identity.
1875
- # The bands would be identical, so the duplicate is dropped rather than
1876
- # resolved twice.
1877
- found[(week.week_start_date, None, None)] = None
1878
- return tuple(found)
1879
-
1880
-
1881
1071
  def _credit_aware_block_weekly_axes(conn, block, *, now_utc, cache=None):
1882
- """A block's two weekly axes, with a credit-RETIRED value replaced by ``None``.
1883
-
1884
- #834 S1 (#835) Gate A S1. THE ONE READ-TIME FILTER for
1885
- `five_hour_blocks.seven_day_pct_at_block_start` and
1886
- `seven_day_pct_at_block_end`, called by `cmd_five_hour_blocks` and
1887
- `cmd_five_hour_breakdown` so the two cannot diverge.
1888
-
1889
- WHY READ TIME, AND WHY A WRITE-TIME FILTER CANNOT SUBSTITUTE. A credit can
1890
- retire a value AFTER the block row was written — that is the ordinary case,
1891
- because the live upsert writes the block's weekly end on every tick while the
1892
- block is open and the credit fires later — so no write-time pass can ever be
1893
- sufficient. FOUR writers store these columns, not three: the live upsert in
1894
- bin/_cctally_record.py, `five_hour_block_close`'s frozen harvest in
1895
- bin/_cctally_journal.py, `_backfill_five_hour_blocks` below, and
1896
- `_migration_merge_5h_block_duplicates_v1` in bin/_cctally_db.py, which writes
1897
- `seven_day_pct_at_block_end` from the group row whose `last_observed_at_utc` is
1898
- MAX. The migration was omitted from the original count; the argument is
1899
- unaffected, because it also stores what was observed, but the number was
1900
- documented and wrong. All four store what the fold OBSERVED; this one read
1901
- decides what a credit retired may still be shown. Making one writer
1902
- credit-aware instead would also break
1903
- rebuild convergence, because the store would then hold a number where a
1904
- rebuilt store holds `NULL` and `tests/test_rederive_command.py` compares
1905
- exactly these two columns across a rebuild.
1906
-
1907
- EACH AXIS IS COMPARED AGAINST THE INSTANT THAT AXIS'S VALUE WAS CAPTURED AT,
1908
- never against a boundary of the block: the start against
1909
- `_block_weekly_start_instant` (`first_observed_at_utc`), the end against
1910
- `_block_weekly_end_instant` (`last_observed_at_utc`). Gate A S1 used the block's
1911
- own boundary stamps instead and that was wrong in both directions — see those
1912
- two helpers. The rule separates the three positions a block's OBSERVATIONS can
1913
- hold relative to a credit floor. Observations entirely BEFORE the floor render
1914
- both stored values, because they genuinely were effective while the block ran
1915
- and withholding them would destroy historical truth. Observations SPANNING the
1916
- floor render the start and withhold a retired end. Observations entirely AFTER
1917
- the floor withhold both, which is the shape a post-credit replay at the retired
1918
- value produces.
1919
-
1920
- The bands come from the block's OWN week and `account_key` — a credit under
1921
- one account retires nothing under another, and a crossed-reset block's two
1922
- axes legitimately resolve two different weeks.
1923
-
1924
- ``cache`` is an optional dict shared across a multi-row render, and it holds
1925
- FOUR reserved keys in ONE namespace: this function's own ``__bands``, resolved
1926
- once per `(week, account_key)`, plus the ``__weeks``, ``__stamps`` and
1927
- ``__by_date`` that `_block_weekly_axis_weeks` carves out of the same dict. That
1928
- function's docstring states each one's shape and what happens to a caller who
1929
- passes the pre-R8 shape, which is a silent loss of memoization rather than a
1930
- wrong answer. Together they hold the account's whole week set, the capture
1931
- records of that account's block axis stamps, the per-`week_start_date` index of
1932
- bounds-carrying identities, and the bands. Each axis instant is then resolved
1933
- against the loaded sets in memory, so the number of queries does not grow with
1934
- the number of rendered blocks. The caller owns the dict and it must not outlive
1935
- the connection.
1936
- """
1937
- _c = _cctally()
1938
- p_start = block.get("seven_day_pct_at_block_start")
1939
- p_end = block.get("seven_day_pct_at_block_end")
1940
- if p_start is None and p_end is None:
1941
- return None, None
1942
- if cache is None:
1943
- cache = {}
1944
- bands = cache.setdefault("__bands", {})
1945
- account_key = block.get("account_key") or _lib_accounts.UNATTRIBUTED
1946
- out = []
1947
- for value, instant in (
1948
- (p_start, _block_weekly_start_instant(block)),
1949
- (p_end, _block_weekly_end_instant(block, now_utc=now_utc)),
1950
- ):
1951
- if value is None or not instant:
1952
- out.append(value)
1953
- continue
1954
- week_ids = _block_weekly_axis_weeks(
1955
- conn, account_key=account_key, instant_iso=instant, cache=cache)
1956
- if not week_ids:
1957
- out.append(value)
1958
- continue
1959
- # THE UNION OF EVERY CANDIDATE WEEK'S BANDS, so a value any one of them
1960
- # retired is withheld. One bounds-carrying week is one candidate; several
1961
- # bounds-free ones are several, because nothing in their own columns says
1962
- # which of them an instant belongs to. Each is still resolved by
1963
- # `_credit_retirement_bands` from its OWN triple, so the per-leg
1964
- # degradation is unchanged: a bounds-free candidate contributes its
1965
- # `weekly_credit_floors` leg and no `week_reset_events` leg, which cannot
1966
- # be scoped without bounds. The memo stays per `(week, account_key)`, so
1967
- # overlapping candidates shared across blocks resolve once each.
1968
- merged: list = []
1969
- for week in week_ids:
1970
- bk = (week, account_key)
1971
- if bk not in bands:
1972
- bands[bk] = _c._credit_retirement_bands(
1973
- conn, week_start_date=week[0], week_start_at=week[1],
1974
- week_end_at=week[2], account_key=account_key)
1975
- merged.extend(bands[bk])
1976
- out.append(
1977
- None if _c._weekly_value_is_retired_replica(
1978
- merged, captured_at_utc=instant, weekly_percent=value)
1979
- else value)
1980
- return out[0], out[1]
1072
+ """A block's two stored weekly axes,
1073
+ ``(seven_day_pct_at_block_start, seven_day_pct_at_block_end)``.
1074
+
1075
+ #840 (spec section 4.8). #835 replaced a value within a point of one a
1076
+ credit retired with ``None`` here, which also withheld every genuine climb
1077
+ back to that value; the retirement band is removed. Stale contributors are
1078
+ refused before publication (spec section 4.4), and a value already
1079
+ poisoned in a store is corrected by `db rederive`'s reviewed `retract`,
1080
+ never by a percentage heuristic. The name, the signature and the callers
1081
+ (`cmd_five_hour_blocks`, `cmd_five_hour_breakdown` and the dashboard's
1082
+ block envelope) are unchanged; ``conn``, ``now_utc`` and ``cache`` are no
1083
+ longer read."""
1084
+ return (block.get("seven_day_pct_at_block_start"),
1085
+ block.get("seven_day_pct_at_block_end"))
1981
1086
 
1982
1087
 
1983
1088
  #: The six `five_hour_milestones` columns every surface already published, plus
1984
1089
  #: the joined EFFECTIVE weekly value. Ordering is `captured_at_utc, id` — spec
1985
1090
  #: §5.2's merged-stream order, which post-credit threshold repeats depend on.
1986
- #: The five `s.*` columns after the joined weekly value are INTERNAL. They exist
1987
- #: only so the credit retirement bands can be resolved against the JOINED ROW's
1988
- #: own account and week, and they are aliased because `captured_at_utc` exists on
1989
- #: both sides of the join. `_load_five_hour_milestones` builds its result dicts
1990
- #: key by key, so none of them can reach a published shape.
1991
1091
  _FIVE_HOUR_MILESTONE_SQL = """
1992
1092
  SELECT m.percent_threshold, m.captured_at_utc, m.block_cost_usd,
1993
1093
  m.marginal_cost_usd, m.seven_day_pct_at_crossing, m.reset_event_id,
1994
- s.weekly_percent AS effective_seven_day_pct_at_crossing,
1995
- s.captured_at_utc AS snapshot_captured_at_utc,
1996
- s.week_start_date AS snapshot_week_start_date,
1997
- s.week_start_at AS snapshot_week_start_at,
1998
- s.week_end_at AS snapshot_week_end_at,
1999
- s.account_key AS snapshot_account_key
1094
+ s.weekly_percent AS effective_seven_day_pct_at_crossing
2000
1095
  FROM five_hour_milestones m
2001
1096
  LEFT JOIN weekly_usage_snapshots s ON s.id = m.usage_snapshot_id
2002
1097
  WHERE {selector}
@@ -2036,34 +1131,12 @@ def _load_five_hour_milestones(
2036
1131
  put the number this change exists to stop showing back on the screen,
2037
1132
  silently and only in the failure case.
2038
1133
 
2039
- **The joined value is itself CREDIT-AWARE** (#834 S1 Gate A R2). A plain
2040
- `LEFT JOIN` is not enough: `maybe_record_milestone` sets `usage_snapshot_id`
2041
- to the row the tick just wrote, and the FIVE-HOUR milestone is not gated off
2042
- on a weekly-clamped tick — only the weekly one is — so in a credited week the
2043
- milestone can name a HELD row carrying a pre-credit weekly value the credit
2044
- retired. Publishing that as the "effective" value would render the exact
2045
- number #836 exists to remove, on all three human surfaces, on a new read.
2046
-
2047
- So the retirement bands are resolved against the JOINED ROW's own account and
2048
- week — never the reader's, because a credit under one account retires nothing
2049
- under another — memoized per `(week, account)` across the rows of one call, and
2050
- a retired joined value is published as None. The surfaces already render a None
2051
- as the unavailable marker, so no renderer changes, and the rule above still
2052
- holds: never a fallback to the raw value. (The memoization is this function's
2053
- own: it walks many rows spanning potentially several `(week, account)` pairs.
2054
- `_latest_seven_day_and_window` does NOT memoize, because its walk resolves the
2055
- scope, the bands and the floor once from the newest row and may not leave that
2056
- scope, so there is nothing to memoize per candidate. An earlier version of this
2057
- paragraph said it did.)
2058
-
2059
- THE BAND IS APPLIED HERE WITHOUT R1's FLOOR BOUND, deliberately. That bound
2060
- exists because a WALK can descend past every post-floor row onto a pre-floor
2061
- one and return the pre-credit reading the band exists to suppress. This read
2062
- performs no walk: each milestone names exactly ONE snapshot row through
2063
- `usage_snapshot_id`, and a pre-floor row's weekly value genuinely WAS the
2064
- effective percentage at that crossing instant. Blanking it would delete
2065
- historical truth rather than protect the reader, which is the same reason the
2066
- block's pre-floor weekly axes render.
1134
+ **The joined value is the snapshot's STORED value** (#840, spec section
1135
+ 4.8). #834 S1 Gate A R2 replaced a joined value within a point of one a
1136
+ credit retired with None, which also withheld genuine readings at that
1137
+ value; the retirement band is removed, and a value already poisoned in a
1138
+ store is corrected by `db rederive`'s reviewed `retract`. A missing joined
1139
+ snapshot still publishes None and never falls back to the raw value.
2067
1140
 
2068
1141
  SELECTOR. Exactly one of `block_id` or `five_hour_window_key` must be given.
2069
1142
  `block_id` is the precise form — `five_hour_milestones.block_id` names one
@@ -2094,29 +1167,10 @@ def _load_five_hour_milestones(
2094
1167
  rows = conn.execute(
2095
1168
  _FIVE_HOUR_MILESTONE_SQL.format(selector=selector), params,
2096
1169
  ).fetchall()
2097
- _c = _cctally()
2098
- bands_cache: dict = {}
2099
1170
  out: list[dict] = []
2100
1171
  for r in rows:
2101
1172
  eff = r["effective_seven_day_pct_at_crossing"]
2102
1173
  raw = r["seven_day_pct_at_crossing"]
2103
- if eff is not None:
2104
- band_key = (r["snapshot_week_start_date"],
2105
- r["snapshot_week_start_at"],
2106
- r["snapshot_week_end_at"],
2107
- r["snapshot_account_key"])
2108
- if band_key not in bands_cache:
2109
- bands_cache[band_key] = _c._credit_retirement_bands(
2110
- conn,
2111
- week_start_date=r["snapshot_week_start_date"],
2112
- week_start_at=r["snapshot_week_start_at"],
2113
- week_end_at=r["snapshot_week_end_at"],
2114
- account_key=r["snapshot_account_key"])
2115
- if _c._weekly_value_is_retired_replica(
2116
- bands_cache[band_key],
2117
- captured_at_utc=r["snapshot_captured_at_utc"],
2118
- weekly_percent=eff):
2119
- eff = None
2120
1174
  out.append({
2121
1175
  "percent_threshold": int(r["percent_threshold"]),
2122
1176
  "captured_at_utc": r["captured_at_utc"],
@@ -2818,16 +1872,24 @@ def _backfill_five_hour_blocks(
2818
1872
  AND snapshots.five_hour_percent IS NOT NULL
2819
1873
  """
2820
1874
  if only_missing:
1875
+ # #997 D19: each account's latest qualifying snapshot comes from
1876
+ # one ranking pass over the qualifying rows (O(n log n)). A
1877
+ # correlated subquery here re-scanned and re-sorted the table per
1878
+ # qualifying row (quadratic: 68.4 s on a 27,521-row store, past
1879
+ # the rebuild's progress-gap bound). The partition groups NULL
1880
+ # account keys as `IS` did; the ORDER BY is unchanged.
2821
1881
  keys_sql += (
2822
- " AND snapshots.id = ("
2823
- " SELECT latest.id"
2824
- " FROM weekly_usage_snapshots AS latest"
2825
- " WHERE latest.account_key IS snapshots.account_key"
2826
- " AND latest.five_hour_window_key IS NOT NULL"
2827
- " AND latest.five_hour_percent IS NOT NULL"
2828
- " ORDER BY unixepoch(latest.captured_at_utc) DESC,"
2829
- " latest.id DESC"
2830
- " LIMIT 1"
1882
+ " AND snapshots.id IN ("
1883
+ " SELECT id FROM ("
1884
+ " SELECT id, ROW_NUMBER() OVER ("
1885
+ " PARTITION BY account_key"
1886
+ " ORDER BY unixepoch(captured_at_utc) DESC,"
1887
+ " id DESC"
1888
+ " ) AS rn"
1889
+ " FROM weekly_usage_snapshots"
1890
+ " WHERE five_hour_window_key IS NOT NULL"
1891
+ " AND five_hour_percent IS NOT NULL"
1892
+ " ) WHERE rn = 1"
2831
1893
  " )"
2832
1894
  " AND NOT EXISTS ("
2833
1895
  " SELECT 1 FROM five_hour_blocks AS blocks"
@@ -2898,7 +1960,11 @@ def _backfill_five_hour_blocks(
2898
1960
  # See cctally-dev#87.
2899
1961
  conn.execute("BEGIN IMMEDIATE")
2900
1962
  try:
2901
- for key, acct in keys:
1963
+ for block_index, (key, acct) in enumerate(keys, 1):
1964
+ if not block_index & 1023:
1965
+ # #997 D9: a rebuild's open-block projection reports
1966
+ # progress (throttled; a no-op without a progress sink).
1967
+ _lib_rebuild_progress.tick(blocks=block_index)
2902
1968
  # The start row (the first writer, else the MIN-captured
2903
1969
  # survivor) defines the immutable block boundary values
2904
1970
  # (deterministic — picking "any in-window row" would be