cctally 1.114.0 → 1.116.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/CHANGELOG.md +41 -0
  2. package/bin/_cctally_account.py +32 -8
  3. package/bin/_cctally_cache.py +4491 -295
  4. package/bin/_cctally_completion.py +1355 -0
  5. package/bin/_cctally_config.py +37 -0
  6. package/bin/_cctally_core.py +282 -9
  7. package/bin/_cctally_dashboard.py +8 -11
  8. package/bin/_cctally_dashboard_conversation.py +292 -23
  9. package/bin/_cctally_dashboard_envelope.py +42 -35
  10. package/bin/_cctally_dashboard_sources.py +18 -9
  11. package/bin/_cctally_db.py +401 -9
  12. package/bin/_cctally_doctor.py +469 -27
  13. package/bin/_cctally_five_hour.py +166 -1049
  14. package/bin/_cctally_forecast.py +81 -9
  15. package/bin/_cctally_journal.py +5076 -177
  16. package/bin/_cctally_journal_repair.py +110 -19
  17. package/bin/_cctally_oauth_samples.py +280 -0
  18. package/bin/_cctally_parser.py +47 -0
  19. package/bin/_cctally_project.py +8 -2
  20. package/bin/_cctally_quarantine.py +662 -0
  21. package/bin/_cctally_quota.py +96 -12
  22. package/bin/_cctally_quota_model.py +391 -83
  23. package/bin/_cctally_record.py +1088 -269
  24. package/bin/_cctally_rederive.py +854 -192
  25. package/bin/_cctally_refresh.py +113 -13
  26. package/bin/_cctally_source_analytics.py +23 -2
  27. package/bin/_cctally_statusline.py +7062 -505
  28. package/bin/_cctally_store.py +1244 -91
  29. package/bin/_cctally_transcript.py +8 -7
  30. package/bin/_cctally_tui.py +119 -60
  31. package/bin/_cctally_update.py +25 -0
  32. package/bin/_lib_account_binding.py +152 -0
  33. package/bin/_lib_account_protocol.py +569 -0
  34. package/bin/_lib_aggregators.py +4 -0
  35. package/bin/_lib_cache_coverage.py +13 -1
  36. package/bin/_lib_cache_writer_lock.py +19 -3
  37. package/bin/_lib_clone_copy.py +177 -0
  38. package/bin/_lib_codex_conversation_query.py +133 -19
  39. package/bin/_lib_codex_conversation_watch.py +59 -14
  40. package/bin/_lib_codex_drain_admission.py +214 -0
  41. package/bin/_lib_codex_metadata.py +18 -1
  42. package/bin/_lib_codex_publication_barrier.py +277 -0
  43. package/bin/_lib_codex_rollout_container.py +502 -0
  44. package/bin/_lib_conversation_dispatch.py +3 -0
  45. package/bin/_lib_conversation_query.py +11 -3
  46. package/bin/_lib_dashboard_sources.py +57 -15
  47. package/bin/_lib_detector_access_registry.py +4038 -0
  48. package/bin/_lib_doctor.py +753 -1
  49. package/bin/_lib_five_hour.py +126 -0
  50. package/bin/_lib_ingest_frontier.py +414 -20
  51. package/bin/_lib_journal.py +261 -10
  52. package/bin/_lib_journal_router.py +166 -48
  53. package/bin/_lib_jsonl.py +5 -4
  54. package/bin/_lib_legacy_spool.py +126 -0
  55. package/bin/_lib_legacy_writer_evidence.py +136 -0
  56. package/bin/_lib_meter_rate_change.py +10 -0
  57. package/bin/_lib_oauth_samples.py +238 -0
  58. package/bin/_lib_outcome_quarantine.py +690 -0
  59. package/bin/_lib_pricing.py +7 -2
  60. package/bin/_lib_pricing_check.py +0 -12
  61. package/bin/_lib_provider_activity.py +770 -0
  62. package/bin/_lib_quarantine.py +155 -0
  63. package/bin/_lib_quota_calibration.py +53 -8
  64. package/bin/_lib_quota_copy.py +2 -1
  65. package/bin/_lib_quota_model.py +384 -52
  66. package/bin/_lib_rebuild_progress.py +146 -0
  67. package/bin/_lib_record.py +22 -0
  68. package/bin/_lib_rederive.py +393 -8
  69. package/bin/_lib_segment_summary.py +35 -2
  70. package/bin/_lib_snapshot_cache.py +43 -1
  71. package/bin/_lib_stale_proof.py +708 -0
  72. package/bin/_lib_statusline_candidates.py +356 -27
  73. package/bin/_lib_worker_launches.py +315 -0
  74. package/bin/cctally +29 -12
  75. package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
  76. package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
  77. package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
  78. package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
  79. package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
  80. package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
  81. package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
  82. package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
  83. package/dashboard/static/dashboard.html +1 -1
  84. package/package.json +20 -2
  85. package/bin/_lib_merge_gate.py +0 -725
  86. package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
@@ -38,13 +38,14 @@ Spec: docs/superpowers/specs/2026-05-30-extract-five-hour-statusline-cmd-design.
38
38
  from __future__ import annotations
39
39
 
40
40
  import argparse
41
- import collections
42
41
  import datetime as dt
43
42
  import json
44
43
  import sqlite3
45
44
  import sys
46
45
 
47
46
  import _lib_accounts # pure stdlib kernel; UNATTRIBUTED sentinel (#341)
47
+ import _lib_rebuild_progress # #997 D9 progress ticks (stdlib only)
48
+ from _lib_five_hour import latest_five_hour_reading
48
49
  from _cctally_core import _command_as_of, eprint, now_utc_iso, open_db, parse_iso_datetime
49
50
 
50
51
 
@@ -983,73 +984,41 @@ def _latest_seven_day_and_window(
983
984
  overlooked (#769 S11, #824). A `weekly_observation_held` row's five-hour
984
985
  fields are the writing tick's own, so it is the freshest statement of which
985
986
  window is current; excluding it would resolve an older window and report the
986
- live block closed. Rows with no ``five_hour_window_key`` are included here
987
- too, unchanged, because the latest row's key is the answer even when it is
988
- NULL.
989
-
990
- **The weekly value is CREDIT-AWARE**, which is the #835 change. It comes from
991
- the most recent row that is not a post-floor stale replica — the same band
992
- and floor predicate the stale-replica DELETE applies, applied as a read
993
- filter. Held-inclusive is still right in principle here, because a held row
994
- stores the EFFECTIVE weekly value and that is what a block's weekly end
995
- means. What is not right is reporting a value a credit RETIRED, and #835
996
- made that reachable by preserving held rows: before it, the DELETE removed
997
- every post-floor replica, so this read could not see one. Excluding held rows
998
- outright would be the wrong repair — it would report the block end from an
999
- older tick, which is the mistake #824's comment here warned against.
1000
-
1001
- **The weekly walk is BOUNDED three ways, and each bound closes a hole tranche
1002
- A left open (#834 S1 Gate A R1).**
1003
-
1004
- *Scope.* The walk descends only within the NEWEST row's own
1005
- ``(week_start_date, account_key)``. A candidate belonging to another week or
1006
- another account ends the walk with ``None``, because week W-1's percentage is
1007
- not week W's weekly end and account B's is not account A's, under any reading.
1008
- Before #835 this read took exactly one row, so the absent predicates could not
1009
- matter; a walk makes them matter.
1010
-
1011
- *Floor.* The walk also ends with ``None`` at the first row captured BEFORE the
1012
- latest credit floor in that scope. This is the soundness bound: a pre-floor row
1013
- is never classified as retired, so without it the walk terminated on one and
1014
- returned its value — and that value is by construction the PRE-credit reading,
1015
- which is exactly the value a retirement band exists to suppress. A sub-one-point
1016
- credit reaches it immediately, because such a credit is legal
1017
- (`_doomed_snapshot_rows`) and places the post-credit synthetic inside a band
1018
- centred on ``from_pct``, leaving every post-floor row in the band. An
1019
- unparseable capture stamp inside a credited week is treated as pre-floor for
1020
- the same reason: the row cannot be placed relative to the floor, so it cannot
1021
- be certified as able to state the current value.
1022
-
1023
- *Retirement.* Within the scope and at or after the floor, a retired replica is
1024
- skipped, and a retired value is therefore never returned.
1025
-
1026
- The floor bound applies to the NEWEST row as well as to the candidates below
1027
- it, so a newest row captured before the latest floor answers ``None`` where the
1028
- pre-#835 code answered its value. That state is unreachable through either
1029
- production credit path: `record-credit` writes a post-credit synthetic snapshot
1030
- at its own effective instant, and the automatic path's floor is derived from a
1031
- capture that is itself in the store, so in both cases at least one row exists at
1032
- or after the floor and it is the newest.
1033
-
1034
- An UNCREDITED week resolves no bands, so there is no floor, nothing is skipped,
1035
- and the read returns the newest row's value exactly as it did before #835. A
1036
- ``NULL`` weekly value is not a retired replica, so it still terminates the walk
1037
- and still answers ``None``. Both contracts are byte-stable against today.
1038
-
1039
- **The residual, deliberately unfixed and filed as #840.** A genuine later climb
1040
- back INTO the band is still classified as a replay, so the weekly end reads the
1041
- newest out-of-band value until the true value passes the band's upper edge.
1042
- That is preferred over returning a value a credit retired, and no sound
1043
- discriminator is available: `record-credit`'s own synthetic post-credit
1044
- snapshot is written before any later replay, so "a below-band row has been
1045
- observed" would close the band immediately and let every replay through.
1046
- `test_840_characterization_a_genuine_climb_back_reads_the_older_value`
1047
- characterizes it.
987
+ live block closed. It is the key of the newest row that CARRIES a five-hour
988
+ reading (#825/#832 spec 4.7 revision 7, ``latest_five_hour_reading``): a
989
+ retired replica or a weekly-only reading can be the newest row, and its NULL
990
+ key would report the live block closed and leave its weekly end stale.
991
+
992
+ **The weekly value is the newest row's STORED value** (#840, spec section
993
+ 4.8). Held-inclusive, because a held row stores the effective weekly value
994
+ and that is what a block's weekly end means. #835 filtered it through a
995
+ retirement band (a value within a point of one a credit retired was skipped
996
+ as a replay), which also withheld every genuine climb back to that value;
997
+ the band is removed. Stale contributors are refused before publication
998
+ (spec section 4.4), and a value already poisoned in a store is corrected by
999
+ `db rederive`'s reviewed `retract`, never by a percentage heuristic.
1000
+
1001
+ *Scope.* The value is the NEWEST row's, within its own
1002
+ ``(week_start_date, account_key)``; a read never reports another week's or
1003
+ another account's percentage.
1004
+
1005
+ *Floor.* The read answers ``None`` when that row was captured BEFORE the
1006
+ latest credit floor in its scope, or carries an unparseable capture stamp
1007
+ inside a credited week: such a row cannot state the week's current
1008
+ effective weekly value, and its value is by construction the pre-credit
1009
+ reading. That state is unreachable through either production credit path:
1010
+ `record-credit` writes a post-credit synthetic snapshot at its own effective
1011
+ instant, and the automatic path's floor is derived from a capture that is
1012
+ itself in the store, so in both cases at least one row exists at or after
1013
+ the floor and it is the newest.
1014
+
1015
+ An UNCREDITED week resolves no floor, and the read returns the newest row's
1016
+ value. A ``NULL`` weekly value answers ``None``.
1048
1017
 
1049
1018
  ``account_key`` (#341, spec §3): ``None`` = merged / byte-stable; a real key /
1050
- ``unattributed`` scopes both selections to that account. The retirement bands
1051
- are resolved against the NEWEST ROW's own account, which is also the walk's
1052
- scope, because a credit under one account retires nothing under another.
1019
+ ``unattributed`` scopes both selections to that account. The credit floor is
1020
+ resolved against the NEWEST ROW's own account and week, because a credit
1021
+ under one account bounds nothing under another.
1053
1022
  """
1054
1023
  _c = _cctally()
1055
1024
  acct_pred = "" if account_key is None else " WHERE account_key = ?"
@@ -1065,45 +1034,32 @@ def _latest_seven_day_and_window(
1065
1034
  row = cur.fetchone()
1066
1035
  if row is None:
1067
1036
  return None, None
1068
- # Axis 1: the five-hour window key, from the latest row, held-inclusive.
1069
- key = row["five_hour_window_key"]
1070
- # Axis 2: the weekly value. The scope, the bands and the floor all come
1071
- # from the NEWEST row and are resolved ONCE, because the walk may not
1072
- # leave that scope — so there is nothing to memoize per candidate.
1073
- #
1074
- # Dropping `LIMIT 1` does cost something at the storage layer: SQLite
1075
- # materializes the full sort before yielding the first row, measured at
1076
- # 9.0ms against 3.4ms on a 25,053-row store. The cost is immaterial
1077
- # because this runs once per invocation, but the earlier claim that the
1078
- # ordinary case "reads exactly one row, which is the work the single-row
1079
- # query did" described only the rows the cursor yields, not the work the
1080
- # query does, and was wrong.
1081
- scope = (row["week_start_date"], row["account_key"])
1037
+ # Axis 1: the five-hour window key, held-inclusive, from the newest
1038
+ # five-hour READING of the newest row's own account (#825/#832 spec
1039
+ # 4.7 revision 7, the one shared selection). The newest row can be a
1040
+ # retired replica or a weekly-only reading, which names no window.
1041
+ # Merged mode scopes the weekly walk to that account too, so an
1042
+ # account-blind selection here would pair another account's window
1043
+ # with this account's weekly value.
1044
+ reading = latest_five_hour_reading(conn, account_key=row["account_key"])
1045
+ key = None if reading is None else reading.window_key
1046
+ # Axis 2: the weekly value, the newest row's stored value, bounded by
1047
+ # the latest credit floor of its own week and account (#840).
1082
1048
  bands = _c._credit_retirement_bands(
1083
1049
  conn, week_start_date=row["week_start_date"],
1084
1050
  week_start_at=row["week_start_at"],
1085
1051
  week_end_at=row["week_end_at"],
1086
1052
  account_key=row["account_key"])
1087
1053
  floor = _c._latest_credit_floor_instant(bands)
1088
- pct = None
1089
- while row is not None:
1090
- if (row["week_start_date"], row["account_key"]) != scope:
1091
- break
1092
- if floor is not None:
1093
- try:
1094
- captured = parse_iso_datetime(
1095
- row["captured_at_utc"], "captured_at").timestamp()
1096
- except (ValueError, TypeError):
1097
- captured = None
1098
- if captured is None or captured < floor:
1099
- break
1100
- if not _c._weekly_value_is_retired_replica(
1101
- bands,
1102
- captured_at_utc=row["captured_at_utc"],
1103
- weekly_percent=row["weekly_percent"]):
1104
- pct = row["weekly_percent"]
1105
- break
1106
- row = cur.fetchone()
1054
+ pct = row["weekly_percent"]
1055
+ if floor is not None:
1056
+ try:
1057
+ captured = parse_iso_datetime(
1058
+ row["captured_at_utc"], "captured_at").timestamp()
1059
+ except (ValueError, TypeError):
1060
+ captured = None
1061
+ if captured is None or captured < floor:
1062
+ pct = None
1107
1063
  except sqlite3.DatabaseError:
1108
1064
  return None, None
1109
1065
  return (
@@ -1112,882 +1068,30 @@ def _latest_seven_day_and_window(
1112
1068
  )
1113
1069
 
1114
1070
 
1115
- def _parsed_block_instant(block, column):
1116
- """``block[column]`` when it is present and parses as an instant, else ``None``.
1117
-
1118
- #834 S1 (#835) Gate A R8. Both observation stamps are declared `TEXT NOT NULL`,
1119
- so absence is not a live shape — but a fixture, a hand-built store or a legacy
1120
- projection can still carry an unparseable one, and the two instant helpers
1121
- below must degrade rather than raise.
1122
- """
1123
- value = block.get(column)
1124
- if not value:
1125
- return None
1126
- try:
1127
- parse_iso_datetime(value, column)
1128
- except (ValueError, TypeError):
1129
- return None
1130
- return value
1131
-
1132
-
1133
- def _block_weekly_start_instant(block):
1134
- """The instant a block's stored weekly START was captured at, as ISO text.
1135
-
1136
- #834 S1 (#835) Gate A R8. `first_observed_at_utc`, because that is the stamp
1137
- of the tick that wrote `seven_day_pct_at_block_start`. The live upsert in
1138
- bin/_cctally_record.py sets both from one fold dict on the INSERT (both are
1139
- omitted from its `DO UPDATE`, so the first writer owns the pair), and
1140
- `_backfill_five_hour_blocks` takes `first_obs` and `pct_start_7d` from its
1141
- single MIN-captured snapshot row.
1142
-
1143
- NOT `block_start_at`, which is the window's NOMINAL start — `five_hour_resets_at`
1144
- minus five hours — and therefore at or before the first tick. The gap is not
1145
- cosmetic: a block whose nominal start precedes a credit floor but whose first
1146
- tick landed after it would have the floor's retired value compared against a
1147
- PRE-floor instant and rendered. It falls back to `block_start_at` only when the
1148
- observation stamp is absent or unparseable, which keeps the pre-R8 disposition
1149
- rather than blanking the axis.
1150
- """
1151
- return (_parsed_block_instant(block, "first_observed_at_utc")
1152
- or block.get("block_start_at"))
1153
-
1154
-
1155
- def _block_weekly_end_instant(block, *, now_utc):
1156
- """The instant a block's stored weekly END was captured at, as an ISO text.
1157
-
1158
- #834 S1 (#835) Gate A R8. `last_observed_at_utc`, because that is the stamp of
1159
- the tick that wrote `seven_day_pct_at_block_end`. All four writers of the pair
1160
- move the two together: the live upsert assigns
1161
- `last_observed_at_utc = excluded.last_observed_at_utc` and
1162
- `seven_day_pct_at_block_end = excluded.seven_day_pct_at_block_end` in one
1163
- `DO UPDATE SET` from one fold dict, `_backfill_five_hour_blocks` takes
1164
- `last_obs` and `pct_end_7d` from its single MAX-captured snapshot row,
1165
- `five_hour_block_close` freezes the whole row into one evt, and
1166
- `_migration_merge_5h_block_duplicates_v1` (bin/_cctally_db.py) copies both
1167
- from the group row whose `last_observed_at_utc` is MAX.
1168
-
1169
- Gate A S1 used `five_hour_resets_at` instead, falling back to the current
1170
- instant while that reset was still in the future. That is the block's END, not
1171
- a capture instant, and it was wrong in BOTH directions. A crossed-reset block's
1172
- reset falls in the SUCCESSOR week, so `_block_weekly_axis_weeks` resolved the
1173
- successor's bounds and the credited week's bands were never consulted. A closed
1174
- block whose last tick predates the floor had its genuinely effective end
1175
- withheld, which contradicts the historical-truth rule this filter rests on.
1176
-
1177
- The reset-or-now form is KEPT as the fallback for an absent or unparseable
1178
- stamp, so a store that cannot supply a capture instant behaves as it did
1179
- before R8 rather than losing the axis.
1180
- """
1181
- _c = _cctally()
1182
- observed = _parsed_block_instant(block, "last_observed_at_utc")
1183
- if observed is not None:
1184
- return observed
1185
- now_iso = _c._iso_z(now_utc)
1186
- resets = block.get("five_hour_resets_at")
1187
- if not resets:
1188
- return now_iso
1189
- try:
1190
- moment = parse_iso_datetime(resets, "five_hour_resets_at").timestamp()
1191
- except (ValueError, TypeError):
1192
- return now_iso
1193
- return resets if moment <= now_utc.timestamp() else now_iso
1194
-
1195
-
1196
- #: One candidate week for `_block_weekly_axis_weeks`, as loaded once per account
1197
- #: per render. ``lo`` / ``hi`` are the parsed `[week_start_at, week_end_at)`
1198
- #: epochs, or ``None`` when the week carries no usable bounds.
1199
- _AxisWeek = collections.namedtuple(
1200
- "_AxisWeek",
1201
- "week_start_date week_end_date week_start_at week_end_at lo hi")
1202
-
1203
-
1204
- def _parsed_week_bounds(week_start_at, week_end_at):
1205
- """``(lo, hi)`` epochs for a week's bounds, or ``(None, None)``.
1206
-
1207
- #834 S1 (#835) Gate A R9. Both loaders need the same verdict on the same two
1208
- nullable columns — `_account_axis_weeks` to decide whether a week is placed by
1209
- epoch or by date, `_account_axis_capture_weeks` to decide whether the identity
1210
- it returns carries bounds for the `week_reset_events` leg — so the parse lives
1211
- in one place and the two cannot disagree about which weeks have usable bounds.
1212
- Absent or unparseable on either side yields no bounds, which is what
1213
- `_credit_retirement_bands` already treats as a leg it cannot scope.
1214
- """
1215
- if week_start_at is None or week_end_at is None:
1216
- return None, None
1217
- try:
1218
- return (parse_iso_datetime(week_start_at, "week_start_at").timestamp(),
1219
- parse_iso_datetime(week_end_at, "week_end_at").timestamp())
1220
- except (ValueError, TypeError):
1221
- return None, None
1222
-
1223
-
1224
- def _account_axis_weeks(conn, *, account_key, cache):
1225
- """Every distinct week ``account_key``'s snapshot rows name, in no order.
1226
-
1227
- #834 S1 (#835) Gate A R8. ONE query per account per render, replacing one query
1228
- per axis instant. The per-instant form was measured at 22x the whole render's
1229
- prior cost on a 25,000-row store with 50 blocks, because its
1230
- ``unixepoch(week_start_at)`` predicate makes `idx_usage_week_start_at_time`
1231
- unusable — `EXPLAIN QUERY PLAN` reported `SCAN weekly_usage_snapshots` plus
1232
- `USE TEMP B-TREE FOR ORDER BY`, and the 50-block render performed about 51 full
1233
- scans and 51 full sorts. An instant-keyed memo could not collapse them: two
1234
- instants per block are two distinct keys.
1235
-
1236
- THE WIN IS THE COLLAPSE TO ONE QUERY, NOT RESTORED INDEX USE. This query is
1237
- equally unindexed: `EXPLAIN QUERY PLAN` reports `SCAN weekly_usage_snapshots`
1238
- plus a temp B-tree. About 51 unindexed scans became one unindexed scan, which is
1239
- the whole of the improvement; do not read `idx_usage_week_start_at_time` as being
1240
- back in play. The scan verdict was measured on an empty store, on a 25,500-row
1241
- store and again after `ANALYZE` while this query was still a `GROUP BY`; R10
1242
- replaced that clause with `SELECT DISTINCT`, which changed the B-tree's stated
1243
- purpose and not the scan (see the R10 paragraph below for the measured plan).
1244
-
1245
- THE RESIDUAL IS NOT IRREDUCIBLE, and it is accepted as shipped rather than
1246
- defended as minimal. `idx_usage_week_time` leads on `week_start_date`, so
1247
- narrowing the projection to that column alone does use it — `SELECT DISTINCT
1248
- week_start_date` reports `SCAN weekly_usage_snapshots USING INDEX
1249
- idx_usage_week_time`, measured on the same two stores, and an earlier form said
1250
- "grouping on that column alone", which is stale terminology since R10 replaced
1251
- the `GROUP BY` with `SELECT DISTINCT` — and a render only
1252
- needs the weeks overlapping its blocks' instant range rather than the account's
1253
- whole history. Both are real reductions and neither is taken, because the
1254
- property that matters is that the cost is CONSTANT in the number of rendered
1255
- blocks, which it already is. `week_end_date` and the two bounds are in the
1256
- projection because the date pass needs all four, which is exactly what a
1257
- `week_start_date`-only projection would give up.
1258
-
1259
- ONE COMPARISON SEMANTIC CHANGED WITH THE FORM, unreachably. The per-instant
1260
- predicate compared SQLite `unixepoch()` integers, which truncate a fractional
1261
- second; this form compares `parse_iso_datetime(...).timestamp()` floats, which
1262
- keep it. The two disagree for a week bound carrying a fractional second: a
1263
- `week_start_at` of `…T00:00:00.500000+00:00` against a probe at
1264
- `…T00:00:00+00:00` is INSIDE the week under the integer compare and outside it
1265
- under the float one. No writer can produce that bound.
1266
- `_canonicalize_optional_iso` returns `isoformat(timespec="seconds")` over a
1267
- `_normalize_week_boundary_dt` result, which zeroes second and microsecond and
1268
- snaps to the hour, and `cmd_record_usage`'s pipeline derives both bounds from
1269
- that same normalizer and writes them with the same `timespec="seconds"`.
1270
-
1271
- THE SET IS UNORDERED, AND THE QUERY IS A PLAIN `SELECT DISTINCT` (Gate A R10).
1272
- Through R9 it carried a `MAX(captured_at_utc || char(1) || printf('%020d', id))`
1273
- aggregate whose SQL alias was `rk` and whose `_AxisWeek` field was `rank` — call
1274
- it `rk`, because that is the alias, and earlier text here called the column
1275
- `rank` — introduced to reproduce the per-instant query's
1276
- ``ORDER BY captured_at_utc DESC, id DESC`` tie-break exactly so that which week
1277
- won an overlap did not change with the query's form, and kept as documentary
1278
- after R9 deleted the tie-break. Nothing read it: this function has one caller,
1279
- every pass in `_block_weekly_axis_weeks` builds an order-insensitive set, and
1280
- the caller ORs a predicate over the result. The grouping was there only to carry
1281
- the aggregate, so both went.
1282
-
1283
- NOTHING PINS THE ORDER NOW. There was never a SQL `ORDER BY` on this query: the
1284
- order came from a Python `weeks.sort(key=lambda w: w.rank, reverse=True)` over the
1285
- aggregate. Through R8 a consumer tie-break in the epoch pass read it; R9 deleted
1286
- that tie-break and left the aggregate documentary; R10 deleted the aggregate and
1287
- the sort together. So the returned order is now whatever SQLite produces, and no
1288
- test asserts order-insensitivity of the consumer, so a future consumer that cares
1289
- about order has nothing to fail against. It is safe today because the one consumer
1290
- ORs a monotone band predicate over the whole set. A consumer that needs an order
1291
- must establish it itself rather than assume this one.
1292
-
1293
- The statement-count test cannot see R10's REMOVAL of the aggregate and the
1294
- grouping, because the statement count is unchanged by it; the removal was
1295
- measured instead, best of five on a 25,194-row 13-week 50-block store, with the
1296
- order of the three forms reversed to rule out page-cache bias and the three
1297
- result sets asserted identical: 13.57 ms with the aggregate, 9.36 ms for the
1298
- bare `GROUP BY`, 2.34 ms for `SELECT DISTINCT` — so removing both took about 83%
1299
- off the statement, and the plan changed from `SCAN weekly_usage_snapshots` +
1300
- `USE TEMP B-TREE FOR GROUP BY` to the same scan + `USE TEMP B-TREE FOR DISTINCT`.
1301
- It is still an unindexed scan; the `idx_usage_week_time` reduction named above is
1302
- still not taken, for the reason given there.
1303
-
1304
- DO NOT QUOTE A SPLIT BETWEEN THE TWO REMOVALS. On those figures the aggregate is
1305
- 31% of the statement and the grouping a further 52%, but the review that checked
1306
- this measured 12.87-13.09 / 6.30-6.77 / 2.02-2.23 ms on its own 25,194-row store,
1307
- which makes the aggregate about 50% and the grouping about 34% — the same 83%
1308
- total, with the two components in the opposite order. The total reproduces and
1309
- the ranking does not, so treat the split as machine- and data-dependent.
1310
-
1311
- `week_start_at` / `week_end_at` are both nullable and
1312
- `_derive_week_from_payload` leaves them `None` on three of its four paths, so
1313
- a week with no bounds is a live shape; it is retained here with ``lo``/``hi``
1314
- of ``None`` and placed by date instead (see `_block_weekly_axis_weeks`).
1315
- """
1316
- if account_key in cache:
1317
- return cache[account_key]
1318
- try:
1319
- rows = conn.execute(
1320
- "SELECT DISTINCT week_start_date, week_end_date, week_start_at, "
1321
- " week_end_at "
1322
- " FROM weekly_usage_snapshots "
1323
- " WHERE account_key = ?",
1324
- (account_key,),
1325
- ).fetchall()
1326
- except sqlite3.DatabaseError:
1327
- rows = []
1328
- weeks: list = []
1329
- for r in rows:
1330
- lo, hi = _parsed_week_bounds(r["week_start_at"], r["week_end_at"])
1331
- weeks.append(_AxisWeek(
1332
- week_start_date=r["week_start_date"],
1333
- week_end_date=r["week_end_date"],
1334
- week_start_at=r["week_start_at"] if lo is not None else None,
1335
- week_end_at=r["week_end_at"] if hi is not None else None,
1336
- lo=lo, hi=hi))
1337
- cache[account_key] = weeks
1338
- return weeks
1339
-
1340
-
1341
- def _account_axis_capture_weeks(conn, *, account_key, cache):
1342
- """``{capture second -> the week identities a snapshot row at that instant
1343
- names}``, for the capture instants ``account_key``'s blocks actually carry.
1344
-
1345
- #834 S1 (#835) Gate A R9. THE STORE KEEPS A RECORD OF WHICH WEEK A CAPTURE WAS
1346
- OBSERVED UNDER, and this is the read of it. Round five documented its
1347
- over-withholding on the premise that it does not. All four writers of
1348
- `five_hour_blocks.seven_day_pct_at_block_start` / `_at_block_end` take the
1349
- matching axis STAMP from a `weekly_usage_snapshots` row's `captured_at_utc`:
1350
- the live upsert passes `captured_at = saved["capturedAt"]` into
1351
- `first_observed_at_utc` and `last_observed_at_utc` in one `INSERT … DO UPDATE`
1352
- and that value IS the row's `captured_at_utc` (bin/_cctally_record.py:2351,
1353
- :2633-2634, :6828), `_backfill_five_hour_blocks` takes them from its MIN- and
1354
- MAX-captured rows, `five_hour_block_close` freezes the row, and
1355
- `_migration_merge_5h_block_duplicates_v1` copies from the MAX-captured group
1356
- row. Every snapshot row carries all four week columns. So a row whose
1357
- `captured_at_utc` equals an axis instant NAMES the week in force at that
1358
- capture, with no date-range guess involved.
1359
-
1360
- HELD ROWS ARE NOT EXCLUDED, which is the one judgement call in this function.
1361
- The rest of this tranche excludes `weekly_observation_held = 1` from weekly
1362
- VALUE reads, because a held row carries a reading no fresh observation
1363
- confirmed. This function reads no value: it reads which week the value on the
1364
- block's axis came from, and on a held tick that is the BASIS week by
1365
- construction — the held write copies the basis's four week columns verbatim
1366
- while keeping the tick's own `capture_at` (bin/_cctally_record.py:7190-7203),
1367
- and the block's axis takes `fold.weekly.effective_pct`, the same
1368
- carried-forward value, stamped with that same `capture_at`. The held row is
1369
- therefore the provenance record of the number on the axis, and the credits
1370
- that retired that number are the basis week's.
1371
-
1372
- Two consequences follow. A held row is exactly how a pre-credit weekly value
1373
- reaches a post-credit capture instant, which is the case this whole tranche
1374
- exists to handle, and excluding it would drop the only exact evidence of which
1375
- week's floor applies. And because no predicate names
1376
- `weekly_observation_held`, a store that stops short of epoch 1015 is served
1377
- unchanged and `weekly_held_exclusion` is not needed here. Pinned by
1378
- `tests/test_five_hour_breakdown.py::test_835_r9_a_held_capture_record_names_the_basis_week_it_carried`
1379
- and `::test_835_r9_a_pre_column_store_resolves_the_capture_record`.
1380
-
1381
- THE SET IS BOUNDED BY THE ACCOUNT'S BLOCK COUNT, NOT BY ITS SNAPSHOT HISTORY,
1382
- and the form was chosen on measurement rather than on shape. Re-measured at Gate
1383
- A R10, because R10 replaced `_account_axis_weeks`' grouped query with a plain
1384
- `SELECT DISTINCT` and the earlier round's figures for that statement no longer
1385
- described what ships. All three were timed in ONE session on a 25,194-row,
1386
- 13-week, 50-block store — the store `_account_axis_weeks` documents its own
1387
- numbers against — best of five each, with the order reversed to rule out
1388
- page-cache bias:
1389
-
1390
- - this narrowed query: 2.10-2.27 ms, 50 rows, `SCAN weekly_usage_snapshots`
1391
- plus a `LIST SUBQUERY` that SEARCHes `five_hour_blocks` on its own index;
1392
- - `_account_axis_weeks`' `SELECT DISTINCT`: 2.06-2.13 ms, 13 rows;
1393
- - one ungrouped full-row scan serving BOTH passes from a single statement:
1394
- 14.18-14.38 ms, 25,194 rows, a bare `SCAN weekly_usage_snapshots`.
1395
-
1396
- So the two statements together cost 4.40 ms against the single statement's
1397
- 14.18 ms and retain 63 rows against 25,194 — a 400th. Two statements per account
1398
- per render is the cheaper form on both axes, and it keeps the property that
1399
- matters: the cost is CONSTANT in the number of rendered blocks. Bounded by the
1400
- account's own `five_hour_blocks` rows rather than by which of them this render
1401
- shows, which is why no instant range has to be threaded down from the caller.
1402
-
1403
- TWO EARLIER ROUNDS DISAGREED ABOUT THE SINGLE-STATEMENT FIGURE — this docstring
1404
- said 22.03 ms and `docs/five-hour-gotchas.md` said 21.97 ms for the same query on
1405
- the same store — and neither number survives, because both were taken against the
1406
- superseded grouped form of the week query. The figures above replace both rather
1407
- than reconciling them.
1408
-
1409
- THE NARROWING AND THE MATCH USE DIFFERENT READERS, deliberately. SQLite's
1410
- `unixepoch()` selects the rows; `parse_iso_datetime(...).timestamp()` keys
1411
- them and keys the probe, so the match itself is Python's and is consistent
1412
- between map and probe. Both readers truncate to the second, which is how two
1413
- ISO spellings of one moment — `Z` against `+00:00` — compare equal.
1414
-
1415
- A NAIVE STAMP IS A LIVE SHAPE, and the two readings of it differ. An earlier
1416
- form of this paragraph said no writer produces one, citing `now_utc_iso()`.
1417
- That is wrong: `_coerce_payload_captured_at` (bin/_cctally_record.py:6576-6585)
1418
- returns the PAYLOAD's `capturedAt` verbatim whenever `parse_iso_datetime`
1419
- accepts it, `parse_iso_datetime` accepts a naive string and reads it host-local,
1420
- and `insert_usage_snapshot` then stores that text. Driven with
1421
- `TZ=America/Los_Angeles`: a payload `capturedAt` of `2026-04-30T13:00:00` is
1422
- stored as `captured_at_utc = '2026-04-30T13:00:00'`, and `unixepoch()` reads it
1423
- 1777554000 against Python's 1777579200.
1424
-
1425
- THE FAILURE IS A MIXED PAIR, NOT A NAIVE STAMP, AND IT IS THE MATCH THAT MISSES
1426
- RATHER THAN THE NARROWING THAT EXCLUDES. The same earlier paragraph stated it
1427
- the other way round. Each reader is self-consistent across the two sides it
1428
- compares, so the spelling only matters when the snapshot stamp and the block
1429
- stamp are spelled DIFFERENTLY. All four pairings, driven on that same non-UTC
1430
- host: equal naive text narrows IN and the Python match HITS; `Z` against
1431
- `+00:00` narrows IN and HITS; naive against offset-carrying, in either
1432
- direction, narrows IN and the PYTHON MATCH MISSES, after which the instant falls
1433
- through to the two fallback passes.
1434
-
1435
- SO THE RESIDUAL IS NARROWER THAN A NAIVE STAMP'S EXISTENCE. Every one of the
1436
- four axis writers takes the block stamp from a `weekly_usage_snapshots` row's
1437
- own `captured_at_utc` text — the live upsert from `saved["capturedAt"]`, the
1438
- backfill from its MIN- and MAX-captured rows, the close from the frozen row, the
1439
- merge migration from the MAX-captured group row — so the pair the capture pass
1440
- compares always shares a spelling and the pass is correct on a naive store. A
1441
- mixed pair needs the two texts to diverge after both were written, which is a
1442
- hand-built or externally-edited store. The degradation is then a fallback rather
1443
- than a misclassification, which is the one part of the earlier paragraph that
1444
- held.
1445
- """
1446
- if account_key in cache:
1447
- return cache[account_key]
1448
- try:
1449
- rows = conn.execute(
1450
- "SELECT captured_at_utc, week_start_date, week_start_at, week_end_at "
1451
- " FROM weekly_usage_snapshots "
1452
- " WHERE account_key = ? "
1453
- " AND unixepoch(captured_at_utc) IN ("
1454
- " SELECT unixepoch(first_observed_at_utc) "
1455
- " FROM five_hour_blocks WHERE account_key = ? "
1456
- " UNION "
1457
- " SELECT unixepoch(last_observed_at_utc) "
1458
- " FROM five_hour_blocks WHERE account_key = ?)",
1459
- (account_key, account_key, account_key),
1460
- ).fetchall()
1461
- except sqlite3.DatabaseError:
1462
- rows = []
1463
- carried: dict = {}
1464
- for r in rows:
1465
- try:
1466
- second = int(parse_iso_datetime(
1467
- r["captured_at_utc"], "captured_at_utc").timestamp())
1468
- except (ValueError, TypeError):
1469
- continue
1470
- lo, _hi = _parsed_week_bounds(r["week_start_at"], r["week_end_at"])
1471
- identity = (
1472
- r["week_start_date"],
1473
- r["week_start_at"] if lo is not None else None,
1474
- r["week_end_at"] if lo is not None else None,
1475
- )
1476
- # A dict as an ordered set: two rows of one week at one instant are one
1477
- # identity, and `_credit_retirement_bands` would answer both alike.
1478
- carried.setdefault(second, {})[identity] = None
1479
- out = {second: tuple(ids) for second, ids in carried.items()}
1480
- cache[account_key] = out
1481
- return out
1482
-
1483
-
1484
- def _account_axis_bounds_by_start_date(weeks):
1485
- """``{week_start_date -> that week's bounds-carrying identities}``, from an
1486
- already-loaded `_account_axis_weeks` set.
1487
-
1488
- #834 S1 (#835) Gate A R10. The index the capture pass substitutes from when its
1489
- matched row carries no bounds of its own. Derived from the account's OWN week
1490
- set and nothing else, which is the property that separates it from
1491
- `_get_canonical_boundary_for_date`; built once per account per render, so the
1492
- substitution adds no query and no per-instant scan.
1493
-
1494
- Bounds-free weeks are absent by construction: an entry exists only for a
1495
- `week_start_date` some row of that account spells bounds for, which is exactly
1496
- the condition under which a substitution is possible.
1497
- """
1498
- out: dict = {}
1499
- for week in weeks:
1500
- if week.lo is None:
1501
- continue
1502
- out.setdefault(week.week_start_date, {})[
1503
- (week.week_start_date, week.week_start_at, week.week_end_at)] = None
1504
- return {date: tuple(ids) for date, ids in out.items()}
1505
-
1506
-
1507
- def _block_weekly_axis_weeks(conn, *, account_key, instant_iso, cache):
1508
- """Every week identity whose credits can have retired a value captured at
1509
- ``instant_iso`` under ``account_key``, as ``(week_start_date, week_start_at,
1510
- week_end_at)`` triples. Empty when no week names the instant.
1511
-
1512
- THREE PASSES, IN THIS ORDER: the CAPTURE pass matches the instant against the
1513
- `captured_at_utc` of the account's own snapshot rows and answers from the week
1514
- those rows name; the EPOCH pass places the instant inside a week's parsed
1515
- `[week_start_at, week_end_at)` bounds; the DATE pass asks which bounds-free
1516
- weeks declare the instant's calendar day. Each runs only when the one before it
1517
- resolved nothing.
1518
-
1519
- THE CAPTURE PASS IS EXACT ABOUT WHICH WEEK, AND IT REPLACED AN ARGUMENT THAT
1520
- WAS WRONG. Gate A R8 had only the other two, and round five defended the
1521
- over-withholding they cause on the premise that the store keeps no record of
1522
- which week a capture was observed under. It keeps exactly that record: every
1523
- axis stamp on a block IS some snapshot row's `captured_at_utc`, and that row
1524
- carries all four week columns — see `_account_axis_capture_weeks`, which also
1525
- records why held rows are consulted rather than excluded. So for an instant a
1526
- capture record carries there is no week to guess, and neither fallback runs.
1527
-
1528
- BEING EXACT ABOUT THE WEEK IS NOT THE SAME AS RESOLVING BOTH LEGS, which an
1529
- earlier form of this paragraph left unsaid while the date pass's own paragraphs
1530
- below state it carefully. `week_start_at` and `week_end_at` are nullable on the
1531
- matched row too, and `_credit_retirement_bands`' `week_reset_events` leg is
1532
- scoped purely by containment of `effective_reset_at_utc` in
1533
- ``[week_start_at, week_end_at)`` — that table has no `week_start_date` column at
1534
- all — so an identity carrying no bounds resolves the `weekly_credit_floors` leg
1535
- and NOT the automatic one. Through R9 that degradation combined with this pass's
1536
- precedence into the tranche's own defect class: a bounds-free capture row
1537
- pre-empted a bounds-carrying placement of the SAME week and the value an
1538
- automatic credit retired was published. R10 closes it by SUBSTITUTING the
1539
- account's own bounds (next paragraph), so the per-leg behaviour now is that both
1540
- legs resolve whenever any row of the named week carries parseable bounds, and
1541
- the manual leg alone when none does.
1542
-
1543
- A BOUNDS-FREE CAPTURE IDENTITY IS REPLACED BY THE ACCOUNT'S OWN BOUNDS-CARRYING
1544
- ROWS OF THE SAME `week_start_date`, and the substitution is lossless. Those rows
1545
- are already in memory from `_account_axis_weeks`, so no query is added. It is
1546
- lossless because every substituted identity shares the bounds-free one's
1547
- `week_start_date`, so each resolves a `weekly_credit_floors` leg identical to the
1548
- one it replaces and ADDS the automatic leg its bounds can scope; the replaced
1549
- identity therefore contributes no band the substitutes do not. Several of them
1550
- are unioned, the way the epoch pass unions overlapping candidates, and that
1551
- union is here FOR ROBUSTNESS rather than for a reachable shape: no writer is
1552
- known to put two bound pairs under one `week_start_date`. What pins one pair per
1553
- date is `_usage_snapshot_columns` (bin/_cctally_record.py:6698-6710), which
1554
- overrides every later row's derived bounds with `_get_canonical_boundary_for_date`
1555
- — the FIRST established pair for that date, `ORDER BY captured_at_utc ASC, id ASC
1556
- LIMIT 1` — so a sub-day re-anchoring is overridden back to the first pair, and a
1557
- day-scale one lands under a DIFFERENT `week_start_date` because the date derives
1558
- from the payload's own week start. No code change is owed either way, because
1559
- `named.update(dict.fromkeys(… or (identity,)))` already handles one through N.
1560
- Two earlier forms of this sentence cited a producer and both were false. Do NOT
1561
- cite the timezone fork: it produces the INVERSE shape, identical bounds under two
1562
- different `week_start_date` values (see the two-identities paragraph below),
1563
- which puts one identity under each date rather than two under one. Do NOT cite
1564
- the boundary-shift branch of `_backfill_week_reset_events`
1565
- (bin/_cctally_weekrefs.py) either: it writes only `week_reset_events` and never
1566
- the snapshot bounds this substitution reads, and its reset test needs THREE
1567
- signals rather than the two an earlier form named — the boundary shifted, the
1568
- prior boundary was still in the future, AND `_is_reset_drop(prior_pct, cur_pct,
1569
- allow_reset_to_zero=False)`, which with the zero leg off is the >=25pp drop
1570
- (`bin/_cctally_weekrefs.py:625-637`).
1571
-
1572
- CONTAINMENT OF THE INSTANT IS DELIBERATELY NOT REQUIRED OF A SUBSTITUTE, and
1573
- requiring it would reopen the defect on the shape this tranche exists for. A
1574
- HELD row carries the BASIS week's four columns with the tick's own capture
1575
- stamp, so a held tick after the week's end names a week whose bounds do not
1576
- contain its own capture instant. Filtering substitutes by containment would
1577
- decline to enrich exactly there.
1578
-
1579
- THAT GENERALITY IS WIDER THAN THE HELD-ROW ARGUMENT NEEDS, and the excess is a
1580
- stated limit rather than a defended choice. The held-row shape requires only that
1581
- containment of the CAPTURE instant not be required. Nothing here additionally
1582
- requires a substitute's bounds to be consistent with the `week_start_date` they
1583
- are filed under, so a same-date row spelling bounds that start a week early lets
1584
- the `week_reset_events` leg admit a reset belonging to the PREDECESSOR week, and
1585
- the value is withheld although no credit of the named week retired it. No writer
1586
- produces a date/bounds inconsistency of that size, so the shape needs a
1587
- hand-built or externally-edited store; the narrower rule that would close it is
1588
- to require the substitute's `week_start_at` UTC date to sit within a day of
1589
- `week_start_date`, and it is not taken because the reachable benefit is nil and a
1590
- date/bounds tolerance is one more constant to get wrong.
1591
-
1592
- NOTHING WIDER THAN THE ACCOUNT'S OWN ROWS IS CONSULTED.
1593
- `_get_canonical_boundary_for_date` (bin/_cctally_weekrefs.py:51) is the repo's
1594
- canonical-bounds reader and the obvious candidate, and it carries no
1595
- `account_key` predicate: it returns the EARLIEST bounds-carrying row for a
1596
- `week_start_date` across every account. On a multi-account store that does not
1597
- merely attach the wrong provenance, it fails to fix the defect, because another
1598
- account's re-anchored bounds cannot scope THIS account's credit — driven and
1599
- pinned by
1600
- `tests/test_five_hour_breakdown.py::test_835_r10_the_substitution_takes_the_accounts_own_bounds_not_another_accounts`.
1601
-
1602
- ONE RESIDUAL SURVIVES, AND IT IS PRE-EXISTING RATHER THAN INTENDED. When NO row
1603
- of the named week carries parseable bounds there is nothing account-scoped to
1604
- substitute, so an automatic credit in that week is structurally invisible to
1605
- this filter and the value it retired is published. Closing it needs a source of
1606
- a week's bounds outside the account's own snapshot rows, and the paragraph above
1607
- says why the one that exists is not usable. Pinned, as a gap rather than as
1608
- correct behaviour, by the second half of
1609
- `::test_835_r10_a_bounds_free_capture_record_resolves_the_weeks_automatic_credit`.
1610
-
1611
- A CAPTURE INSTANT CARRIED BY TWO WEEK IDENTITIES RETURNS BOTH. No tie-break is
1612
- invented for it and the caller's union withholds if either week retired the
1613
- value, which is this subsystem's established direction and what both fallback
1614
- passes do as well. The shape is not hypothetical: bin/_cctally_record.py:6601-6614
1615
- records a host briefly running the wrong timezone forking ONE physical week's
1616
- `week_start_date` across 18 rows, and `weekly_credit_floors` matches
1617
- `week_start_date` by equality, so a floor can sit under one fork and not the
1618
- other.
1619
-
1620
- #834 S1 (#835) Gate A S1. `five_hour_blocks` carries no week columns, and the
1621
- retirement bands need all three: the `weekly_credit_floors` leg matches
1622
- `week_start_date` by equality and the `week_reset_events` leg is scoped by the
1623
- week's bounds. The week is therefore resolved from the account's own snapshot
1624
- rows, whose `week_start_at` / `week_end_at` are the same bounds
1625
- `_credit_retirement_bands` is written against.
1626
-
1627
- The comparison is on PARSED epochs rather than on text, for the reason every
1628
- SQL site in this area wraps both sides in ``unixepoch()``: `block_start_at` is
1629
- host-local while `week_start_at` is `+00:00`, so a lexicographic compare would
1630
- mis-order them.
1631
-
1632
- THE TWO LEGS DEGRADE INDEPENDENTLY, the way `_credit_retirement_bands`' own two
1633
- legs already do (#834 S1 Gate A R8). `week_start_at` and `week_end_at` are both
1634
- nullable and `_derive_week_from_payload` leaves them ``None`` on three of its
1635
- four paths, so a week with no bounds is a live shape. Gate A S1 required both
1636
- bounds and resolved NOTHING without them, which threw away a leg it could still
1637
- answer: the `weekly_credit_floors` leg matches on `week_start_date` alone, and
1638
- `week_start_date` / `week_end_date` are `TEXT NOT NULL`. So a bounds-free week
1639
- is placed by DATE instead and returned with ``None`` bounds, which resolves the
1640
- manual-credit leg and skips the `week_reset_events` leg — exactly what
1641
- `_credit_retirement_bands` does with absent bounds.
1642
-
1643
- A bounds-carrying week outranks a bounds-free one, so the date pass runs only
1644
- when the epoch pass places the instant nowhere. The
1645
- date window is INCLUSIVE of `week_end_date`, because the usual bounds-free week
1646
- carries the week's LAST DAY there: `_derive_week_from_payload`'s third path
1647
- writes `week_start_date + 6 days` and its fourth derives the same through
1648
- `compute_week_bounds`. THE TWO READINGS DO MEET, which an earlier form of this
1649
- docstring denied. `_derive_week_from_payload`'s second path takes both dates
1650
- from the payload verbatim behind nothing but a `weekEndDate >= weekStartDate`
1651
- check, and `_account_axis_weeks` sends a week down this pass when its bounds are
1652
- present but UNPARSEABLE, where `week_end_date` came from the bounds-carrying
1653
- writer and is the exclusive boundary's date by construction. Either shape puts
1654
- one calendar day inside two weeks' inclusive ranges.
1655
-
1656
- SO THE DATE PASS RETURNS EVERY CANDIDATE THAT CONTAINS THE DAY AND PICKS NONE,
1657
- and the caller unions their bands. Two single-choice rules were tried and each
1658
- published a retired value on the mirror image of the shape it fixed. Rank order
1659
- placed an instant in the PREDECESSOR whenever the predecessor's newest snapshot
1660
- row was the newer of the two. Preferring the greatest `week_start_date` fixed
1661
- that and broke the case where the floor sits on a LONGER declared week
1662
- `2026-04-20..2026-05-04` beside an uncredited `2026-04-27..2026-05-04` — the
1663
- exact shape whose existence is the argument against a seven-day window, so the
1664
- prohibition and that repair rested on one premise pointing opposite ways. No
1665
- reading of one week's own columns separates two candidates that both declare the
1666
- day, so choosing is a guess in either direction and consulting all of them is
1667
- not.
1668
-
1669
- NEITHER FALLBACK PASS HAS A TIE-BREAK ANY MORE. Both return every candidate they
1670
- place the instant in, and the caller unions the bands. Through Gate A R8 the
1671
- epoch pass returned ONE week — the one whose group held the newest
1672
- `(captured_at_utc, id)`, which is what the since-removed `rk` aggregate ordered
1673
- by — justified by the claim that the newest anchoring is the best available
1674
- evidence of which window was in force at capture. That claim is vacuous on the shape that makes overlapping
1675
- bounds-carrying anchorings reachable at all: bin/_cctally_record.py:6601-6614
1676
- records a host running the wrong timezone for seven minutes forking ONE physical
1677
- week's `week_start_date` across 18 rows, and those two anchorings carry
1678
- IDENTICAL bounds and differ only in the column `weekly_credit_floors` is keyed
1679
- on. Both anchorings are the same window, so there is no "which window was in
1680
- force" to answer, and a newest-anchoring choice decides only which of two
1681
- spellings of one week's floors gets consulted. It also inverts the claim's own
1682
- logic: where two anchorings genuinely differ, that tie-break publishes the
1683
- loser's retired value rather than withholding it.
1684
-
1685
- The two passes still differ in PRECEDENCE, deliberately. The epoch pass has real
1686
- half-open boundaries to test an instant against, so every candidate it keeps
1687
- genuinely contains the instant; the date pass has no boundaries, only two date
1688
- columns of which a live writer fills one with the EXCLUSIVE boundary's date, so
1689
- "contains the day" there does not mean "belongs to the week". A bounds-carrying
1690
- placement is therefore the more precise one and pre-empts the date pass
1691
- entirely. Pinned by
1692
- `tests/test_five_hour_breakdown.py::test_835_r9_the_epoch_pass_unions_every_bounds_carrying_week_it_contains`
1693
- and `::test_835_r8_a_bounds_carrying_week_outranks_a_bounds_free_one`.
1694
-
1695
- THE DATE PASS'S UNION OVER-WITHHOLDS, AND THAT RESIDUAL IS SMALLER THAN ROUND
1696
- FIVE'S. When an instant genuinely belongs to an uncredited candidate and an
1697
- overlapping candidate was credited, the union withholds a value no credit ever
1698
- retired. Round five paid that on EVERY block observed on a week-boundary day,
1699
- because `cmd_record_usage` writes `week_end_date = week_start_date + 7 days` —
1700
- the exclusive boundary's date — while this pass compares inclusively, so every
1701
- pair of adjacent weeks shares one calendar day. The capture pass now answers
1702
- every such instant some snapshot row carries, which is the ordinary case, and
1703
- what is left is an instant NO capture record carries on a store whose weeks have
1704
- no parseable bounds.
1705
-
1706
- The direction is the conservative one — an unavailable marker in preference to a
1707
- number a credit retired — and both fallback passes now take it. It is still not
1708
- the direction EVERY fork in this change takes: an empty result resolves no bands
1709
- and publishes the raw value, which is the pre-#835 disposition kept on purpose.
1710
- `::test_835_r8_the_union_withholds_an_uncredited_weeks_own_value` pins the trade
1711
- rather than leaving a later reader to find it and read it as a defect, and
1712
- `::test_835_r9_a_capture_record_resolves_the_week_a_shared_boundary_day_withheld`
1713
- pins the boundary-day case that no longer pays it.
1714
-
1715
- A SEVEN-DAY WINDOW OFF `week_start_date` IS STILL NOT AN ALTERNATIVE. It assumes
1716
- every bounds-free week is seven days long, and the second payload path can
1717
- declare a longer one, whose tail a seven-day window would place NOWHERE —
1718
- withdrawing the manual-credit leg this pass exists to resolve. It also buys
1719
- nothing on the weeks whose `week_end_date` is computed rather than declared: the
1720
- third path and `compute_week_bounds` both write `week_start_date + 6 days`, and
1721
- `cmd_record_usage` writes `week_end_date` exactly `week_start_date + 7 days`, so
1722
- over those rows a seven-day exclusive window and this inclusive one name the
1723
- same days and neither separates two overlapping candidates.
1724
-
1725
- ONE DIRECTION IS BOUNDED BY `cmd_record_credit`, AND THE OTHER IS NOT. That
1726
- command refuses without a canonical weekly boundary and always writes a
1727
- bounds-carrying synthetic snapshot of its own, so a manually credited week
1728
- normally has at least one row with bounds — which bounds the direction where a
1729
- credited week is reachable ONLY through the date pass. It does not bound the
1730
- opposite direction, the one round five introduced: that same credited week's
1731
- OTHER rows still form a bounds-free candidate group, because `week_end_date`
1732
- joins the grouping, so an instant in an adjacent bounds-free week still picks up
1733
- the credited week's floor and over-withholds. Do not read the synthetic snapshot
1734
- as bounding both.
1735
-
1736
- EMPTY when no week names the instant on any pass. That resolves no bands and
1737
- therefore withholds nothing, which is the pre-#835 behaviour.
1738
-
1739
- ``cache`` IS THE OUTER RENDER CACHE, NOT A WEEK MEMO, AND IT CHANGED SHAPE
1740
- SILENTLY AT GATE A R8. Gate A S1 passed a dict this function used directly as
1741
- ``{(account_key, instant_iso) -> weeks}``. It now carves its own sub-dicts out
1742
- of the dict the caller owns, and that dict is the same one
1743
- `_credit_aware_block_weekly_axes` carves `__bands` out of, so FOUR reserved
1744
- keys share one namespace:
1745
-
1746
- ``__weeks`` `{account_key -> [_AxisWeek]}`, `_account_axis_weeks`' memo
1747
- ``__stamps`` `{account_key -> {second -> identities}}`, the capture records
1748
- ``__by_date`` `{account_key -> {week_start_date -> identities}}` (R10)
1749
- ``__bands`` `{(week, account_key) -> bands}`, owned by the caller
1750
-
1751
- A caller passing the OLD shape is not detected and does not raise: every
1752
- `setdefault` above finds no reserved key, creates a fresh sub-dict, and the
1753
- pre-existing entries are simply never read. The failure is therefore a silent
1754
- loss of memoization — one extra pair of queries per account per render, the cost
1755
- the per-account load exists to remove — and never a wrong answer, because
1756
- nothing reads a key it did not write. The leading double underscore is what
1757
- keeps the two populations apart: an `account_key` is either a 32-character
1758
- lowercase sha256 prefix (`digest.hexdigest()[:32]`, bin/_lib_accounts.py:10,
1759
- :63 and :70) or the literal `unattributed` (bin/_lib_accounts.py:41), so none of
1760
- the four names can collide with one.
1761
-
1762
- EVERY PASS IS GATED ON `_account_axis_weeks` SUCCEEDING, the capture pass
1763
- included, and that coupling is deliberate. It looks gratuitous, because NEITHER
1764
- QUERY'S COLUMN DEPENDENCY CONTAINS THE OTHER'S, so in principle a store that
1765
- fails one could answer the other. The week query reads `week_start_date`,
1766
- `week_end_date`, `week_start_at` and `week_end_at`; the capture query reads
1767
- `captured_at_utc`, `week_start_date`, `week_start_at` and `week_end_at`, plus
1768
- `first_observed_at_utc` and `last_observed_at_utc` from a SECOND table,
1769
- `five_hour_blocks`. So the divergence has two directions — a store carrying the
1770
- bounds without `week_end_date`, and one carrying `week_end_date` without
1771
- `captured_at_utc` or without the block table — and the claim that the capture
1772
- query's dependency was strictly smaller was FALSE WHEN WRITTEN in R10, because
1773
- the capture query has read a second table since R9. Two earlier statements of
1774
- this are withdrawn: the claim was authored by `f32554505`, which is later than
1775
- `df80ca1dd` that removed the week query's aggregate, so it cannot have "stopped
1776
- being true" in that round; and it was never true, because the capture query has
1777
- selected `first_observed_at_utc` / `last_observed_at_utc` from `five_hour_blocks`
1778
- since `9690c0fae`. Three reasons the gate is kept:
1779
-
1780
- - The divergence is unreachable. `week_end_date` is `TEXT NOT NULL` in the
1781
- base `CREATE TABLE` (bin/_cctally_core.py:2127) and no migration adds it, so
1782
- no cctally store has the other four columns without it. The legacy shape
1783
- `bin/build-doctor-fixtures.py` builds lacks `week_start_at` / `week_end_at`
1784
- as well, so BOTH queries fail on it together.
1785
- - An empty week set from the ordinary cause — the account has no snapshot rows
1786
- — implies an empty capture map, because the capture query reads the same
1787
- table filtered on the same `account_key`.
1788
- - Since R10 the capture pass READS `weeks`, to build the substitution index
1789
- `_account_axis_bounds_by_start_date` returns, so running it without them
1790
- degrades it to returning bounds-free identities unenriched.
1791
-
1792
- THAT THIRD REASON IS NOT THAT DECOUPLING WOULD BUY NOTHING. An unenriched
1793
- bounds-free identity still resolves the `weekly_credit_floors` leg, and the empty
1794
- tuple resolves neither leg, so on a store where the week query failed and the
1795
- capture query succeeded, decoupling WOULD buy the manual leg. What makes the gain
1796
- unreachable is reason 1, not any equivalence between the two dispositions. The
1797
- gate is kept because the divergence cannot occur on a store any cctally version
1798
- created, and decoupling would add a second exit path to reason about for a gain
1799
- no store can realise. If a future store shape makes the divergence real, the fix
1800
- is to run the capture pass first and gate only the two fallback passes.
1801
- """
1802
- weeks = _account_axis_weeks(
1803
- conn, account_key=account_key, cache=cache.setdefault("__weeks", {}))
1804
- if not weeks:
1805
- # EVERY pass is gated on the week load, the capture pass included. See the
1806
- # docstring: the coupling is deliberate and, since R10, load-bearing.
1807
- return ()
1808
- try:
1809
- instant = parse_iso_datetime(instant_iso, "axis_instant")
1810
- except (ValueError, TypeError):
1811
- return ()
1812
- moment = instant.timestamp()
1813
- # PASS 1 — the capture record. Truncated to the second on both sides, matching
1814
- # the `unixepoch()` the narrowing query uses; see `_account_axis_capture_weeks`.
1815
- carried = _account_axis_capture_weeks(
1816
- conn, account_key=account_key,
1817
- cache=cache.setdefault("__stamps", {})).get(int(moment))
1818
- if carried:
1819
- # A bounds-free matched row names the week but cannot scope the
1820
- # `week_reset_events` leg, and this pass pre-empts the epoch pass, so
1821
- # substitute the SAME week's bounds as this account's OWN other rows spell
1822
- # them — see the docstring for why the substitution is lossless, why
1823
- # containment is not required of a substitute, and what residual survives.
1824
- named: dict = {}
1825
- for identity in carried:
1826
- if identity[1] is not None:
1827
- named[identity] = None
1828
- continue
1829
- by_date = cache.setdefault("__by_date", {})
1830
- if account_key not in by_date:
1831
- by_date[account_key] = _account_axis_bounds_by_start_date(weeks)
1832
- # Built once per account from the already-loaded week set, so the
1833
- # substitution costs no query and no per-instant scan.
1834
- named.update(dict.fromkeys(
1835
- by_date[account_key].get(identity[0]) or (identity,)))
1836
- return tuple(named)
1837
- # PASS 2 — containment in a week's parsed half-open bounds. A dict as an
1838
- # ordered set throughout: the caller's union is order-insensitive, so the
1839
- # order these are visited in carries no meaning, and membership must not be a
1840
- # linear scan.
1841
- found: dict = {}
1842
- for week in weeks:
1843
- if week.lo is not None and week.lo <= moment < week.hi:
1844
- found[(week.week_start_date, week.week_start_at, week.week_end_at)] = (
1845
- None)
1846
- if found:
1847
- return tuple(found)
1848
- # PASS 3 — the instant's calendar day against the bounds-free weeks' declared
1849
- # date ranges. `.astimezone(utc)` before `.date()` for
1850
- # `_derive_week_from_payload`'s own reason: `parse_iso_datetime` ends in
1851
- # `.astimezone()`, so on a host whose offset puts the moment on another calendar
1852
- # date a bare `.date()` would place the instant in the wrong week. EVERY
1853
- # candidate that declares the day, with no preference among them — see the
1854
- # docstring for why no preference can be grounded here and what the union costs.
1855
- day = instant.astimezone(dt.timezone.utc).date().isoformat()
1856
- for week in weeks:
1857
- if week.lo is not None:
1858
- continue
1859
- # Both columns are `TEXT NOT NULL`, so a None here is a store nobody can
1860
- # place; skip it rather than raise comparing a string to None.
1861
- if week.week_start_date is None or week.week_end_date is None:
1862
- continue
1863
- if week.week_start_date <= day <= week.week_end_date:
1864
- # `week_end_date` is in `_account_axis_weeks`' DISTINCT projection, so
1865
- # two rows differing only there are two candidates carrying ONE identity.
1866
- # The bands would be identical, so the duplicate is dropped rather than
1867
- # resolved twice.
1868
- found[(week.week_start_date, None, None)] = None
1869
- return tuple(found)
1870
-
1871
-
1872
1071
  def _credit_aware_block_weekly_axes(conn, block, *, now_utc, cache=None):
1873
- """A block's two weekly axes, with a credit-RETIRED value replaced by ``None``.
1874
-
1875
- #834 S1 (#835) Gate A S1. THE ONE READ-TIME FILTER for
1876
- `five_hour_blocks.seven_day_pct_at_block_start` and
1877
- `seven_day_pct_at_block_end`, called by `cmd_five_hour_blocks` and
1878
- `cmd_five_hour_breakdown` so the two cannot diverge.
1879
-
1880
- WHY READ TIME, AND WHY A WRITE-TIME FILTER CANNOT SUBSTITUTE. A credit can
1881
- retire a value AFTER the block row was written — that is the ordinary case,
1882
- because the live upsert writes the block's weekly end on every tick while the
1883
- block is open and the credit fires later — so no write-time pass can ever be
1884
- sufficient. FOUR writers store these columns, not three: the live upsert in
1885
- bin/_cctally_record.py, `five_hour_block_close`'s frozen harvest in
1886
- bin/_cctally_journal.py, `_backfill_five_hour_blocks` below, and
1887
- `_migration_merge_5h_block_duplicates_v1` in bin/_cctally_db.py, which writes
1888
- `seven_day_pct_at_block_end` from the group row whose `last_observed_at_utc` is
1889
- MAX. The migration was omitted from the original count; the argument is
1890
- unaffected, because it also stores what was observed, but the number was
1891
- documented and wrong. All four store what the fold OBSERVED; this one read
1892
- decides what a credit retired may still be shown. Making one writer
1893
- credit-aware instead would also break
1894
- rebuild convergence, because the store would then hold a number where a
1895
- rebuilt store holds `NULL` and `tests/test_rederive_command.py` compares
1896
- exactly these two columns across a rebuild.
1897
-
1898
- EACH AXIS IS COMPARED AGAINST THE INSTANT THAT AXIS'S VALUE WAS CAPTURED AT,
1899
- never against a boundary of the block: the start against
1900
- `_block_weekly_start_instant` (`first_observed_at_utc`), the end against
1901
- `_block_weekly_end_instant` (`last_observed_at_utc`). Gate A S1 used the block's
1902
- own boundary stamps instead and that was wrong in both directions — see those
1903
- two helpers. The rule separates the three positions a block's OBSERVATIONS can
1904
- hold relative to a credit floor. Observations entirely BEFORE the floor render
1905
- both stored values, because they genuinely were effective while the block ran
1906
- and withholding them would destroy historical truth. Observations SPANNING the
1907
- floor render the start and withhold a retired end. Observations entirely AFTER
1908
- the floor withhold both, which is the shape a post-credit replay at the retired
1909
- value produces.
1910
-
1911
- The bands come from the block's OWN week and `account_key` — a credit under
1912
- one account retires nothing under another, and a crossed-reset block's two
1913
- axes legitimately resolve two different weeks.
1914
-
1915
- ``cache`` is an optional dict shared across a multi-row render, and it holds
1916
- FOUR reserved keys in ONE namespace: this function's own ``__bands``, resolved
1917
- once per `(week, account_key)`, plus the ``__weeks``, ``__stamps`` and
1918
- ``__by_date`` that `_block_weekly_axis_weeks` carves out of the same dict. That
1919
- function's docstring states each one's shape and what happens to a caller who
1920
- passes the pre-R8 shape, which is a silent loss of memoization rather than a
1921
- wrong answer. Together they hold the account's whole week set, the capture
1922
- records of that account's block axis stamps, the per-`week_start_date` index of
1923
- bounds-carrying identities, and the bands. Each axis instant is then resolved
1924
- against the loaded sets in memory, so the number of queries does not grow with
1925
- the number of rendered blocks. The caller owns the dict and it must not outlive
1926
- the connection.
1927
- """
1928
- _c = _cctally()
1929
- p_start = block.get("seven_day_pct_at_block_start")
1930
- p_end = block.get("seven_day_pct_at_block_end")
1931
- if p_start is None and p_end is None:
1932
- return None, None
1933
- if cache is None:
1934
- cache = {}
1935
- bands = cache.setdefault("__bands", {})
1936
- account_key = block.get("account_key") or _lib_accounts.UNATTRIBUTED
1937
- out = []
1938
- for value, instant in (
1939
- (p_start, _block_weekly_start_instant(block)),
1940
- (p_end, _block_weekly_end_instant(block, now_utc=now_utc)),
1941
- ):
1942
- if value is None or not instant:
1943
- out.append(value)
1944
- continue
1945
- week_ids = _block_weekly_axis_weeks(
1946
- conn, account_key=account_key, instant_iso=instant, cache=cache)
1947
- if not week_ids:
1948
- out.append(value)
1949
- continue
1950
- # THE UNION OF EVERY CANDIDATE WEEK'S BANDS, so a value any one of them
1951
- # retired is withheld. One bounds-carrying week is one candidate; several
1952
- # bounds-free ones are several, because nothing in their own columns says
1953
- # which of them an instant belongs to. Each is still resolved by
1954
- # `_credit_retirement_bands` from its OWN triple, so the per-leg
1955
- # degradation is unchanged: a bounds-free candidate contributes its
1956
- # `weekly_credit_floors` leg and no `week_reset_events` leg, which cannot
1957
- # be scoped without bounds. The memo stays per `(week, account_key)`, so
1958
- # overlapping candidates shared across blocks resolve once each.
1959
- merged: list = []
1960
- for week in week_ids:
1961
- bk = (week, account_key)
1962
- if bk not in bands:
1963
- bands[bk] = _c._credit_retirement_bands(
1964
- conn, week_start_date=week[0], week_start_at=week[1],
1965
- week_end_at=week[2], account_key=account_key)
1966
- merged.extend(bands[bk])
1967
- out.append(
1968
- None if _c._weekly_value_is_retired_replica(
1969
- merged, captured_at_utc=instant, weekly_percent=value)
1970
- else value)
1971
- return out[0], out[1]
1072
+ """A block's two stored weekly axes,
1073
+ ``(seven_day_pct_at_block_start, seven_day_pct_at_block_end)``.
1074
+
1075
+ #840 (spec section 4.8). #835 replaced a value within a point of one a
1076
+ credit retired with ``None`` here, which also withheld every genuine climb
1077
+ back to that value; the retirement band is removed. Stale contributors are
1078
+ refused before publication (spec section 4.4), and a value already
1079
+ poisoned in a store is corrected by `db rederive`'s reviewed `retract`,
1080
+ never by a percentage heuristic. The name, the signature and the callers
1081
+ (`cmd_five_hour_blocks`, `cmd_five_hour_breakdown` and the dashboard's
1082
+ block envelope) are unchanged; ``conn``, ``now_utc`` and ``cache`` are no
1083
+ longer read."""
1084
+ return (block.get("seven_day_pct_at_block_start"),
1085
+ block.get("seven_day_pct_at_block_end"))
1972
1086
 
1973
1087
 
1974
1088
  #: The six `five_hour_milestones` columns every surface already published, plus
1975
1089
  #: the joined EFFECTIVE weekly value. Ordering is `captured_at_utc, id` — spec
1976
1090
  #: §5.2's merged-stream order, which post-credit threshold repeats depend on.
1977
- #: The five `s.*` columns after the joined weekly value are INTERNAL. They exist
1978
- #: only so the credit retirement bands can be resolved against the JOINED ROW's
1979
- #: own account and week, and they are aliased because `captured_at_utc` exists on
1980
- #: both sides of the join. `_load_five_hour_milestones` builds its result dicts
1981
- #: key by key, so none of them can reach a published shape.
1982
1091
  _FIVE_HOUR_MILESTONE_SQL = """
1983
1092
  SELECT m.percent_threshold, m.captured_at_utc, m.block_cost_usd,
1984
1093
  m.marginal_cost_usd, m.seven_day_pct_at_crossing, m.reset_event_id,
1985
- s.weekly_percent AS effective_seven_day_pct_at_crossing,
1986
- s.captured_at_utc AS snapshot_captured_at_utc,
1987
- s.week_start_date AS snapshot_week_start_date,
1988
- s.week_start_at AS snapshot_week_start_at,
1989
- s.week_end_at AS snapshot_week_end_at,
1990
- s.account_key AS snapshot_account_key
1094
+ s.weekly_percent AS effective_seven_day_pct_at_crossing
1991
1095
  FROM five_hour_milestones m
1992
1096
  LEFT JOIN weekly_usage_snapshots s ON s.id = m.usage_snapshot_id
1993
1097
  WHERE {selector}
@@ -2027,34 +1131,12 @@ def _load_five_hour_milestones(
2027
1131
  put the number this change exists to stop showing back on the screen,
2028
1132
  silently and only in the failure case.
2029
1133
 
2030
- **The joined value is itself CREDIT-AWARE** (#834 S1 Gate A R2). A plain
2031
- `LEFT JOIN` is not enough: `maybe_record_milestone` sets `usage_snapshot_id`
2032
- to the row the tick just wrote, and the FIVE-HOUR milestone is not gated off
2033
- on a weekly-clamped tick — only the weekly one is — so in a credited week the
2034
- milestone can name a HELD row carrying a pre-credit weekly value the credit
2035
- retired. Publishing that as the "effective" value would render the exact
2036
- number #836 exists to remove, on all three human surfaces, on a new read.
2037
-
2038
- So the retirement bands are resolved against the JOINED ROW's own account and
2039
- week — never the reader's, because a credit under one account retires nothing
2040
- under another — memoized per `(week, account)` across the rows of one call, and
2041
- a retired joined value is published as None. The surfaces already render a None
2042
- as the unavailable marker, so no renderer changes, and the rule above still
2043
- holds: never a fallback to the raw value. (The memoization is this function's
2044
- own: it walks many rows spanning potentially several `(week, account)` pairs.
2045
- `_latest_seven_day_and_window` does NOT memoize, because its walk resolves the
2046
- scope, the bands and the floor once from the newest row and may not leave that
2047
- scope, so there is nothing to memoize per candidate. An earlier version of this
2048
- paragraph said it did.)
2049
-
2050
- THE BAND IS APPLIED HERE WITHOUT R1's FLOOR BOUND, deliberately. That bound
2051
- exists because a WALK can descend past every post-floor row onto a pre-floor
2052
- one and return the pre-credit reading the band exists to suppress. This read
2053
- performs no walk: each milestone names exactly ONE snapshot row through
2054
- `usage_snapshot_id`, and a pre-floor row's weekly value genuinely WAS the
2055
- effective percentage at that crossing instant. Blanking it would delete
2056
- historical truth rather than protect the reader, which is the same reason the
2057
- block's pre-floor weekly axes render.
1134
+ **The joined value is the snapshot's STORED value** (#840, spec section
1135
+ 4.8). #834 S1 Gate A R2 replaced a joined value within a point of one a
1136
+ credit retired with None, which also withheld genuine readings at that
1137
+ value; the retirement band is removed, and a value already poisoned in a
1138
+ store is corrected by `db rederive`'s reviewed `retract`. A missing joined
1139
+ snapshot still publishes None and never falls back to the raw value.
2058
1140
 
2059
1141
  SELECTOR. Exactly one of `block_id` or `five_hour_window_key` must be given.
2060
1142
  `block_id` is the precise form — `five_hour_milestones.block_id` names one
@@ -2085,29 +1167,10 @@ def _load_five_hour_milestones(
2085
1167
  rows = conn.execute(
2086
1168
  _FIVE_HOUR_MILESTONE_SQL.format(selector=selector), params,
2087
1169
  ).fetchall()
2088
- _c = _cctally()
2089
- bands_cache: dict = {}
2090
1170
  out: list[dict] = []
2091
1171
  for r in rows:
2092
1172
  eff = r["effective_seven_day_pct_at_crossing"]
2093
1173
  raw = r["seven_day_pct_at_crossing"]
2094
- if eff is not None:
2095
- band_key = (r["snapshot_week_start_date"],
2096
- r["snapshot_week_start_at"],
2097
- r["snapshot_week_end_at"],
2098
- r["snapshot_account_key"])
2099
- if band_key not in bands_cache:
2100
- bands_cache[band_key] = _c._credit_retirement_bands(
2101
- conn,
2102
- week_start_date=r["snapshot_week_start_date"],
2103
- week_start_at=r["snapshot_week_start_at"],
2104
- week_end_at=r["snapshot_week_end_at"],
2105
- account_key=r["snapshot_account_key"])
2106
- if _c._weekly_value_is_retired_replica(
2107
- bands_cache[band_key],
2108
- captured_at_utc=r["snapshot_captured_at_utc"],
2109
- weekly_percent=eff):
2110
- eff = None
2111
1174
  out.append({
2112
1175
  "percent_threshold": int(r["percent_threshold"]),
2113
1176
  "captured_at_utc": r["captured_at_utc"],
@@ -2718,6 +1781,7 @@ def cmd_five_hour_breakdown(args: argparse.Namespace) -> int:
2718
1781
 
2719
1782
  def _backfill_five_hour_blocks(
2720
1783
  conn: sqlite3.Connection, *, only_missing: bool = False,
1784
+ journaled_first: "dict | None" = None,
2721
1785
  ) -> int:
2722
1786
  """One-shot historical backfill of five_hour_blocks from existing
2723
1787
  weekly_usage_snapshots data. Idempotent via UNIQUE(five_hour_window_key)
@@ -2740,9 +1804,58 @@ def _backfill_five_hour_blocks(
2740
1804
  re-materializes only the open block (block-only, no milestones — the
2741
1805
  journaled 5h-milestone evts carry their own values and resolve block_id
2742
1806
  against this row), keeping the pass O(open windows) instead of O(all blocks).
1807
+
1808
+ #825/#832 fix round J4 (#954): the rebuild projection reproduces the open
1809
+ block live ingest left, on its accounting columns too.
1810
+
1811
+ * ``journaled_first`` maps ``(account_key, five_hour_window_key)`` to the
1812
+ window's FIRST WRITER — its first effective `snapshot_accept` decision
1813
+ in journal order (`_cctally_journal.
1814
+ journaled_block_first_observations`) — and supplies the block's
1815
+ immutable start values (`first_observed_at_utc`,
1816
+ `five_hour_resets_at`, `block_start_at`,
1817
+ `seven_day_pct_at_block_start`) whenever it is present. Live ingest's
1818
+ upsert never updates those columns: the writer that materializes the
1819
+ row owns them. Two things separate that writer from the earliest
1820
+ surviving snapshot. A five-hour credit's stale-replica removal can
1821
+ delete its snapshot (the band starts at floor10 of the arming tick, so
1822
+ it reaches the genuine pre-credit reading); the snapshot still observed
1823
+ the block, because the removal retires its weekly VALUE, not the
1824
+ observation (J4). And a reading captured earlier can be appended, and
1825
+ so derived, later (fix round J5, 825-832-CD4-V01): its row survives
1826
+ with the earlier capture, but it updated a block another reading had
1827
+ already opened. Neither survivor-derived pick reproduces live. A
1828
+ window without an entry (a payload without an explicit account) falls
1829
+ back to the earliest surviving snapshot.
1830
+ * The projection stamps the computation-time pricing provenance of the
1831
+ totals it computes, through the live upsert's own helper
1832
+ (`_block_pricing_provenance_json`), so a successor's
1833
+ `five_hour_block_close` freezes it exactly as after live ingest. The
1834
+ ordinary (legacy, pre-journal) backfill keeps NULL: its blocks predate
1835
+ provenance, which `_lib_cost_provenance.historical_close_marker` reads
1836
+ as unproven.
2743
1837
  """
2744
1838
  _c = _cctally()
2745
1839
  inserted = 0
1840
+ journaled_first = (journaled_first or {}) if only_missing else {}
1841
+
1842
+ def _first_row(key, acct):
1843
+ """The window's start row: its first writer (J4/J5), else the
1844
+ earliest surviving snapshot."""
1845
+ journaled = journaled_first.get((acct, key))
1846
+ if journaled is not None:
1847
+ return journaled
1848
+ return conn.execute(
1849
+ """
1850
+ SELECT five_hour_resets_at, captured_at_utc, weekly_percent
1851
+ FROM weekly_usage_snapshots
1852
+ WHERE five_hour_window_key = ? AND account_key = ?
1853
+ AND five_hour_percent IS NOT NULL
1854
+ ORDER BY captured_at_utc ASC, id ASC LIMIT 1
1855
+ """,
1856
+ (key, acct),
1857
+ ).fetchone()
1858
+
2746
1859
  try:
2747
1860
  # Iterate distinct windows that have BOTH a canonical key AND a
2748
1861
  # 5h percent. The percent guard is critical: MAX(five_hour_percent)
@@ -2759,16 +1872,24 @@ def _backfill_five_hour_blocks(
2759
1872
  AND snapshots.five_hour_percent IS NOT NULL
2760
1873
  """
2761
1874
  if only_missing:
1875
+ # #997 D19: each account's latest qualifying snapshot comes from
1876
+ # one ranking pass over the qualifying rows (O(n log n)). A
1877
+ # correlated subquery here re-scanned and re-sorted the table per
1878
+ # qualifying row (quadratic: 68.4 s on a 27,521-row store, past
1879
+ # the rebuild's progress-gap bound). The partition groups NULL
1880
+ # account keys as `IS` did; the ORDER BY is unchanged.
2762
1881
  keys_sql += (
2763
- " AND snapshots.id = ("
2764
- " SELECT latest.id"
2765
- " FROM weekly_usage_snapshots AS latest"
2766
- " WHERE latest.account_key IS snapshots.account_key"
2767
- " AND latest.five_hour_window_key IS NOT NULL"
2768
- " AND latest.five_hour_percent IS NOT NULL"
2769
- " ORDER BY unixepoch(latest.captured_at_utc) DESC,"
2770
- " latest.id DESC"
2771
- " LIMIT 1"
1882
+ " AND snapshots.id IN ("
1883
+ " SELECT id FROM ("
1884
+ " SELECT id, ROW_NUMBER() OVER ("
1885
+ " PARTITION BY account_key"
1886
+ " ORDER BY unixepoch(captured_at_utc) DESC,"
1887
+ " id DESC"
1888
+ " ) AS rn"
1889
+ " FROM weekly_usage_snapshots"
1890
+ " WHERE five_hour_window_key IS NOT NULL"
1891
+ " AND five_hour_percent IS NOT NULL"
1892
+ " ) WHERE rn = 1"
2772
1893
  " )"
2773
1894
  " AND NOT EXISTS ("
2774
1895
  " SELECT 1 FROM five_hour_blocks AS blocks"
@@ -2812,18 +1933,9 @@ def _backfill_five_hour_blocks(
2812
1933
  for key, acct in keys:
2813
1934
  if (acct, key) in seen_keys:
2814
1935
  continue
2815
- # Same MIN-captured pick the insert loop makes below, so the
1936
+ # Same start-row pick the insert loop makes below, so the
2816
1937
  # ownership interval and the inserted row cannot disagree.
2817
- anchor_row = conn.execute(
2818
- """
2819
- SELECT five_hour_resets_at
2820
- FROM weekly_usage_snapshots
2821
- WHERE five_hour_window_key = ? AND account_key = ?
2822
- AND five_hour_percent IS NOT NULL
2823
- ORDER BY captured_at_utc ASC, id ASC LIMIT 1
2824
- """,
2825
- (key, acct),
2826
- ).fetchone()
1938
+ anchor_row = _first_row(key, acct)
2827
1939
  if anchor_row is None:
2828
1940
  continue
2829
1941
  try:
@@ -2848,23 +1960,19 @@ def _backfill_five_hour_blocks(
2848
1960
  # See cctally-dev#87.
2849
1961
  conn.execute("BEGIN IMMEDIATE")
2850
1962
  try:
2851
- for key, acct in keys:
2852
- # MIN-captured row defines the immutable block boundary
2853
- # values (deterministic — picking "any in-window row"
2854
- # would be nondeterministic under seconds-level Anthropic
2855
- # ISO jitter that the canonical key collapses). Scoped to
1963
+ for block_index, (key, acct) in enumerate(keys, 1):
1964
+ if not block_index & 1023:
1965
+ # #997 D9: a rebuild's open-block projection reports
1966
+ # progress (throttled; a no-op without a progress sink).
1967
+ _lib_rebuild_progress.tick(blocks=block_index)
1968
+ # The start row (the first writer, else the MIN-captured
1969
+ # survivor) defines the immutable block boundary values
1970
+ # (deterministic — picking "any in-window row" would be
1971
+ # nondeterministic under seconds-level Anthropic ISO jitter
1972
+ # that the canonical key collapses). Scoped to
2856
1973
  # (window_key, account_key) so a shared window's per-account
2857
1974
  # blocks each read their own account's boundary rows (#341).
2858
- min_row = conn.execute(
2859
- """
2860
- SELECT five_hour_resets_at, captured_at_utc, weekly_percent
2861
- FROM weekly_usage_snapshots
2862
- WHERE five_hour_window_key = ? AND account_key = ?
2863
- AND five_hour_percent IS NOT NULL
2864
- ORDER BY captured_at_utc ASC, id ASC LIMIT 1
2865
- """,
2866
- (key, acct),
2867
- ).fetchone()
1975
+ min_row = _first_row(key, acct)
2868
1976
  max_row = conn.execute(
2869
1977
  """
2870
1978
  SELECT captured_at_utc, weekly_percent, five_hour_percent
@@ -2941,9 +2049,15 @@ def _backfill_five_hour_blocks(
2941
2049
  # JSONL if cache.db is unreadable.
2942
2050
  # skip_sync=True: backfill runs inside open_db(); a sync_cache
2943
2051
  # cascade here would reopen cache.db recursively.
2052
+ pricing_evidence = {} if only_missing else None
2944
2053
  totals = _c._compute_block_totals(
2945
2054
  block_start_dt, last_obs_dt, skip_sync=True,
2946
2055
  owner_key=key, windows=ownership.get(acct, []),
2056
+ pricing_evidence=pricing_evidence,
2057
+ )
2058
+ pricing_provenance_json = (
2059
+ _c._block_pricing_provenance_json(pricing_evidence)
2060
+ if pricing_evidence is not None else None
2947
2061
  )
2948
2062
 
2949
2063
  cur = conn.execute(
@@ -2963,12 +2077,14 @@ def _backfill_five_hour_blocks(
2963
2077
  total_cache_create_tokens,
2964
2078
  total_cache_read_tokens,
2965
2079
  total_cost_usd,
2080
+ pricing_provenance_json,
2966
2081
  is_closed,
2967
2082
  created_at_utc,
2968
2083
  last_updated_at_utc,
2969
2084
  account_key
2970
2085
  )
2971
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
2086
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
2087
+ ?)
2972
2088
  """,
2973
2089
  (
2974
2090
  key,
@@ -2985,6 +2101,7 @@ def _backfill_five_hour_blocks(
2985
2101
  totals["cache_create_tokens"],
2986
2102
  totals["cache_read_tokens"],
2987
2103
  totals["cost_usd"],
2104
+ pricing_provenance_json,
2988
2105
  is_closed,
2989
2106
  projection_created_at,
2990
2107
  projection_updated_at,