cctally 1.99.0 → 1.100.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,6 +15,7 @@ from zoneinfo import ZoneInfo
15
15
 
16
16
  from _cctally_core import get_week_start_name
17
17
  from _cctally_quota import (
18
+ QuotaProjectionIncomplete,
18
19
  assert_projection_readable,
19
20
  codex_five_hour_percent_at_crossing,
20
21
  codex_quota_breakdown,
@@ -40,6 +41,8 @@ from _lib_dashboard_sources import (
40
41
  assess_codex_projection_coherence,
41
42
  canonical_alerted_at,
42
43
  canonical_alerted_at_sql,
44
+ claude_stats_digest,
45
+ codex_stats_digest,
43
46
  dashboard_resource_key,
44
47
  )
45
48
  from _lib_quota import (
@@ -85,44 +88,184 @@ _CODEX_QUOTA_OBSERVATION_CACHE: dict[object, tuple[object, ...]] = {}
85
88
 
86
89
 
87
90
  def reset_codex_quota_observation_cache() -> None:
88
- """Clear #582's value-only quota-read memo."""
91
+ """Discard #582's value-only quota-read memo. TEST HOOK — no production caller.
92
+
93
+ Since #583 S5 removed the per-build clear, nothing in the shipped code path
94
+ calls this: the memo is invalidated by its key moving, never by a caller
95
+ emptying it. It survives as the isolation hook the pytest fixtures and the
96
+ cold-reference comparisons use, and `_codex_source_caches` is what the
97
+ build's own checkpoint/restore goes through.
98
+ """
89
99
  _CODEX_QUOTA_OBSERVATION_CACHE.clear()
90
100
 
91
101
 
92
- def _cached_codex_quota_observations(**kwargs) -> tuple[object, ...]:
93
- """Reuse bounded quota reads while their dedicated mutation stream is idle."""
94
- conn = kwargs.get("cache_conn")
95
- if not isinstance(conn, sqlite3.Connection):
96
- return load_codex_quota_observations(**kwargs)
102
+ def _codex_quota_reuse_identity(
103
+ cache_conn, stats_conn, *, stats_identity: tuple | None = None,
104
+ ) -> tuple | None:
105
+ """The full invalidation basis for the cross-build quota memo (#583 S5).
106
+
107
+ The cache legs alone are not sufficient. ``_build_codex_source_state`` used
108
+ to discard this memo on every build precisely because a stats-side
109
+ decoration or account-registry change advances no cache.db leg, so a key
110
+ built only from cache.db would not move for a mutation that changes the
111
+ answer. Returning ``None`` means "cannot establish identity", and every
112
+ caller must treat that as a cold read rather than as a cache hit.
113
+
114
+ Six legs, in two halves. The cache.db half is the quota change ledger's
115
+ high-water sequence, the window-attribution revision and the ``main``
116
+ database path -- an unreadable ledger or an absent ledger table is not an
117
+ idle ledger, so both return ``None``. The stats.db half is the three
118
+ digests the build already computes for its own composite signature
119
+ (``bin/_cctally_tui.py``), reused rather than replaced by a fourth
120
+ identity.
121
+
122
+ ``stats_identity`` is that reuse, made literal: the caller passes
123
+ ``(codex_stats_digest, accounts_identity_digest, claude_stats_digest)``
124
+ already computed for its own signature and this function does not recompute
125
+ them. Deriving them here is not cheap -- the three helpers issue twelve
126
+ stats.db statements, several of them whole-relation scans with an ``ORDER
127
+ BY``, plus one uncached filesystem read per configured provider root once
128
+ the ``accounts`` registry is non-empty. Measured on a copy of the real store
129
+ (276,391 retained Codex quota rows), warm, five samples after one untimed
130
+ warm-up: 35.2 ms median, range 33.1-35.7 ms per derivation.
131
+
132
+ Without a supplied identity the digests are derived here, and then the
133
+ projection gate applies. ``codex_stats_digest``'s relations include
134
+ ``quota_projection_state`` and ``quota_window_blocks``, whose read site is
135
+ classified ``gate_at_caller`` in ``_cctally_quota.PROJECTION_READ_SITE_
136
+ ACTIONS`` precisely because that kernel may not import the gate; this
137
+ function is therefore named in ``PROJECTION_GATE_CALLERS`` and runs it. An
138
+ incomplete projection returns ``None`` rather than propagating, because the
139
+ honest answer for a REUSE IDENTITY is "cannot establish identity" and the
140
+ fail-safe for that is a cold read. Nothing is suppressed by swallowing it:
141
+ every caller of this memo runs a gated projection read of its own within a
142
+ few statements -- ``codex_projection_coherence`` on the build path -- so the
143
+ retry signal still reaches whoever owns the transaction.
144
+
145
+ A ``stats_identity`` of the wrong shape is the one input that RAISES instead
146
+ of returning ``None``, and the two cases are different in kind. Everything
147
+ above is an environmental inability to establish an identity -- an absent
148
+ ledger table, an unreadable database, an incomplete projection -- for which
149
+ a cold read is the correct degrade. A caller that hands this function a
150
+ two-element tuple has a defect in the caller, and taking the cold path for
151
+ it would leave the memo permanently and silently disabled. It is unreachable
152
+ from the production callers, all of which pass either ``None`` or
153
+ ``DashboardReadContext.stats_identity``.
154
+ """
155
+ if not isinstance(cache_conn, sqlite3.Connection):
156
+ return None
157
+ if stats_identity is not None:
158
+ if not isinstance(stats_identity, tuple) or len(stats_identity) != 3:
159
+ raise ValueError(
160
+ "stats_identity must be the three-digest tuple or None")
161
+ elif not isinstance(stats_conn, sqlite3.Connection):
162
+ return None
97
163
  try:
98
- seq_row = conn.execute(
164
+ seq_row = cache_conn.execute(
99
165
  "SELECT seq FROM sqlite_sequence "
100
166
  "WHERE name='quota_window_change_log'"
101
167
  ).fetchone()
102
- table = conn.execute(
168
+ table = cache_conn.execute(
103
169
  "SELECT 1 FROM sqlite_master WHERE type='table' "
104
170
  "AND name='quota_window_change_log'"
105
171
  ).fetchone()
106
- revision_row = conn.execute(
172
+ revision_row = cache_conn.execute(
107
173
  "SELECT value FROM cache_meta "
108
174
  "WHERE key='codex_window_attribution_revision'"
109
175
  ).fetchone()
110
176
  db_path = next(
111
- str(row[2]) for row in conn.execute("PRAGMA database_list")
177
+ str(row[2]) for row in cache_conn.execute("PRAGMA database_list")
112
178
  if str(row[1]) == "main"
113
179
  )
114
180
  except (sqlite3.Error, StopIteration):
115
- return load_codex_quota_observations(**kwargs)
181
+ return None
116
182
  if table is None:
183
+ return None
184
+ if stats_identity is None:
185
+ try:
186
+ assert_projection_readable(stats_conn)
187
+ stats_identity = (
188
+ codex_stats_digest(stats_conn),
189
+ accounts_identity_digest(stats_conn),
190
+ claude_stats_digest(stats_conn),
191
+ )
192
+ except QuotaProjectionIncomplete:
193
+ return None
194
+ except sqlite3.Error:
195
+ return None
196
+ return (
197
+ db_path,
198
+ 0 if seq_row is None else int(seq_row[0]),
199
+ "" if revision_row is None else str(revision_row[0]),
200
+ *stats_identity,
201
+ )
202
+
203
+
204
+ def _cached_codex_quota_observations(**kwargs) -> tuple[object, ...]:
205
+ """Reuse bounded quota reads while their dedicated mutation stream is idle.
206
+
207
+ #583 S5: the memo now survives across source builds, so the key must carry
208
+ the stats-side identity as well as the cache.db legs. ``stats_conn``,
209
+ ``stats_identity`` and ``memoize`` are memo-only inputs; none is forwarded
210
+ to the loader. A caller that already holds the three digests passes
211
+ ``stats_identity`` so they are not recomputed once per memo call.
212
+
213
+ ``memoize=False`` is the opt-out for a key that CANNOT match, and it is a
214
+ cost decision rather than a correctness one. A caller whose key carries the
215
+ tick instant -- ``active_at=now`` and a ``now``-derived
216
+ ``captured_at_or_after`` -- moves both terms on every call, so it can never
217
+ read a prior entry. Going through the memo anyway would establish an
218
+ identity it cannot use (35 ms of stats.db work when no ``stats_identity`` is
219
+ supplied) and retain a result nobody can hit, which also evicts the
220
+ clock-free entries that DO reuse, because the memo is discarded whole on
221
+ reaching 32 entries.
222
+
223
+ THE GENERATION CONTRACT IS WEAKER THAN IT WAS, and the next reader of
224
+ ``_tui_build_source_bundle``'s pin needs to know exactly how. This dict is a
225
+ plain module global with no ``_assert_owner``, and the per-build clear that
226
+ used to stand in ``_build_codex_source_state`` is gone, so an entry
227
+ populated on a non-builder path -- the share render or the per-request
228
+ cycle-detail route, from an unpinned connection at whatever cache generation
229
+ it saw -- can now be served to the builder INSIDE its pinned read
230
+ transaction. Before #583 S5 every cache-backed read inside that pin was
231
+ physically issued on the pinned connection.
232
+
233
+ What still holds, and why that is safe:
234
+
235
+ * Every read the builder issues ITSELF still runs on the pinned connection
236
+ and sees the frozen generation. Only a memo HIT returns rows another
237
+ caller loaded, and after the ``memoize=False`` sites above the sole read
238
+ that can hit inside a build is the five-hour correlation read.
239
+ * A hit requires the whole key to match, and the key is the invalidation
240
+ basis: the ``main`` database path, ``quota_window_change_log``'s
241
+ high-water sequence, ``codex_window_attribution_revision`` and the three
242
+ stats digests. Every ``quota_window_snapshots`` mutation is recorded by
243
+ the three ledger triggers in ``bin/_cctally_db.py``
244
+ (``trg_qws_ledger_ins`` / ``_del`` / ``_upd``, the last firing on the
245
+ semantic column set the loader interprets), and the attribution overlay
246
+ bumps its revision in the same transaction as the rows it applies
247
+ (``bin/_cctally_journal.py``). A retained value is therefore served only
248
+ while the evidence it was derived from has not moved, whatever generation
249
+ the connection that loaded it was on.
250
+ * The value is a tuple of frozen ``QuotaObservation``s, so a hit hands back
251
+ no cursor, connection or lazily-read state belonging to the other caller.
252
+ """
253
+ stats_conn = kwargs.pop("stats_conn", None)
254
+ stats_identity = kwargs.pop("stats_identity", None)
255
+ memoize = bool(kwargs.pop("memoize", True))
256
+ conn = kwargs.get("cache_conn")
257
+ if not memoize or not isinstance(conn, sqlite3.Connection):
258
+ return load_codex_quota_observations(**kwargs)
259
+ identity = _codex_quota_reuse_identity(
260
+ conn, stats_conn, stats_identity=stats_identity)
261
+ if identity is None:
117
262
  return load_codex_quota_observations(**kwargs)
118
263
  roots = kwargs.get("source_root_keys")
119
264
  physical_signatures = kwargs.get("physical_signatures")
120
265
  physical_groups = kwargs.get("physical_groups")
121
266
  key = (
122
267
  id(load_codex_quota_observations),
123
- db_path,
124
- 0 if seq_row is None else int(seq_row[0]),
125
- "" if revision_row is None else str(revision_row[0]),
268
+ identity,
126
269
  None if roots is None else tuple(sorted(str(root) for root in roots)),
127
270
  kwargs.get("captured_at_or_after"),
128
271
  kwargs.get("active_at"),
@@ -139,6 +282,27 @@ def _cached_codex_quota_observations(**kwargs) -> tuple[object, ...]:
139
282
  if cached is not None:
140
283
  return cached
141
284
  loaded = tuple(load_codex_quota_observations(**kwargs))
285
+ # The written memory bound (#583 S5 spec §2.1 "Memory"). Now that this memo
286
+ # survives a build, "32 entries" is a bound on ENTRIES and says nothing on
287
+ # its own about rows, so the worst-case retained row count is stated here:
288
+ #
289
+ # * at most 32 results are retained, and the whole memo is discarded on
290
+ # reaching that count rather than evicted one entry at a time;
291
+ # * a bounded dashboard read contributes at most
292
+ # `DASHBOARD_QUOTA_OBSERVATION_LIMIT` (1,000) `QuotaObservation`s;
293
+ # * the five-hour correlation read in `_quota_read_model` carries NO
294
+ # `max_rows`. It is bounded only by one root's retained observations
295
+ # captured at or after one weekly block's nominal start, which is the
296
+ # term that actually sizes this cache, and it grows with the block's
297
+ # age because an older block's nominal start reaches further back.
298
+ # Measured on a store holding 276,391 Codex quota rows, over the seven
299
+ # weekly blocks live at that instant: 15,129 observations for the
300
+ # newest and 36,943 for the oldest. The worst case is therefore 32
301
+ # times a population in the tens of thousands, not 32 times 1,000.
302
+ # * the doctor's unbounded `latest_per_identity` read is NOT retained
303
+ # here. `bin/_cctally_doctor.py` calls `load_codex_quota_observations`
304
+ # directly rather than through this memo, so its all-history population
305
+ # never enters this dict.
142
306
  if len(_CODEX_QUOTA_OBSERVATION_CACHE) >= 32:
143
307
  _CODEX_QUOTA_OBSERVATION_CACHE.clear()
144
308
  _CODEX_QUOTA_OBSERVATION_CACHE[key] = loaded
@@ -327,10 +491,10 @@ def _resolve_codex_weekly_cycle(
327
491
 
328
492
  #350 — FRESH-FIRST ranking (spec §3.2). Codex has no background quota poll,
329
493
  so ``stale_after_seconds(10_080) == 3600`` makes an idle weekly observation
330
- stale after exactly one hour. Discarding a stale-but-FUTURE boundary blanked
494
+ stale after exactly one hour. Discarding a stale-but-ACTIVE boundary blanked
331
495
  the hero's backward-looking actuals even though the spend was never lost, so
332
- each account's future weekly boundaries are now collected into a fresh set
333
- and a stale set and ranked:
496
+ each account's weekly boundaries that contain ``now`` are collected into a
497
+ fresh set and a stale set and ranked:
334
498
 
335
499
  1. exactly one FRESH boundary -> valid, cycle fresh;
336
500
  2. else, no fresh boundaries and exactly one STALE boundary -> valid, cycle
@@ -363,7 +527,15 @@ def _resolve_codex_weekly_cycle(
363
527
  # current-cycle milestone filter matches `resets_at` exactly, so the
364
528
  # ladder empties for precisely the jittered cycles canonicalization
365
529
  # exists to collapse. Liveness rides the same instant the hero shows.
366
- if baseline is None or baseline.canonical_resets_at <= now_utc:
530
+ # #599: a future reset is not active when its nominal start is also in
531
+ # the future; admitting it makes the bounded accounting range empty.
532
+ if (
533
+ baseline is None
534
+ or baseline.canonical_resets_at <= now_utc
535
+ or baseline.canonical_resets_at - dt.timedelta(
536
+ minutes=history.identity.window_minutes
537
+ ) >= now_utc
538
+ ):
367
539
  continue
368
540
  state = quota_freshness(history.physical_observations, now_utc).state
369
541
  boundary = (history.identity.window_minutes, baseline.canonical_resets_at)
@@ -435,6 +607,7 @@ def resolve_codex_cycle_detail_identity(
435
607
  source_root_keys: Iterable[str],
436
608
  now_utc: dt.datetime,
437
609
  account_key: str | None = None,
610
+ stats_conn=None,
438
611
  ):
439
612
  """The live-cycle identity for a per-request Codex cycle-DETAIL read (#373).
440
613
 
@@ -488,6 +661,10 @@ def resolve_codex_cycle_detail_identity(
488
661
  observations = _cached_codex_quota_observations(
489
662
  source_root_keys=active_roots,
490
663
  cache_conn=cache_conn,
664
+ stats_conn=stats_conn,
665
+ # Both bounds are derived from this request's own instant, so this
666
+ # key can never match a retained entry. See `memoize` there.
667
+ memoize=False,
491
668
  captured_at_or_after=(
492
669
  now_utc - dt.timedelta(days=DASHBOARD_QUOTA_RECENT_DAYS)
493
670
  ),
@@ -918,6 +1095,14 @@ class DashboardReadContext:
918
1095
  codex_quota_actual_thresholds: tuple[int, ...] = ()
919
1096
  codex_quota_projected_thresholds: tuple[int, ...] = ()
920
1097
  cache_report_anomaly_threshold_pp: int = 15
1098
+ #: ``(codex_stats_digest, accounts_identity_digest, claude_stats_digest)``
1099
+ #: when the caller already computed them for its own composite signature,
1100
+ #: so the quota memo's reuse identity does not derive a second copy once
1101
+ #: per call (#583 S5 spec §2.1 item 1, Preserve 2). ``None`` means the
1102
+ #: identity derives them itself under the projection gate; that is the
1103
+ #: share path and the per-request cycle-detail route, neither of which
1104
+ #: computes a build signature.
1105
+ stats_identity: tuple[str, str, str] | None = None
921
1106
 
922
1107
  def __post_init__(self) -> None:
923
1108
  for name in ("range_start", "now_utc"):
@@ -2384,7 +2569,12 @@ def _configured_codex_budget_status(
2384
2569
  )
2385
2570
 
2386
2571
  def _sum_cost(start: dt.datetime, end: dt.datetime) -> float:
2387
- return sum(cost for timestamp, cost in resolved_events if start <= timestamp < end)
2572
+ # stable_sum, not sum: this feeds the byte-compared budget wire, and
2573
+ # the built-in sum() switched to Neumaier compensated summation for
2574
+ # floats in CPython 3.12, so 3.11 renders a different figure.
2575
+ return stable_sum(
2576
+ cost for timestamp, cost in resolved_events if start <= timestamp < end
2577
+ )
2388
2578
 
2389
2579
  recent_start = max(start_at, context.now_utc - dt.timedelta(hours=24))
2390
2580
  # #556 S5 §3.1/§3.3: ONE producer of the wire status, shared with Claude.
@@ -2425,11 +2615,25 @@ def _codex_budget_status_domain(
2425
2615
  ``{"status": None}`` for it made the client render "No budget set." beside
2426
2616
  the command that sets one — to a user who has budgets set. It publishes the
2427
2617
  same ``not_configured.disposition`` the Claude half publishes, which the
2428
- client already handles. An ACCOUNT-SCOPED read is deliberately excluded:
2429
- ``account_budgets_only`` describes the vendor-wide axis, and that account's
2430
- own missing budget is genuinely unset.
2618
+ client already handles. #586 gives the genuinely-unset ACCOUNT-SCOPED read
2619
+ its own ``account_budget_unset`` disposition plus immutable account key, so
2620
+ the card can name ``budget.codex.accounts`` rather than the unrelated
2621
+ vendor-wide command.
2431
2622
  """
2432
2623
  config = context.codex_budget
2624
+ if account_key is not None:
2625
+ per_account = config.get("accounts") if isinstance(config, Mapping) else None
2626
+ if not isinstance(per_account, Mapping) or account_key not in per_account:
2627
+ return {
2628
+ "status": None,
2629
+ "not_configured": {
2630
+ "disposition": "account_budget_unset",
2631
+ "account_key": account_key,
2632
+ # `config set` replaces this map wholesale. Preserve every
2633
+ # sibling in the client-rendered example command (#586).
2634
+ "configured_accounts": dict(per_account or {}),
2635
+ },
2636
+ }
2433
2637
  if (
2434
2638
  account_key is None
2435
2639
  and isinstance(config, Mapping)
@@ -2645,6 +2849,8 @@ def _quota_read_model(
2645
2849
  for observation in _cached_codex_quota_observations(
2646
2850
  source_root_keys={identity.source_root_key},
2647
2851
  cache_conn=context.cache_conn,
2852
+ stats_conn=context.stats_conn,
2853
+ stats_identity=context.stats_identity,
2648
2854
  captured_at_or_after=block.nominal_start_at,
2649
2855
  )
2650
2856
  if observation.identity.window_minutes == 300
@@ -2819,7 +3025,9 @@ def _refresh_budget_status_clock(
2819
3025
  str(status["window_end_at"]).replace("Z", "+00:00")
2820
3026
  ).astimezone(UTC)
2821
3027
  recent_start = max(start_at, now_utc - dt.timedelta(hours=24))
2822
- recent_24h_usd = sum(
3028
+ # stable_sum, not sum: same byte-compared budget wire as the producer
3029
+ # above, and the same CPython 3.12 sum() change applies.
3030
+ recent_24h_usd = stable_sum(
2823
3031
  float(cost) for timestamp, cost in cost_events
2824
3032
  if isinstance(timestamp, dt.datetime)
2825
3033
  and start_at <= timestamp.astimezone(UTC) < now_utc
@@ -3126,8 +3334,8 @@ def refresh_codex_source_clock(
3126
3334
  availability = "partial"
3127
3335
  # #556 S1 §4.1: an EXPIRED boundary is exactly the state the
3128
3336
  # accounting axis reports as stale. Build time can never see it
3129
- # (`_resolve_codex_weekly_cycle` retains only `resets_at > now`),
3130
- # so this clock is the only writer of that value.
3337
+ # (`_resolve_codex_weekly_cycle` retains only boundaries containing
3338
+ # `now`), so this clock is the only writer of that value.
3131
3339
  domain_freshness["hero"] = "stale"
3132
3340
  cycle_changed = True
3133
3341
  # 3. budget last
@@ -3894,9 +4102,19 @@ def _codex_accounts_wire(
3894
4102
  cycles: list["CodexCycleBoundary"],
3895
4103
  accounting_start: dt.datetime,
3896
4104
  accounting_end: dt.datetime,
4105
+ population: tuple[object, ...],
3897
4106
  ) -> tuple[list[dict[str, object]], list[dict[str, object]]]:
3898
4107
  """Return ``(accounts_wire, hero_cycles_wire)`` for a decorated Codex source.
3899
4108
 
4109
+ ``population`` is the merged, encounter-ordered accounting read every card
4110
+ is derived from (#583 S5 change 2). This function issues no accounting SQL
4111
+ of its own. It walks the population ONCE, partitioning it by account in
4112
+ encounter order, and then range-filters each card's own partition, so
4113
+ neither the read count nor the in-memory work grows with the account count.
4114
+ The caller loads it over a range that starts no later than any card's own
4115
+ range, which on the share path is EARLIER than the published accounting
4116
+ range.
4117
+
3900
4118
  Caller must gate on ``provider_is_decorated(stats_conn, "codex")`` — this
3901
4119
  builds nothing for a <=1-real-account install (the whole surface is absent,
3902
4120
  so the envelope stays byte-identical, spec R8). Each account carries
@@ -3940,11 +4158,79 @@ def _codex_accounts_wire(
3940
4158
  "startAt": fallback_start.astimezone(UTC).isoformat(),
3941
4159
  "endAt": context.now_utc.astimezone(UTC).isoformat(),
3942
4160
  }
4161
+ # ONE encounter-ordered pass over the population, in place of one pass per
4162
+ # card (#583 S5 spec §2.2). `_codex_partition_by_account` applies the same
4163
+ # `or UNATTRIBUTED` normalization each card's filter used to apply itself,
4164
+ # so the buckets are the sets those filters selected from, and it preserves
4165
+ # encounter order, which `_aggregate_codex_buckets` depends on for a
4166
+ # bucket's first-seen model order.
4167
+ by_account = _codex_partition_by_account(population)
4168
+
4169
+ def _slice(
4170
+ account_key: str,
4171
+ start: dt.datetime,
4172
+ end: dt.datetime,
4173
+ source_root_keys: Iterable[str] | None = None,
4174
+ ) -> tuple[object, ...]:
4175
+ """One card's rows, range-filtered from that account's partition.
4176
+
4177
+ The predicates match the reader these filters replaced, member for
4178
+ member: half-open ``[start, end)`` on a UTC-normalized timestamp, the
4179
+ reserved sentinel standing for a NULL stamp, and a root set that selects
4180
+ nothing when it is supplied empty. Encounter order is the partition's,
4181
+ which is the population's, which is the reader's ``ORDER BY`` over a
4182
+ wider range, so a bucket's first-seen model order is unchanged.
4183
+
4184
+ Two comparisons are nevertheless not the SAME comparison, and both are
4185
+ equivalent only because ingest cannot write the row that separates them.
4186
+ Stated here so the next reader does not have to re-derive it, and
4187
+ asserted at the producer by
4188
+ ``test_ingest_writes_only_canonical_account_and_timestamp_values``.
4189
+
4190
+ 1. The replaced per-card query passed ``account_key=<key>`` to
4191
+ ``load_cached_rooted_codex_accounting_entries``, whose sentinel
4192
+ predicate is ``account_key IS NULL OR account_key = 'unattributed'``.
4193
+ A row stored as the EMPTY STRING matched neither that nor any real
4194
+ account's ``account_key = ?``, so it appeared on no card at all,
4195
+ while the reader maps ``''`` to the sentinel and this filter
4196
+ therefore includes it. No shipped writer emits it: the three
4197
+ statements that set this column (`bin/_cctally_cache.py` -- the
4198
+ ingest insert, the window spend-adoption update and the attribution
4199
+ restore) each write NULL or a resolved 32-hex account key, and the
4200
+ ingest value comes from ``codex_file_accounts.account_key``, itself
4201
+ written only by two statements with the same two outcomes. Measured:
4202
+ zero empty strings among 163,424 rows of the maintainer's store.
4203
+ Note also that every OTHER account-partitioned surface already files
4204
+ such a row under the sentinel -- ``_codex_partition_by_account``
4205
+ applies the identical ``or _lib_accounts.UNATTRIBUTED`` -- so the
4206
+ card query was the sole outlier rather than the rule.
4207
+ 2. The reader bounds ``timestamp_utc`` by comparing ISO TEXT inside
4208
+ SQLite; this filter compares parsed UTC datetimes. For the canonical
4209
+ ``...+00:00`` spelling the two orders coincide, including against a
4210
+ bound carrying microseconds, because ``+`` sorts before ``.``. They
4211
+ diverge only for a stamp carrying a different valid offset, and a
4212
+ NAIVE stamp is not a divergence at all: ``_parse_timestamp`` refuses
4213
+ it, so such a row fails the whole read on the old path and the new
4214
+ one alike. Codex ingest writes
4215
+ ``entry.timestamp.astimezone(UTC).isoformat()``, always ``+00:00``.
4216
+ Measured: 163,424 of 163,424 rows canonical.
4217
+ """
4218
+ roots = (
4219
+ None if source_root_keys is None
4220
+ else {key for key in source_root_keys
4221
+ if isinstance(key, str) and key}
4222
+ )
4223
+ if roots is not None and not roots:
4224
+ return ()
4225
+ return tuple(
4226
+ row for row in by_account.get(account_key, ())
4227
+ if start <= row.timestamp < end
4228
+ and (roots is None or row.source_root_key in roots)
4229
+ )
4230
+
3943
4231
  # Include unattributed last iff it has cycle/5h/spend evidence.
3944
- unattributed_rows = load_cached_rooted_codex_accounting_entries(
3945
- accounting_start, accounting_end, speed=context.speed,
3946
- cache_conn=context.cache_conn, account_key=_lib_accounts.UNATTRIBUTED,
3947
- )
4232
+ unattributed_rows = _slice(
4233
+ _lib_accounts.UNATTRIBUTED, accounting_start, accounting_end)
3948
4234
  # Existence is decided over the accounting range so a sentinel holding only
3949
4235
  # older spend keeps its card; the totals below cover the bounded window, so
3950
4236
  # a resolved $0.00 is an honest empty state rather than an absence (#564).
@@ -3989,10 +4275,9 @@ def _codex_accounts_wire(
3989
4275
  is_unattributed = key == _lib_accounts.UNATTRIBUTED
3990
4276
  if cyc is not None and not is_unattributed:
3991
4277
  cycle_end = min(accounting_end, cyc.resets_at)
3992
- rows = load_cached_rooted_codex_accounting_entries(
3993
- cyc.start_at, cycle_end, speed=context.speed,
3994
- cache_conn=context.cache_conn,
3995
- source_root_keys=cyc.source_root_keys, account_key=key,
4278
+ rows = _slice(
4279
+ key, cyc.start_at, cycle_end,
4280
+ source_root_keys=cyc.source_root_keys,
3996
4281
  )
3997
4282
  totals = _totals(rows)
3998
4283
  elif is_unattributed:
@@ -4002,10 +4287,7 @@ def _codex_accounts_wire(
4002
4287
  # native cycle width ending now, so this card can be summed into a
4003
4288
  # week-labelled headline without overstating it (#564). No bars or
4004
4289
  # reset, because there is no live cycle to describe.
4005
- rows = load_cached_rooted_codex_accounting_entries(
4006
- fallback_start, accounting_end, speed=context.speed,
4007
- cache_conn=context.cache_conn, account_key=key,
4008
- )
4290
+ rows = _slice(key, fallback_start, accounting_end)
4009
4291
  totals = _totals(rows)
4010
4292
  card: dict[str, object] = {
4011
4293
  "accountKey": key,
@@ -4588,6 +4870,40 @@ def _codex_ingest_backlog_wire(
4588
4870
  }
4589
4871
 
4590
4872
 
4873
+ def _codex_source_caches() -> tuple[dict, ...]:
4874
+ """Every process cache a Codex source build reuses across builds.
4875
+
4876
+ ONE definition, so the set the build checkpoints and the set a cold
4877
+ reference discards cannot drift apart. A cold reference that discards only
4878
+ some of them compares two partly-warm builds and calls the second cold
4879
+ (#583 S5 criterion 10).
4880
+ """
4881
+ return (
4882
+ _CODEX_QUOTA_OBSERVATION_CACHE,
4883
+ _CODEX_PERIOD_VIEW_CACHE,
4884
+ _CODEX_CACHE_REPORT_ROWS,
4885
+ _CODEX_SESSION_VIEW_CACHE,
4886
+ _CODEX_PROJECT_LABEL_CACHE,
4887
+ _CODEX_PROJECT_WIRE_CACHE,
4888
+ _CODEX_ENTRY_ADAPTER_CACHE,
4889
+ _CODEX_WEEKLY_VIEW_CACHE,
4890
+ _CODEX_ACCOUNT_SCOPE_CACHE,
4891
+ )
4892
+
4893
+
4894
+ def reset_codex_source_caches() -> None:
4895
+ """Discard every reuse a Codex source build could draw on.
4896
+
4897
+ The cold reference for a warm-versus-cold equality check: after this, the
4898
+ next build reconstructs from the databases alone. It covers the accounting
4899
+ cache state too, which lives in ``_lib_snapshot_cache`` rather than in a
4900
+ dict here and is the other half of what a build retains.
4901
+ """
4902
+ for cache in _codex_source_caches():
4903
+ cache.clear()
4904
+ _lib_snapshot_cache.reset_codex_accounting_cache_state()
4905
+
4906
+
4591
4907
  def build_codex_source_state(
4592
4908
  context: DashboardReadContext,
4593
4909
  *,
@@ -4606,23 +4922,12 @@ def build_codex_source_state(
4606
4922
  every Codex session view in the build, and it is discarded when the read
4607
4923
  returns.
4608
4924
  """
4609
- # This memo deduplicates the several account/parent consumers inside ONE
4610
- # coordinated source build. It may not cross that boundary: a caller can
4611
- # deliberately request a fresh build after stats/account decoration changes
4612
- # without advancing cache.db's quota ledger, and the established contract
4613
- # requires one bounded physical load for that new build.
4614
- reset_codex_quota_observation_cache()
4615
- caches = (
4616
- _CODEX_QUOTA_OBSERVATION_CACHE,
4617
- _CODEX_PERIOD_VIEW_CACHE,
4618
- _CODEX_CACHE_REPORT_ROWS,
4619
- _CODEX_SESSION_VIEW_CACHE,
4620
- _CODEX_PROJECT_LABEL_CACHE,
4621
- _CODEX_PROJECT_WIRE_CACHE,
4622
- _CODEX_ENTRY_ADAPTER_CACHE,
4623
- _CODEX_WEEKLY_VIEW_CACHE,
4624
- _CODEX_ACCOUNT_SCOPE_CACHE,
4625
- )
4925
+ # #583 S5: this memo now survives across builds. It may do so ONLY because
4926
+ # its key carries the stats-side identity as well as the cache legs -- see
4927
+ # `_codex_quota_reuse_identity`. The clear that used to stand here existed
4928
+ # because a decoration or account-registry change advances no cache.db leg
4929
+ # and so could not move a cache-only key. Do not narrow that identity.
4930
+ caches = _codex_source_caches()
4626
4931
  cache_checkpoint = tuple(dict(cache) for cache in caches)
4627
4932
  accounting_checkpoint = (
4628
4933
  _lib_snapshot_cache.checkpoint_codex_accounting_cache_state()
@@ -4652,9 +4957,46 @@ def _build_codex_source_state(
4652
4957
  "SELECT source_root_key FROM codex_source_roots"
4653
4958
  )
4654
4959
  ))
4960
+ # This read is a function of the tick instant, so it does NOT reuse across
4961
+ # ticks and the cross-build memo above cannot help it. Both bounds move
4962
+ # with `context.now_utc`, which in production is the tick's wall clock
4963
+ # (`bin/_cctally_tui.py`: `now_utc or dt.datetime.now(dt.timezone.utc)`).
4964
+ #
4965
+ # It cannot be made reusable by loading a memoizable superset and filtering
4966
+ # in memory, and the reason is a property of the loader rather than of this
4967
+ # call: `max_rows` is a SQL `LIMIT` under an `ORDER BY` whose leading term
4968
+ # is `resets_at_utc > active_at`, so a superset truncates a DIFFERENT set of
4969
+ # rows; and `_apply_codex_window_attribution_overlay` plus
4970
+ # `adopt_unidentified_observations` are population-dependent folds run after
4971
+ # the bound, so a wider population can change the account a retained
4972
+ # observation carries. Filtering a superset would therefore change the
4973
+ # published answer, not just its cost.
4974
+ #
4975
+ # Measured on a copy of the real store (276,391 retained Codex quota rows),
4976
+ # warm, five samples after one untimed warm-up: 222.5 ms median per tick,
4977
+ # range 210.7-231.3 ms.
4978
+ #
4979
+ # As a share, that is 13.9% of 1.60 s, and the denominator has to be named:
4980
+ # 1.60 s is the sum of the three quota call sites when each is timed ALONE
4981
+ # and UN-profiled on that store (this bounded read 222.5 ms, the five-hour
4982
+ # correlation read 209.6 ms, the doctor's all-history `latest_per_identity`
4983
+ # read 1,164.7 ms). The S5 measurement record's 2.571 s for
4984
+ # `load_codex_quota_observations` is a cProfile total instead, so putting an
4985
+ # un-profiled numerator over it reads about 8% and understates the share.
4986
+ # Either way the site is well under the threshold that would have required a
4987
+ # superset fix, which is why the design conclusion above stands. The
4988
+ # doctor's read is about 73% of the un-profiled total and this memo never
4989
+ # sees it.
4655
4990
  quota_observations = _cached_codex_quota_observations(
4656
4991
  source_root_keys=active_roots,
4657
4992
  cache_conn=context.cache_conn,
4993
+ stats_conn=context.stats_conn,
4994
+ stats_identity=context.stats_identity,
4995
+ # A key carrying the tick instant cannot match a retained entry, so the
4996
+ # memo is bypassed rather than asked. On the share path
4997
+ # `stats_identity` is None as well, and establishing an identity for an
4998
+ # unreachable key cost 35 ms of stats.db work per render.
4999
+ memoize=False,
4658
5000
  captured_at_or_after=(
4659
5001
  context.now_utc - dt.timedelta(days=DASHBOARD_QUOTA_RECENT_DAYS)
4660
5002
  ),
@@ -5043,12 +5385,64 @@ def _build_codex_source_state(
5043
5385
  budget_events_by_account: dict[str, tuple[tuple[dt.datetime, float], ...]] = {}
5044
5386
  if _codex_decorated:
5045
5387
  try:
5388
+ # #583 S5 change 2: ONE population the cards are partitioned from,
5389
+ # in place of one accounting read per card. The PUBLISHED accounting
5390
+ # range is unchanged -- widening it would move the envelope.
5391
+ #
5392
+ # This derivation population starts earlier than the published range
5393
+ # on purpose. On the dashboard path `accounting_start` is already a
5394
+ # superset of every card's own range, because it is the 30-day
5395
+ # shared range and a live cycle's `start_at` is inside
5396
+ # `(now - 7d, now)` by the two filters in
5397
+ # `_resolve_codex_weekly_cycle`. On the SHARE path it is not:
5398
+ # `bin/_cctally_dashboard_share.py` passes a user-selected
5399
+ # `range_start` verbatim, so a one-day custom share would truncate a
5400
+ # card whose cycle began six days earlier -- silently, with no error
5401
+ # and no moved golden.
5402
+ card_population_start = min(
5403
+ context.range_start,
5404
+ context.now_utc - dt.timedelta(
5405
+ minutes=ACCOUNT_WEEKLY_WINDOW_MINUTES),
5406
+ accounting_start,
5407
+ )
5408
+ if card_population_start == accounting_start:
5409
+ # The build already holds this exact population, so reading it
5410
+ # again would be one extra full-range pass over every retained
5411
+ # Codex accounting row, per tick. The equality is the ordinary
5412
+ # dashboard case rather than an edge: `card_population_start` is
5413
+ # a `min` that includes `accounting_start`, so it can only be
5414
+ # earlier or equal, and it is equal whenever `accounting_start`
5415
+ # is already at or before both `range_start` and `now - 7d` --
5416
+ # which the 30-day shared range always is. Only a custom share
5417
+ # narrower than one native cycle takes the other branch.
5418
+ #
5419
+ # `accounting_entries` is `QualifiedCodexEntry` on the normal
5420
+ # path and `RootedCodexAccountingEntry` on the
5421
+ # `metadata_incomplete` fallback. Both carry the three
5422
+ # attributes `_slice` reads (`account_key`, `timestamp`,
5423
+ # `source_root_key`), both normalize a NULL or empty stored
5424
+ # account to the sentinel, both price through
5425
+ # `_calculate_codex_entry_cost` at this `speed`, and both are
5426
+ # ordered by (timestamp, source_root_key, conversation_key, id)
5427
+ # over the same half-open range -- the qualified read adds only
5428
+ # a LEFT JOIN onto `codex_conversation_threads`, whose
5429
+ # `conversation_key` is that table's PRIMARY KEY and so cannot
5430
+ # duplicate a row.
5431
+ card_population = accounting_entries
5432
+ else:
5433
+ card_population = load_cached_rooted_codex_accounting_entries(
5434
+ card_population_start,
5435
+ accounting_end,
5436
+ speed=context.speed,
5437
+ cache_conn=context.cache_conn,
5438
+ )
5046
5439
  accounts_wire, hero_cycles_wire = _codex_accounts_wire(
5047
5440
  context,
5048
5441
  quota_observations=quota_observations,
5049
5442
  cycles=cycles_all,
5050
5443
  accounting_start=accounting_start,
5051
5444
  accounting_end=accounting_end,
5445
+ population=card_population,
5052
5446
  )
5053
5447
  # #416 §5.3: the per-account CHILDREN beside the merged parent. The
5054
5448
  # scope set is exactly the card set, so every chip the client can
@@ -5298,8 +5692,8 @@ def _build_codex_source_state(
5298
5692
  # while nothing about the accounting changed. The percent age is
5299
5693
  # already carried by `quota` below and by the additive hero-local
5300
5694
  # `cycle_freshness` field. `_resolve_codex_weekly_cycle` retains
5301
- # only boundaries with `resets_at > now`, so a resolved cycle is
5302
- # never expired at build time; the idle clock owns expiry.
5695
+ # only boundaries with `start_at < now < resets_at`, so a resolved
5696
+ # cycle is never expired at build time; the idle clock owns expiry.
5303
5697
  "hero": "stale" if hero_failure else "fresh",
5304
5698
  "quota": (
5305
5699
  "stale"