cctally 1.60.0 → 1.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/bin/_cctally_cache.py +479 -54
- package/bin/_cctally_core.py +8 -0
- package/bin/_cctally_dashboard.py +1467 -234
- package/bin/_cctally_db.py +27 -1
- package/bin/_cctally_doctor.py +2 -0
- package/bin/_cctally_forecast.py +24 -2
- package/bin/_cctally_parser.py +28 -2
- package/bin/_cctally_tui.py +708 -16
- package/bin/_cctally_update.py +49 -9
- package/bin/_cctally_weekrefs.py +101 -2
- package/bin/_lib_aggregators.py +98 -71
- package/bin/_lib_cache_report.py +455 -67
- package/bin/_lib_doctor.py +10 -0
- package/bin/_lib_pricing.py +31 -4
- package/bin/_lib_snapshot_cache.py +1797 -0
- package/bin/_lib_view_models.py +107 -12
- package/bin/cctally +57 -0
- package/dashboard/static/assets/index-0jzYm75p.css +1 -0
- package/dashboard/static/assets/{index-DQH6UPc_.js → index-D_Ylyqsf.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +2 -1
- package/dashboard/static/assets/index-uGwZSssU.css +0 -1
|
@@ -0,0 +1,1797 @@
|
|
|
1
|
+
"""Snapshot-rebuild cache infrastructure for the dashboard/TUI (#268).
|
|
2
|
+
|
|
3
|
+
Pure, unit-testable cache primitives for the per-tick `DataSnapshot`
|
|
4
|
+
rebuild. The dashboard's background sync thread rebuilds the whole
|
|
5
|
+
snapshot every `--sync-interval`; on a large instance a from-scratch
|
|
6
|
+
rebuild re-aggregates the entire history each tick and pegs a CPU core.
|
|
7
|
+
This module holds the cheap change-signals + immutable-aggregate caches
|
|
8
|
+
that let the rebuild recompute only the current/dirty slice and serve
|
|
9
|
+
immutable past periods from memory.
|
|
10
|
+
|
|
11
|
+
Design spec: docs/superpowers/specs/2026-07-04-dashboard-rebuild-perf-design.md
|
|
12
|
+
|
|
13
|
+
What lives here (M0):
|
|
14
|
+
- `SnapshotSignature` + `compute_signature` — a composite data-version
|
|
15
|
+
signature over EVERY table the cached surfaces read (spec §3). Cheap
|
|
16
|
+
`MAX(id)` b-tree descents + a `(count, max-rowid)` change-signal over
|
|
17
|
+
the reset-event tables, plus a monotonic generation counter. When the
|
|
18
|
+
signature is unchanged the rebuild can take the idle path.
|
|
19
|
+
- `new_min_timestamp` — the per-builder timestamp watermark (spec §3,
|
|
20
|
+
Codex F1): the earliest EVENT time among genuinely-new rows, so a
|
|
21
|
+
late/backfilled entry (new `id`, OLD `timestamp_utc`) forces recompute
|
|
22
|
+
of the affected PAST bucket, not just the current one.
|
|
23
|
+
- `bump_generation` / `current_generation` — a monotonic counter bumped
|
|
24
|
+
by any path that deletes/rewrites history in place (orphan prune,
|
|
25
|
+
`cache-sync --rebuild`); part of the signature so a deletion that
|
|
26
|
+
leaves `MAX(id)` unchanged still invalidates (spec §7, Codex F4).
|
|
27
|
+
- `BucketCache` — immutable per-past-bucket `BucketUsage` cache for the
|
|
28
|
+
Group A calendar builders (daily / monthly / weekly), spec §5.1.
|
|
29
|
+
- `SessionCache` — immutable per-session aggregate cache over the FULL
|
|
30
|
+
window for the Group B sessions builder, spec §5.2 / Codex F5.
|
|
31
|
+
|
|
32
|
+
Design invariants (spec §7):
|
|
33
|
+
- Every cached value is treated as IMMUTABLE — callers store finished
|
|
34
|
+
aggregates and never mutate them in place (SSE client threads read the
|
|
35
|
+
published snapshot concurrently).
|
|
36
|
+
- These functions take explicit `sqlite3.Connection` objects and plain
|
|
37
|
+
values; no dashboard/TUI imports, no DB opening of their own.
|
|
38
|
+
|
|
39
|
+
`session_entries` / `codex_session_entries` live in `cache.db`;
|
|
40
|
+
`weekly_usage_snapshots` / `weekly_cost_snapshots` / `week_reset_events`
|
|
41
|
+
/ `weekly_credit_floors` live in `stats.db` (bin/_cctally_db.py,
|
|
42
|
+
bin/_cctally_core.py). `compute_signature` takes both conns for that
|
|
43
|
+
reason.
|
|
44
|
+
"""
|
|
45
|
+
from __future__ import annotations
|
|
46
|
+
|
|
47
|
+
import datetime as dt
|
|
48
|
+
import os
|
|
49
|
+
import sqlite3
|
|
50
|
+
import threading
|
|
51
|
+
from dataclasses import dataclass
|
|
52
|
+
from typing import TYPE_CHECKING, Callable, NamedTuple
|
|
53
|
+
|
|
54
|
+
from _cctally_core import parse_iso_datetime
|
|
55
|
+
|
|
56
|
+
# #271: the current-bucket accumulator (below) reuses the aggregators' single
|
|
57
|
+
# per-entry fold primitive so the incremental append is byte-identical to the
|
|
58
|
+
# full pass (spec §6/§7). This is the ONE deliberate runtime coupling to
|
|
59
|
+
# _lib_aggregators — the cache primitives above stay decoupled (BucketUsage is
|
|
60
|
+
# still a TYPE_CHECKING-only hint). _lib_aggregators imports only _cctally_core,
|
|
61
|
+
# so there is no import cycle.
|
|
62
|
+
from _lib_aggregators import _fold_entry, _finalize_bucket, _new_bucket_acc
|
|
63
|
+
|
|
64
|
+
if TYPE_CHECKING: # type hints only — no runtime coupling to the aggregators
|
|
65
|
+
from _lib_aggregators import BucketUsage, ClaudeSessionUsage
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# === Task 0.1 — composite data-version signature ===========================
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class SnapshotSignature(NamedTuple):
|
|
72
|
+
"""Cheap composite change-signal over every source table (spec §3).
|
|
73
|
+
|
|
74
|
+
Equality is value-equality (NamedTuple), so an unchanged signature
|
|
75
|
+
across two ticks means no cached surface's inputs moved → idle path.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
max_entry_id: int
|
|
79
|
+
max_wus_id: int
|
|
80
|
+
max_wcs_id: int
|
|
81
|
+
reset_sig: tuple[int, int]
|
|
82
|
+
max_codex_id: int
|
|
83
|
+
generation: int
|
|
84
|
+
# #270 §7a: the O(1) `cache_meta` mutation counter. An id-stable in-place
|
|
85
|
+
# finalization UPSERT advances this leg while `max_entry_id` stays flat, so
|
|
86
|
+
# the dashboard leaves the idle path and recomputes the affected bucket.
|
|
87
|
+
entry_mutation_seq: int = 0
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _max_id(conn: sqlite3.Connection, table: str) -> int:
|
|
91
|
+
"""`MAX(id)` on an autoincrement table, 0 on empty or missing table.
|
|
92
|
+
|
|
93
|
+
A single rowid b-tree descent (O(1)-ish). Returns 0 on a fresh /
|
|
94
|
+
partially-migrated DB where the table does not yet exist, so the
|
|
95
|
+
signature never raises.
|
|
96
|
+
"""
|
|
97
|
+
try:
|
|
98
|
+
row = conn.execute(f"SELECT COALESCE(MAX(id), 0) FROM {table}").fetchone()
|
|
99
|
+
return int(row[0])
|
|
100
|
+
except sqlite3.Error:
|
|
101
|
+
return 0
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _entry_mutation_seq(conn: sqlite3.Connection) -> int:
|
|
105
|
+
"""Current ``session_entries`` mutation counter from ``cache_meta`` (§7a).
|
|
106
|
+
|
|
107
|
+
Reads the O(1) KV counter (key ``session_entries_mutation_seq``) that
|
|
108
|
+
``sync_cache`` bumps on every insert AND every WHERE-passing in-place UPSERT
|
|
109
|
+
(#270). This is the new ``entry_mutation_seq`` signature leg: an id-stable
|
|
110
|
+
finalization advances it even though ``MAX(session_entries.id)`` is flat, so
|
|
111
|
+
the dashboard leaves the idle path and recomputes the affected bucket.
|
|
112
|
+
|
|
113
|
+
An O(1) KV read — NOT a ``MAX(mutation_seq)`` full scan (that is exactly the
|
|
114
|
+
per-tick cost #268 removed). Degrades to 0 on a fresh / partially-migrated DB
|
|
115
|
+
where the ``cache_meta`` table or the key is absent (like ``_max_id`` /
|
|
116
|
+
``_reset_sig``), so the signature never raises.
|
|
117
|
+
"""
|
|
118
|
+
try:
|
|
119
|
+
row = conn.execute(
|
|
120
|
+
"SELECT value FROM cache_meta "
|
|
121
|
+
"WHERE key='session_entries_mutation_seq'"
|
|
122
|
+
).fetchone()
|
|
123
|
+
except sqlite3.Error:
|
|
124
|
+
return 0
|
|
125
|
+
if row is None or row[0] is None:
|
|
126
|
+
return 0
|
|
127
|
+
return int(row[0])
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _reset_sig(conn: sqlite3.Connection) -> tuple[int, int]:
|
|
131
|
+
"""Change-signal over the two reset-event tables combined (spec §3).
|
|
132
|
+
|
|
133
|
+
A credit / reset re-shapes a PAST weekly bucket with NO new
|
|
134
|
+
`session_entries` row, so the composite signature must cover it.
|
|
135
|
+
Uses `(COUNT(*), MAX(rowid))` over `week_reset_events` +
|
|
136
|
+
`weekly_credit_floors`: the count catches inserts, the max-rowid
|
|
137
|
+
catches the (rare) case where a delete+insert keeps the count level.
|
|
138
|
+
`rowid` aliases the tables' `INTEGER PRIMARY KEY id`. Returns (0, 0)
|
|
139
|
+
on a fresh DB where the tables are absent.
|
|
140
|
+
"""
|
|
141
|
+
try:
|
|
142
|
+
row = conn.execute(
|
|
143
|
+
"SELECT (SELECT COUNT(*) FROM week_reset_events)"
|
|
144
|
+
" + (SELECT COUNT(*) FROM weekly_credit_floors),"
|
|
145
|
+
" (SELECT COALESCE(MAX(rowid), 0) FROM week_reset_events)"
|
|
146
|
+
" + (SELECT COALESCE(MAX(rowid), 0) FROM weekly_credit_floors)"
|
|
147
|
+
).fetchone()
|
|
148
|
+
return (int(row[0]), int(row[1]))
|
|
149
|
+
except sqlite3.Error:
|
|
150
|
+
return (0, 0)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def compute_signature(
|
|
154
|
+
cache_conn: sqlite3.Connection,
|
|
155
|
+
stats_conn: sqlite3.Connection,
|
|
156
|
+
*,
|
|
157
|
+
generation: int,
|
|
158
|
+
) -> SnapshotSignature:
|
|
159
|
+
"""Composite data-version signature across cache.db + stats.db (spec §3).
|
|
160
|
+
|
|
161
|
+
``cache_conn`` reads ``session_entries`` / ``codex_session_entries``
|
|
162
|
+
(cache.db); ``stats_conn`` reads ``weekly_usage_snapshots`` /
|
|
163
|
+
``weekly_cost_snapshots`` / the reset-event tables (stats.db).
|
|
164
|
+
``generation`` is the current cache-generation counter, folded in so
|
|
165
|
+
an in-place history deletion (which need not lower any ``MAX(id)``)
|
|
166
|
+
still advances the signature.
|
|
167
|
+
"""
|
|
168
|
+
return SnapshotSignature(
|
|
169
|
+
max_entry_id=_max_id(cache_conn, "session_entries"),
|
|
170
|
+
max_wus_id=_max_id(stats_conn, "weekly_usage_snapshots"),
|
|
171
|
+
max_wcs_id=_max_id(stats_conn, "weekly_cost_snapshots"),
|
|
172
|
+
reset_sig=_reset_sig(stats_conn),
|
|
173
|
+
max_codex_id=_max_id(cache_conn, "codex_session_entries"),
|
|
174
|
+
generation=int(generation),
|
|
175
|
+
entry_mutation_seq=_entry_mutation_seq(cache_conn),
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
# === Task 0.2 — new-entry timestamp watermark ==============================
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def new_min_timestamp(
|
|
183
|
+
cache_conn: sqlite3.Connection,
|
|
184
|
+
last_seen_max_id: int,
|
|
185
|
+
) -> "dt.datetime | None":
|
|
186
|
+
"""Earliest EVENT time among genuinely-new session_entries rows.
|
|
187
|
+
|
|
188
|
+
Returns ``MIN(timestamp_utc)`` over rows with ``id > last_seen_max_id``
|
|
189
|
+
as an aware UTC datetime, or ``None`` when there are no such rows.
|
|
190
|
+
|
|
191
|
+
This is the per-builder dirty-bucket watermark (spec §3, Codex F1).
|
|
192
|
+
``session_entries.id`` is INGEST order, not event time — a resumed or
|
|
193
|
+
late-ingested file produces a NEW ``id`` carrying an OLD
|
|
194
|
+
``timestamp_utc``. So when the signature advances, the affected time
|
|
195
|
+
window starts at the earliest event time among the new rows, which may
|
|
196
|
+
reach back into a PAST calendar bucket; each builder recomputes every
|
|
197
|
+
one of its buckets whose window ends after this watermark and serves
|
|
198
|
+
strictly-older buckets from cache.
|
|
199
|
+
|
|
200
|
+
Returns ``None`` on a missing table (fresh DB) so callers can treat it
|
|
201
|
+
as "no new rows".
|
|
202
|
+
"""
|
|
203
|
+
try:
|
|
204
|
+
row = cache_conn.execute(
|
|
205
|
+
"SELECT MIN(timestamp_utc) FROM session_entries WHERE id > ?",
|
|
206
|
+
(last_seen_max_id,),
|
|
207
|
+
).fetchone()
|
|
208
|
+
except sqlite3.Error:
|
|
209
|
+
return None
|
|
210
|
+
if row is None or row[0] is None:
|
|
211
|
+
return None
|
|
212
|
+
# Parse via the repo's canonical ISO helper (handles both `...Z` and
|
|
213
|
+
# `...+00:00` stored forms), then normalize to a UTC-tzinfo instant so
|
|
214
|
+
# downstream comparisons against UTC bucket boundaries are exact.
|
|
215
|
+
parsed = parse_iso_datetime(row[0], "session_entries.timestamp_utc")
|
|
216
|
+
return parsed.astimezone(dt.timezone.utc)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def changed_min_timestamp(
|
|
220
|
+
cache_conn: sqlite3.Connection,
|
|
221
|
+
last_seen_seq: int,
|
|
222
|
+
) -> "dt.datetime | None":
|
|
223
|
+
"""Earliest event time among rows CHANGED since ``last_seen_seq`` (§7b).
|
|
224
|
+
|
|
225
|
+
Returns ``MIN(mutation_min_ts) WHERE mutation_seq > last_seen_seq`` as an
|
|
226
|
+
aware UTC datetime, or ``None`` when there are no such rows. Generalizes
|
|
227
|
+
``new_min_timestamp`` from the ``id`` watermark to the #270 ``mutation_seq``
|
|
228
|
+
watermark; the same parse / normalize-to-UTC path and the same
|
|
229
|
+
``None``-on-missing-table degrade.
|
|
230
|
+
|
|
231
|
+
Load-bearing byte-identity invariant: on a PURE-INSERT interval every changed
|
|
232
|
+
row is a fresh insert carrying a new ``id`` (> last_seen_id), a new
|
|
233
|
+
``mutation_seq`` (> last_seen_seq), and ``mutation_min_ts == timestamp_utc``,
|
|
234
|
+
and no existing row was re-stamped — so ``{mutation_seq > last_seen_seq}`` ==
|
|
235
|
+
``{id > last_seen_id}`` and ``MIN(mutation_min_ts)`` == ``MIN(timestamp_utc)``,
|
|
236
|
+
i.e. this equals ``new_min_timestamp(last_seen_id)`` (late-ingest reach-back
|
|
237
|
+
included). When an in-place update landed, the seq set additionally contains
|
|
238
|
+
the updated row, whose ``mutation_min_ts = min(all event times it has held)``
|
|
239
|
+
pulls the watermark back to the EARLIEST bucket it has touched, so both the
|
|
240
|
+
old bucket (stripped from) and the new bucket (added to) are recomputed;
|
|
241
|
+
over-marking the between-buckets is byte-safe. ``MIN()`` ignores the NULL
|
|
242
|
+
``mutation_min_ts`` of any legacy seq-0 row (which this ``> last_seen_seq``
|
|
243
|
+
predicate never selects anyway).
|
|
244
|
+
|
|
245
|
+
Backed by ``idx_entries_mutation_seq(mutation_seq, mutation_min_ts)`` — an
|
|
246
|
+
index-only range-min over the handful of rows changed since the last tick, so
|
|
247
|
+
#268's per-tick full-scan is not reintroduced.
|
|
248
|
+
"""
|
|
249
|
+
try:
|
|
250
|
+
row = cache_conn.execute(
|
|
251
|
+
"SELECT MIN(mutation_min_ts) FROM session_entries "
|
|
252
|
+
"WHERE mutation_seq > ?",
|
|
253
|
+
(last_seen_seq,),
|
|
254
|
+
).fetchone()
|
|
255
|
+
except sqlite3.Error:
|
|
256
|
+
return None
|
|
257
|
+
if row is None or row[0] is None:
|
|
258
|
+
return None
|
|
259
|
+
# Parse via the repo's canonical ISO helper (both ``...Z`` and ``...+00:00``
|
|
260
|
+
# stored forms), then normalize to a UTC-tzinfo instant so downstream
|
|
261
|
+
# comparisons against UTC bucket boundaries are exact.
|
|
262
|
+
parsed = parse_iso_datetime(row[0], "session_entries.mutation_min_ts")
|
|
263
|
+
return parsed.astimezone(dt.timezone.utc)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
# === Task 0.3 — cache-generation counter ===================================
|
|
267
|
+
#
|
|
268
|
+
# A monotonic counter bumped by any path that deletes/rewrites history in
|
|
269
|
+
# place (orphan prune, `cache-sync --rebuild`). Folded into the composite
|
|
270
|
+
# signature (§3) so a deletion that leaves MAX(id) unchanged still
|
|
271
|
+
# invalidates the caches (spec §7, Codex F4). Guarded by a lock: the
|
|
272
|
+
# counter is bumped from the sync thread and read during signature compute
|
|
273
|
+
# on the same thread, but the lock makes any future off-thread bump safe.
|
|
274
|
+
|
|
275
|
+
_GENERATION_LOCK = threading.Lock()
|
|
276
|
+
_GENERATION = 0
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def bump_generation() -> int:
|
|
280
|
+
"""Monotonically advance the cache-generation counter; return the new value."""
|
|
281
|
+
global _GENERATION
|
|
282
|
+
with _GENERATION_LOCK:
|
|
283
|
+
_GENERATION += 1
|
|
284
|
+
return _GENERATION
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def current_generation() -> int:
|
|
288
|
+
"""Return the current cache-generation counter."""
|
|
289
|
+
with _GENERATION_LOCK:
|
|
290
|
+
return _GENERATION
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
# === Task 0.4 — Group A bucket cache holder ================================
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
class BucketCache:
|
|
297
|
+
"""Immutable per-past-bucket `BucketUsage` cache for the calendar builders.
|
|
298
|
+
|
|
299
|
+
Stores the raw immutable aggregate (a `BucketUsage`) per past bucket,
|
|
300
|
+
keyed by ``(builder_key, bucket_label)`` where ``builder_key`` is one
|
|
301
|
+
of ``"daily"`` / ``"monthly"`` / ``"weekly"`` and ``bucket_label`` is
|
|
302
|
+
that builder's bucket identifier (``"2026-06-30"`` daily, ``"2026-06"``
|
|
303
|
+
monthly, the SubWeek key for weekly).
|
|
304
|
+
|
|
305
|
+
Spec §5.1: the cache holds the RAW aggregate, never the final
|
|
306
|
+
presentation row, and values are treated IMMUTABLE — a recomputed
|
|
307
|
+
bucket is put whole (never mutated in place), so an SSE client thread
|
|
308
|
+
reading a previously-published snapshot's rows can never observe a
|
|
309
|
+
torn value (spec §7 / Codex F7).
|
|
310
|
+
"""
|
|
311
|
+
|
|
312
|
+
def __init__(self) -> None:
|
|
313
|
+
self._store: "dict[tuple[str, str], BucketUsage]" = {}
|
|
314
|
+
|
|
315
|
+
def get(self, builder_key: str, bucket_label: str) -> "BucketUsage | None":
|
|
316
|
+
"""Return the cached aggregate for this bucket, or None on a miss."""
|
|
317
|
+
return self._store.get((builder_key, bucket_label))
|
|
318
|
+
|
|
319
|
+
def put(self, builder_key: str, bucket_label: str, bucket: "BucketUsage") -> None:
|
|
320
|
+
"""Store the (immutable) aggregate for this bucket."""
|
|
321
|
+
self._store[(builder_key, bucket_label)] = bucket
|
|
322
|
+
|
|
323
|
+
def drop_from(
|
|
324
|
+
self, builder_key: str, predicate: "Callable[[str], bool]"
|
|
325
|
+
) -> None:
|
|
326
|
+
"""Evict cached buckets for ``builder_key`` whose label matches ``predicate``.
|
|
327
|
+
|
|
328
|
+
Used to invalidate the dirty tail: pass a predicate that is True for
|
|
329
|
+
every bucket label at/after the watermark (or otherwise known dirty).
|
|
330
|
+
Only the given ``builder_key``'s namespace is touched.
|
|
331
|
+
"""
|
|
332
|
+
doomed = [
|
|
333
|
+
key
|
|
334
|
+
for key in self._store
|
|
335
|
+
if key[0] == builder_key and predicate(key[1])
|
|
336
|
+
]
|
|
337
|
+
for key in doomed:
|
|
338
|
+
del self._store[key]
|
|
339
|
+
|
|
340
|
+
def clear(self) -> None:
|
|
341
|
+
"""Drop every cached bucket across all builders (full invalidation)."""
|
|
342
|
+
self._store.clear()
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
# === Task 0.5 — Group B session cache holder ===============================
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
class SessionCache:
|
|
349
|
+
"""Immutable per-session aggregate cache over the FULL sessions window.
|
|
350
|
+
|
|
351
|
+
Holds ALL sessions in the builder's window (spec §5.2 / Codex F5),
|
|
352
|
+
keyed by the resolved session identity — NOT just the visible top 100.
|
|
353
|
+
Sorting/truncating for the 100-row view is done over ``get_all()`` each
|
|
354
|
+
tick, so a session that was previously below the cut can promote into
|
|
355
|
+
view once it gets new activity; caching only the visible slice would
|
|
356
|
+
make that impossible.
|
|
357
|
+
|
|
358
|
+
Values are aggregated ``ClaudeSessionUsage`` rows, treated immutable: a
|
|
359
|
+
changed session is fully re-aggregated and ``put`` whole (a resumed /
|
|
360
|
+
straddling session re-aggregates from its entire entry set, so there is
|
|
361
|
+
no split-row bug), never mutated in place (spec §7 / Codex F7).
|
|
362
|
+
"""
|
|
363
|
+
|
|
364
|
+
def __init__(self) -> None:
|
|
365
|
+
self._store: "dict[str, ClaudeSessionUsage]" = {}
|
|
366
|
+
|
|
367
|
+
def get_all(self) -> "dict[str, ClaudeSessionUsage]":
|
|
368
|
+
"""Return a shallow COPY of every cached session, keyed by identity.
|
|
369
|
+
|
|
370
|
+
A copy so a caller's sort/truncate over the candidate set can never
|
|
371
|
+
mutate the module-level store (the immutable-cache discipline). The
|
|
372
|
+
aggregate values are shared (they are themselves immutable).
|
|
373
|
+
"""
|
|
374
|
+
return dict(self._store)
|
|
375
|
+
|
|
376
|
+
def put(self, session_key: str, session: "ClaudeSessionUsage") -> None:
|
|
377
|
+
"""Store (or replace) the aggregate for one session identity."""
|
|
378
|
+
self._store[session_key] = session
|
|
379
|
+
|
|
380
|
+
def drop(self, session_keys: "set[str]") -> None:
|
|
381
|
+
"""Remove the given session identities; absent keys are ignored."""
|
|
382
|
+
for key in session_keys:
|
|
383
|
+
self._store.pop(key, None)
|
|
384
|
+
|
|
385
|
+
def clear(self) -> None:
|
|
386
|
+
"""Drop every cached session (full invalidation)."""
|
|
387
|
+
self._store.clear()
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
# === Task 2.1 — Group A cached-bucket recompute helper =====================
|
|
391
|
+
#
|
|
392
|
+
# The shared per-past-bucket aggregate cache for the three calendar
|
|
393
|
+
# builders (daily / monthly / weekly), plus the two entry points the
|
|
394
|
+
# dashboard builders call:
|
|
395
|
+
#
|
|
396
|
+
# - ``cached_buckets`` — the PURE per-bucket assembly loop: recompute the
|
|
397
|
+
# current + caller-marked-dirty buckets whole (from a timestamp-ordered
|
|
398
|
+
# fetch, spec §5.1 / Codex F3), serve clean past buckets from the
|
|
399
|
+
# ``BucketCache``, and recompute (cold-miss) any label the cache lacks.
|
|
400
|
+
# - ``build_cached_group_a`` — the STATEFUL wrapper: each Group A builder
|
|
401
|
+
# is self-caching and independently byte-correct (M2 key decision — no
|
|
402
|
+
# dependency on the not-yet-built M5 dispatch). It tracks THIS builder's
|
|
403
|
+
# own last-seen ``(MAX(session_entries.id), extra_signature)`` alongside
|
|
404
|
+
# the builder's ``BucketCache`` namespace, derives the dirty predicate
|
|
405
|
+
# from the new-entry timestamp watermark (``new_min_timestamp``), and
|
|
406
|
+
# full-invalidates the namespace on an ``extra_signature`` change (weekly
|
|
407
|
+
# snapshot/reset legs, or the daily/monthly display-tz flip) or a
|
|
408
|
+
# ``MAX(id)`` regression (cache.db rebuilt). Cold, warm, and invalidated
|
|
409
|
+
# ticks are all byte-identical to a from-scratch aggregation.
|
|
410
|
+
#
|
|
411
|
+
# Module-level state follows the ``_PROJECTS_ENV_MEMO`` precedent (spec §7):
|
|
412
|
+
# a single shared ``BucketCache`` namespaced by ``builder_key`` and a
|
|
413
|
+
# per-builder last-seen dict. Every cached value is an IMMUTABLE
|
|
414
|
+
# ``BucketUsage``; the builders build FRESH presentation rows over the
|
|
415
|
+
# assembled list each tick and never mutate a cached aggregate (Codex F7).
|
|
416
|
+
|
|
417
|
+
_GROUP_A_CACHE = BucketCache()
|
|
418
|
+
_GROUP_A_LAST_SEEN: "dict[str, dict]" = {}
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def group_a_cache() -> BucketCache:
|
|
422
|
+
"""Return the shared Group A ``BucketCache`` (daily/monthly/weekly)."""
|
|
423
|
+
return _GROUP_A_CACHE
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def reset_group_a_state() -> None:
|
|
427
|
+
"""Clear the Group A bucket cache AND every builder's last-seen state.
|
|
428
|
+
|
|
429
|
+
Test hook + the M5 orphan-prune / ``cache-sync --rebuild`` invalidation
|
|
430
|
+
entry point: after history is deleted or rewritten in place, drop every
|
|
431
|
+
cached bucket and reset the watermarks so the next rebuild recomputes
|
|
432
|
+
from scratch (spec §7 / Codex F4).
|
|
433
|
+
"""
|
|
434
|
+
_GROUP_A_CACHE.clear()
|
|
435
|
+
_GROUP_A_LAST_SEEN.clear()
|
|
436
|
+
reset_group_a_current_state()
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
# === #271 — incremental current-bucket accumulator =========================
|
|
440
|
+
#
|
|
441
|
+
# The Group A builders (daily / monthly / weekly) re-fold the whole OPEN
|
|
442
|
+
# bucket every warm tick (`cached_buckets` recomputes `current_label` from a
|
|
443
|
+
# full window fetch). On a recency-dense instance that open week/month/day
|
|
444
|
+
# holds tens of thousands of entries, so the recompute dominates the residual
|
|
445
|
+
# warm rebuild (#271 §1). This holds a persisted single-left-fold accumulator
|
|
446
|
+
# per builder key: each warm tick folds only the DELTA (new-by-id OR
|
|
447
|
+
# newly-in-window-because-`now`-advanced) into the running aggregate, with a
|
|
448
|
+
# full-recompute fallback when a late older-timestamp row lands mid-bucket.
|
|
449
|
+
#
|
|
450
|
+
# Byte-identity rests on the pinned `(timestamp_utc, id)` fold order (#271 §5)
|
|
451
|
+
# + the shared `_fold_entry` primitive (§6): the incremental append reproduces
|
|
452
|
+
# the full left-fold exactly. Single-writer (sync thread only), module state,
|
|
453
|
+
# never reachable from a published DataSnapshot (F7 — the snapshot holds the
|
|
454
|
+
# finalized BucketUsage, not `acc`).
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
@dataclass
|
|
458
|
+
class CurrentBucketAccumulator:
|
|
459
|
+
"""Persisted running fold of one Group A builder's CURRENT bucket (§7a)."""
|
|
460
|
+
label: str
|
|
461
|
+
acc: dict # running _new_bucket_acc() shape
|
|
462
|
+
tail: "tuple | None" # (timestamp_utc, id) of the last folded entry
|
|
463
|
+
last_seen_id: int # max session_entries.id reconciled (= cur_max_id)
|
|
464
|
+
last_seen_seq: int # #270 §8: the mutation_seq watermark (= cur_max_seq)
|
|
465
|
+
last_now: dt.datetime # now_utc upper bound used to clamp the last fold
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
_GROUP_A_CURRENT: "dict[str, CurrentBucketAccumulator]" = {}
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def reset_group_a_current_state() -> None:
|
|
472
|
+
"""Drop every builder's current-bucket accumulator (prune-site + full-invalidate)."""
|
|
473
|
+
_GROUP_A_CURRENT.clear()
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _finalize_or_none(label, acc, tail):
|
|
477
|
+
"""Finalize ``acc`` into a ``BucketUsage``, or ``None`` when nothing was
|
|
478
|
+
folded (``tail is None`` ⇒ an empty/gap bucket, matching a from-scratch
|
|
479
|
+
``aggregate_one`` returning ``None``)."""
|
|
480
|
+
return None if tail is None else _finalize_bucket(label, acc)
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def accumulate_current_bucket(prior, *, current_label, cur_now, cur_max_id,
|
|
484
|
+
cur_max_seq, fetch_all, fetch_delta, membership,
|
|
485
|
+
mode="auto"):
|
|
486
|
+
"""Pure §7b tick algorithm. Returns ``(BucketUsage | None, CurrentBucketAccumulator)``.
|
|
487
|
+
|
|
488
|
+
``fetch_all() -> list[(id, UsageEntry)]`` over the whole current-bucket
|
|
489
|
+
window (cold / fallback); ``fetch_delta(after_seq, after_ts) ->
|
|
490
|
+
list[(id, UsageEntry)]`` the ``(mutation_seq > after_seq OR ts > after_ts)``
|
|
491
|
+
delta (#270 §8 re-key); both ordered ``(timestamp_utc, id)``.
|
|
492
|
+
``membership(entry) -> bool`` keeps exactly the entries the full pass assigns
|
|
493
|
+
to ``current_label``.
|
|
494
|
+
|
|
495
|
+
#270 §8: the delta is keyed on ``mutation_seq`` (a strict superset of the old
|
|
496
|
+
``id`` leg — every insert carries a fresh seq, so the insert path stays
|
|
497
|
+
byte-identical), so an id-stable in-place finalization of a current-bucket
|
|
498
|
+
row now appears in the delta. But the incremental fold cannot un-fold a stale
|
|
499
|
+
contribution: a finalization that keeps or increases its timestamp sorts
|
|
500
|
+
AT-OR-AFTER ``tail`` and the ``(ts,id) <= tail`` gate would append it,
|
|
501
|
+
double-counting the already-folded partial. So any delta row that is a
|
|
502
|
+
PRE-EXISTING row (``id <= reconciled_max_id``) forces a cold refold of the
|
|
503
|
+
whole current bucket instead. Genuine new inserts (``id > reconciled_max_id``)
|
|
504
|
+
proceed through the unchanged ``(ts,id)``-vs-``tail`` gate + append.
|
|
505
|
+
"""
|
|
506
|
+
def _cold():
|
|
507
|
+
acc = _new_bucket_acc()
|
|
508
|
+
tail = None
|
|
509
|
+
for eid, e in fetch_all():
|
|
510
|
+
if e.model == "<synthetic>" or not membership(e):
|
|
511
|
+
continue
|
|
512
|
+
_fold_entry(acc, e, mode)
|
|
513
|
+
tail = (e.timestamp, eid)
|
|
514
|
+
return acc, tail
|
|
515
|
+
|
|
516
|
+
# Cold refold on: no prior, a bucket rollover (label change), OR a backward
|
|
517
|
+
# wall-clock step (`cur_now < prior.last_now`, e.g. an NTP adjustment). The
|
|
518
|
+
# current-bucket fetch window is clamped to `now_utc`; if `now` moves
|
|
519
|
+
# backward with no data change, the empty-delta fast path would reuse the
|
|
520
|
+
# larger prior fold set and OVER-count vs a from-scratch pass over the
|
|
521
|
+
# shrunken `[start, cur_now]` window. A graceful cold refold matches
|
|
522
|
+
# from-scratch byte-for-byte — never `assert`, a clock step must not crash
|
|
523
|
+
# the dashboard (#271 M1 review).
|
|
524
|
+
if (
|
|
525
|
+
prior is None
|
|
526
|
+
or prior.label != current_label
|
|
527
|
+
or cur_now < prior.last_now
|
|
528
|
+
):
|
|
529
|
+
acc, tail = _cold()
|
|
530
|
+
new = CurrentBucketAccumulator(current_label, acc, tail, cur_max_id,
|
|
531
|
+
cur_max_seq, cur_now)
|
|
532
|
+
return _finalize_or_none(current_label, acc, tail), new
|
|
533
|
+
|
|
534
|
+
delta = [
|
|
535
|
+
(eid, e) for eid, e in fetch_delta(prior.last_seen_seq, prior.last_now)
|
|
536
|
+
if e.model != "<synthetic>" and membership(e)
|
|
537
|
+
]
|
|
538
|
+
if not delta:
|
|
539
|
+
# From-scratch fold set unchanged since last tick → byte-identical.
|
|
540
|
+
new = CurrentBucketAccumulator(current_label, prior.acc, prior.tail,
|
|
541
|
+
cur_max_id, cur_max_seq, cur_now)
|
|
542
|
+
return _finalize_or_none(current_label, prior.acc, prior.tail), new
|
|
543
|
+
|
|
544
|
+
# #270 §8: any PRE-EXISTING row in the seq-delta (id <= reconciled_max_id) is
|
|
545
|
+
# an in-place update (a finalization UPSERT re-stamped its seq) — or the
|
|
546
|
+
# near-impossible clock-advance re-entry — that the incremental fold cannot
|
|
547
|
+
# safely reconcile (appending would double-count the already-folded partial).
|
|
548
|
+
# Discard the delta and cold-refold the whole current bucket (always
|
|
549
|
+
# byte-correct, bounded to this bucket). Checked BEFORE the fold-order gate so
|
|
550
|
+
# a timestamp-non-decreasing finalization (which sorts past `tail`, evading
|
|
551
|
+
# that gate) is caught here.
|
|
552
|
+
if any(eid <= prior.last_seen_id for eid, _e in delta):
|
|
553
|
+
acc, tail = _cold()
|
|
554
|
+
new = CurrentBucketAccumulator(current_label, acc, tail, cur_max_id,
|
|
555
|
+
cur_max_seq, cur_now)
|
|
556
|
+
return _finalize_or_none(current_label, acc, tail), new
|
|
557
|
+
|
|
558
|
+
# Guard: every folded delta row must sort AFTER the tail (a strict suffix).
|
|
559
|
+
# tail None ⇒ prior folded nothing ⇒ the delta IS the full member set (each
|
|
560
|
+
# current member has mutation_seq>last_seen_seq OR ts>last_now), so appending
|
|
561
|
+
# == full fold.
|
|
562
|
+
if prior.tail is not None and any(
|
|
563
|
+
(e.timestamp, eid) <= prior.tail for eid, e in delta
|
|
564
|
+
):
|
|
565
|
+
acc, tail = _cold() # late-ingest mid-bucket fallback
|
|
566
|
+
new = CurrentBucketAccumulator(current_label, acc, tail, cur_max_id,
|
|
567
|
+
cur_max_seq, cur_now)
|
|
568
|
+
return _finalize_or_none(current_label, acc, tail), new
|
|
569
|
+
|
|
570
|
+
acc = prior.acc # mutate in place (module state, never published)
|
|
571
|
+
tail = prior.tail
|
|
572
|
+
for eid, e in delta: # fetch_delta returns (timestamp_utc, id) ascending
|
|
573
|
+
_fold_entry(acc, e, mode)
|
|
574
|
+
tail = (e.timestamp, eid)
|
|
575
|
+
new = CurrentBucketAccumulator(current_label, acc, tail, cur_max_id,
|
|
576
|
+
cur_max_seq, cur_now)
|
|
577
|
+
return _finalize_or_none(current_label, acc, tail), new
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def cached_buckets(
|
|
581
|
+
builder_key: str,
|
|
582
|
+
*,
|
|
583
|
+
cache: BucketCache,
|
|
584
|
+
all_bucket_labels: "list[str]",
|
|
585
|
+
current_label: "str | None",
|
|
586
|
+
dirty_predicate: "Callable[[str], bool]",
|
|
587
|
+
fetch_bucket_entries: "Callable[[str], list]",
|
|
588
|
+
aggregate_one: "Callable[[str, list], object | None]",
|
|
589
|
+
current_override: "Callable[[], object | None] | None" = None,
|
|
590
|
+
) -> "list[object]":
|
|
591
|
+
"""Assemble one Group A builder's per-bucket aggregates (spec §5.1).
|
|
592
|
+
|
|
593
|
+
For each label in ``all_bucket_labels`` (caller order — pass oldest→newest
|
|
594
|
+
so the assembled list matches ``_aggregate_*``'s ascending-key output):
|
|
595
|
+
|
|
596
|
+
- If the label is ``current_label`` and ``current_override`` is supplied
|
|
597
|
+
(#271 §8a), the current bucket is produced by ``current_override()`` — the
|
|
598
|
+
incremental accumulator — INSTEAD of the full ``aggregate_one`` recompute.
|
|
599
|
+
The override returns the finalized ``BucketUsage`` (or ``None`` for a
|
|
600
|
+
no-data current bucket, handled by the same gap-drop below).
|
|
601
|
+
- Else if the label is ``current_label`` or ``dirty_predicate`` returns True
|
|
602
|
+
(the watermark reached it, or a forced recompute), recompute it WHOLE via
|
|
603
|
+
``aggregate_one(label, fetch_bucket_entries(label))`` — a
|
|
604
|
+
timestamp-ordered fetch, so ``_aggregate_buckets`` first-seen model order
|
|
605
|
+
reproduces the full-history pass byte-for-byte (Codex F3).
|
|
606
|
+
- Otherwise serve the cached ``BucketUsage``; on a cache MISS (cold start /
|
|
607
|
+
post-invalidation) recompute it the same way.
|
|
608
|
+
|
|
609
|
+
``aggregate_one`` / ``current_override`` return ``None`` for a label with no
|
|
610
|
+
data (a gap day/month/week): the bucket is omitted from the result and any
|
|
611
|
+
stale cache entry for that label is evicted. The returned list therefore
|
|
612
|
+
contains only buckets-with-data — exactly what
|
|
613
|
+
``_aggregate_daily/_monthly/_weekly`` over the full entry set returns — in
|
|
614
|
+
``all_bucket_labels`` order.
|
|
615
|
+
|
|
616
|
+
Values are stored/served as-is (immutable ``BucketUsage``); this loop never
|
|
617
|
+
mutates a bucket in place (spec §7). ``current_override`` stays a pure hook:
|
|
618
|
+
the assembly order, past-bucket serve/cold-miss, and gap-drop are untouched.
|
|
619
|
+
"""
|
|
620
|
+
result: "list[object]" = []
|
|
621
|
+
for label in all_bucket_labels:
|
|
622
|
+
if label == current_label and current_override is not None:
|
|
623
|
+
bucket = current_override()
|
|
624
|
+
else:
|
|
625
|
+
recompute = (label == current_label) or dirty_predicate(label)
|
|
626
|
+
bucket = None
|
|
627
|
+
if not recompute:
|
|
628
|
+
bucket = cache.get(builder_key, label)
|
|
629
|
+
if bucket is None: # forced recompute OR cold miss
|
|
630
|
+
bucket = aggregate_one(label, fetch_bucket_entries(label))
|
|
631
|
+
if bucket is None:
|
|
632
|
+
# Gap bucket (no data): drop any stale cache entry, omit from output.
|
|
633
|
+
cache.drop_from(builder_key, lambda lbl, _t=label: lbl == _t)
|
|
634
|
+
else:
|
|
635
|
+
cache.put(builder_key, label, bucket)
|
|
636
|
+
result.append(bucket)
|
|
637
|
+
return result
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
def build_cached_group_a(
|
|
641
|
+
builder_key: str,
|
|
642
|
+
*,
|
|
643
|
+
cache_conn: sqlite3.Connection,
|
|
644
|
+
all_bucket_labels: "list[str]",
|
|
645
|
+
current_label: "str | None",
|
|
646
|
+
bucket_end_of: "Callable[[str], dt.datetime | None]",
|
|
647
|
+
fetch_bucket_entries: "Callable[[str], list]",
|
|
648
|
+
aggregate_one: "Callable[[str, list], object | None]",
|
|
649
|
+
extra_signature: object = None,
|
|
650
|
+
use_current_accumulator: bool = False,
|
|
651
|
+
now_utc: "dt.datetime | None" = None,
|
|
652
|
+
current_all_fetch: "Callable[[str], list] | None" = None,
|
|
653
|
+
current_delta_fetch: "Callable[[str, int, dt.datetime], list] | None" = None,
|
|
654
|
+
membership_of: "Callable[[object], bool] | None" = None,
|
|
655
|
+
) -> "list[object]":
|
|
656
|
+
"""Stateful Group A assembly: invalidation + watermark + ``cached_buckets``.
|
|
657
|
+
|
|
658
|
+
``cache_conn`` is a ``cache.db`` connection (reads
|
|
659
|
+
``MAX(session_entries.id)`` + the new-entry watermark). ``bucket_end_of``
|
|
660
|
+
maps a label to an aware-UTC datetime that is at-or-after that bucket's
|
|
661
|
+
window end — a bucket is watermark-dirty when its end is after
|
|
662
|
+
``new_min_timestamp`` (over-estimating the end only over-marks buckets
|
|
663
|
+
dirty, which is safe: it never serves stale past data). ``extra_signature``
|
|
664
|
+
is any hashable value whose change forces a full namespace invalidation
|
|
665
|
+
(the weekly snapshot/reset legs, or the daily/monthly display-tz label) —
|
|
666
|
+
e.g. a weekly bucket still transitioning through a credit rides the
|
|
667
|
+
weekly builder's ``extra_signature`` full-invalidate rather than a
|
|
668
|
+
per-label recompute flag.
|
|
669
|
+
|
|
670
|
+
Returns the assembled ``BucketUsage`` list (cache hits for clean past
|
|
671
|
+
labels, fresh recompute for current + dirty), in ``all_bucket_labels``
|
|
672
|
+
order, and updates this builder's last-seen state.
|
|
673
|
+
"""
|
|
674
|
+
cur_max_id = _max_id(cache_conn, "session_entries")
|
|
675
|
+
# #270 §7c: split max_id's two duties — the incremental watermark + "any
|
|
676
|
+
# new work?" gate move to the O(1) `cache_meta` mutation counter (an
|
|
677
|
+
# id-stable in-place finalization advances the seq while `max_id` stays
|
|
678
|
+
# flat), while `max_id` stays the regression backstop below.
|
|
679
|
+
cur_max_seq = _entry_mutation_seq(cache_conn)
|
|
680
|
+
state = _GROUP_A_LAST_SEEN.get(builder_key)
|
|
681
|
+
full_invalidate = state is not None and (
|
|
682
|
+
state.get("extra") != extra_signature
|
|
683
|
+
or cur_max_id < int(state.get("max_id", 0))
|
|
684
|
+
or cur_max_seq < int(state.get("max_seq", 0)) # belt-and-suspenders
|
|
685
|
+
)
|
|
686
|
+
if full_invalidate:
|
|
687
|
+
_GROUP_A_CACHE.drop_from(builder_key, lambda _lbl: True)
|
|
688
|
+
# The current bucket's boundaries may have shifted (a weekly signature
|
|
689
|
+
# move / cache.db rebuild) → discard the accumulator so the override
|
|
690
|
+
# cold-refolds this tick (#271 §8c).
|
|
691
|
+
_GROUP_A_CURRENT.pop(builder_key, None)
|
|
692
|
+
|
|
693
|
+
if state is None or full_invalidate:
|
|
694
|
+
# Cold start OR post-invalidation: recompute every label. (Cold already
|
|
695
|
+
# cache-misses; being explicit also covers the boundary/signature-shift
|
|
696
|
+
# case where a stale label could otherwise collide with a fresh window.)
|
|
697
|
+
def dirty(_label: str) -> bool:
|
|
698
|
+
return True
|
|
699
|
+
else:
|
|
700
|
+
# #270 §7b: the change-aware watermark over `mutation_min_ts` (the
|
|
701
|
+
# earliest event time any row CHANGED-since-last-tick has held), so an
|
|
702
|
+
# id-stable finalization that landed in / moved to a closed bucket marks
|
|
703
|
+
# it dirty. Reduces to `new_min_timestamp(max_id)` on a pure-insert tick.
|
|
704
|
+
new_min_ts = changed_min_timestamp(cache_conn, int(state.get("max_seq", 0)))
|
|
705
|
+
|
|
706
|
+
def dirty(label: str) -> bool:
|
|
707
|
+
if new_min_ts is None:
|
|
708
|
+
return False
|
|
709
|
+
end = bucket_end_of(label)
|
|
710
|
+
return end is not None and end > new_min_ts
|
|
711
|
+
|
|
712
|
+
# #271 §8a/§8b: when the incremental accumulator is enabled, the current
|
|
713
|
+
# bucket is produced by folding only the delta each tick instead of a full
|
|
714
|
+
# re-aggregate. #270 §8: the delta's lower bound is now the mutation_seq
|
|
715
|
+
# watermark `cur_max_seq` (the same O(1) `cache_meta` counter the signature
|
|
716
|
+
# + Group A dirty-predicate read), reused via prior.last_seen_seq inside the
|
|
717
|
+
# accumulator, so an id-stable in-place finalization of a current-bucket row
|
|
718
|
+
# is caught (and cold-refolded). Gated ON only by the sync-thread
|
|
719
|
+
# `_group_a_*_buckets` closures; every other caller leaves it off
|
|
720
|
+
# (byte-identical to today).
|
|
721
|
+
current_override = None
|
|
722
|
+
if use_current_accumulator and current_label is not None:
|
|
723
|
+
def current_override():
|
|
724
|
+
prior = _GROUP_A_CURRENT.get(builder_key)
|
|
725
|
+
bucket, new_state = accumulate_current_bucket(
|
|
726
|
+
prior,
|
|
727
|
+
current_label=current_label,
|
|
728
|
+
cur_now=now_utc,
|
|
729
|
+
cur_max_id=cur_max_id,
|
|
730
|
+
cur_max_seq=cur_max_seq,
|
|
731
|
+
fetch_all=lambda: current_all_fetch(current_label),
|
|
732
|
+
fetch_delta=lambda aseq, ats: current_delta_fetch(
|
|
733
|
+
current_label, aseq, ats),
|
|
734
|
+
membership=membership_of,
|
|
735
|
+
mode="auto",
|
|
736
|
+
)
|
|
737
|
+
_GROUP_A_CURRENT[builder_key] = new_state
|
|
738
|
+
return bucket
|
|
739
|
+
|
|
740
|
+
buckets = cached_buckets(
|
|
741
|
+
builder_key,
|
|
742
|
+
cache=_GROUP_A_CACHE,
|
|
743
|
+
all_bucket_labels=all_bucket_labels,
|
|
744
|
+
current_label=current_label,
|
|
745
|
+
dirty_predicate=dirty,
|
|
746
|
+
fetch_bucket_entries=fetch_bucket_entries,
|
|
747
|
+
aggregate_one=aggregate_one,
|
|
748
|
+
current_override=current_override,
|
|
749
|
+
)
|
|
750
|
+
_GROUP_A_LAST_SEEN[builder_key] = {
|
|
751
|
+
"max_id": cur_max_id, "max_seq": cur_max_seq, "extra": extra_signature,
|
|
752
|
+
}
|
|
753
|
+
return buckets
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
# === Task 3.1 — changed-session resolution (join + filename-stem fallback) ==
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
def affected_session_keys(
|
|
760
|
+
cache_conn: sqlite3.Connection,
|
|
761
|
+
last_seen_seq: int,
|
|
762
|
+
) -> "set[str]":
|
|
763
|
+
"""Resolved session identities for entries CHANGED since ``last_seen_seq`` (§5.2).
|
|
764
|
+
|
|
765
|
+
Mirrors ``_aggregate_claude_sessions`` grouping EXACTLY: identity is
|
|
766
|
+
``session_files.session_id`` when the ``LEFT JOIN`` on ``source_path``
|
|
767
|
+
yields a non-null id, else the filename-stem of ``source_path``
|
|
768
|
+
(``os.path.splitext(os.path.basename(path))[0]``) — the same fallback
|
|
769
|
+
the aggregator applies when ``entry.session_id is None``
|
|
770
|
+
(``bin/_lib_aggregators.py``). ``<synthetic>``-model rows are skipped
|
|
771
|
+
(the aggregator skips them before the fallback), so a purely-synthetic
|
|
772
|
+
new row contributes no key.
|
|
773
|
+
|
|
774
|
+
``session_entries`` has NO ``session_id`` column — identity comes from
|
|
775
|
+
the join to ``session_files`` — so the returned keys key IDENTICALLY to
|
|
776
|
+
the aggregator's session grouping (Codex F5). Returns an empty set on a
|
|
777
|
+
missing table (fresh / partially-migrated DB) so callers never raise.
|
|
778
|
+
|
|
779
|
+
#270 (§7d, Codex-2c): keyed on ``mutation_seq > last_seen_seq``, NOT
|
|
780
|
+
``id > last_seen`` — so an id-stable in-place finalization of an EXISTING
|
|
781
|
+
session's row (same id, advanced seq) surfaces that session as affected and
|
|
782
|
+
it re-aggregates. On a pure-insert interval every changed row carries a
|
|
783
|
+
fresh seq (> last_seen_seq) exactly when its id > last_seen, so the affected
|
|
784
|
+
set is byte-identical to the old id-keyed one.
|
|
785
|
+
"""
|
|
786
|
+
try:
|
|
787
|
+
rows = cache_conn.execute(
|
|
788
|
+
"SELECT se.source_path, sf.session_id "
|
|
789
|
+
"FROM session_entries se "
|
|
790
|
+
"LEFT JOIN session_files sf ON sf.path = se.source_path "
|
|
791
|
+
"WHERE se.mutation_seq > ? AND se.model != '<synthetic>'",
|
|
792
|
+
(last_seen_seq,),
|
|
793
|
+
).fetchall()
|
|
794
|
+
except sqlite3.Error:
|
|
795
|
+
return set()
|
|
796
|
+
keys: "set[str]" = set()
|
|
797
|
+
for source_path, session_id in rows:
|
|
798
|
+
if session_id is not None:
|
|
799
|
+
keys.add(session_id)
|
|
800
|
+
else:
|
|
801
|
+
keys.add(os.path.splitext(os.path.basename(source_path))[0])
|
|
802
|
+
return keys
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
# === Task 3.2 — Group B session aggregate cache over the FULL window =======
|
|
806
|
+
#
|
|
807
|
+
# Module-level state following the Group A precedent (spec §7): a single
|
|
808
|
+
# shared ``SessionCache`` holding ALL sessions in the 365-day window (NOT
|
|
809
|
+
# just the visible top 100 — Codex F5), plus this builder's own last-seen
|
|
810
|
+
# ``MAX(session_entries.id)`` watermark. Every cached value is an immutable
|
|
811
|
+
# ``ClaudeSessionUsage``; the builder builds FRESH ``TuiSessionRow`` objects
|
|
812
|
+
# from the assembled list each tick and never mutates a cached aggregate
|
|
813
|
+
# (Codex F7). Mutated ONLY on the sync thread (single-writer, like Group A).
|
|
814
|
+
|
|
815
|
+
_SESSION_CACHE = SessionCache()
|
|
816
|
+
_SESSION_LAST_SEEN: "dict" = {}
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
def session_cache() -> SessionCache:
|
|
820
|
+
"""Return the shared Group B ``SessionCache`` (all sessions in-window)."""
|
|
821
|
+
return _SESSION_CACHE
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
def reset_session_cache_state() -> None:
|
|
825
|
+
"""Clear the session cache AND its last-seen watermark (full invalidation).
|
|
826
|
+
|
|
827
|
+
Test hook + the M5 orphan-prune / ``cache-sync --rebuild`` invalidation
|
|
828
|
+
entry point: after history is deleted or rewritten in place, drop every
|
|
829
|
+
cached session and reset the watermark so the next rebuild re-aggregates
|
|
830
|
+
the whole window from scratch (spec §7 / Codex F4).
|
|
831
|
+
"""
|
|
832
|
+
_SESSION_CACHE.clear()
|
|
833
|
+
_SESSION_LAST_SEEN.clear()
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
def build_cached_sessions(
|
|
837
|
+
*,
|
|
838
|
+
cache_conn: sqlite3.Connection,
|
|
839
|
+
aggregate_all: "Callable[[], list]",
|
|
840
|
+
reaggregate: "Callable[[int, set], list]",
|
|
841
|
+
extra_signature: object = None,
|
|
842
|
+
) -> "list":
|
|
843
|
+
"""Stateful Group B assembly: cold full-aggregate / warm affected-only (spec §5.2).
|
|
844
|
+
|
|
845
|
+
``cache_conn`` is a ``cache.db`` connection (reads
|
|
846
|
+
``MAX(session_entries.id)`` + the affected-session set). ``aggregate_all``
|
|
847
|
+
returns the full ``list[ClaudeSessionUsage]`` for the window (cold path).
|
|
848
|
+
``reaggregate(last_seen, affected_keys)`` returns the re-aggregated
|
|
849
|
+
``ClaudeSessionUsage`` for exactly the sessions touched since ``last_seen``
|
|
850
|
+
(warm path) — a straddling/resumed session re-aggregates WHOLE from its
|
|
851
|
+
own full in-window entry set, so no split-row bug. ``extra_signature`` is
|
|
852
|
+
any hashable whose change forces a full cold rebuild.
|
|
853
|
+
|
|
854
|
+
Cold when: no prior state, ``extra_signature`` changed, or a
|
|
855
|
+
``MAX(id)`` regression (cache.db rebuilt). Warm otherwise, re-aggregating
|
|
856
|
+
only the affected sessions and updating their cache rows in place of the
|
|
857
|
+
stale ones. Returns the FULL cached session list (UNSORTED — the caller
|
|
858
|
+
window-filters, sorts by ``last_activity`` desc, and truncates to the
|
|
859
|
+
view limit, which is what preserves correct eviction/**promotion** at the
|
|
860
|
+
100-row boundary, Codex F5).
|
|
861
|
+
|
|
862
|
+
Each returned/stored value is an immutable ``ClaudeSessionUsage`` keyed by
|
|
863
|
+
its resolved ``session_id`` (which the aggregator sets to the stem for
|
|
864
|
+
fallback sessions), so the cache keys match ``affected_session_keys``.
|
|
865
|
+
"""
|
|
866
|
+
cur_max_id = _max_id(cache_conn, "session_entries")
|
|
867
|
+
# #270 §7c/§7d: the warm "any new work?" gate + the affected-set query key
|
|
868
|
+
# on the `mutation_seq` counter (an id-stable in-place finalization of an
|
|
869
|
+
# EXISTING session advances the seq while `max_id` stays flat); `max_id`
|
|
870
|
+
# remains the cache.db-rebuild regression backstop.
|
|
871
|
+
cur_max_seq = _entry_mutation_seq(cache_conn)
|
|
872
|
+
state = _SESSION_LAST_SEEN
|
|
873
|
+
cold = (
|
|
874
|
+
not state
|
|
875
|
+
or state.get("extra") != extra_signature
|
|
876
|
+
or cur_max_id < int(state.get("max_id", 0))
|
|
877
|
+
or cur_max_seq < int(state.get("max_seq", 0)) # belt-and-suspenders
|
|
878
|
+
)
|
|
879
|
+
if cold:
|
|
880
|
+
_SESSION_CACHE.clear()
|
|
881
|
+
for sess in aggregate_all():
|
|
882
|
+
_SESSION_CACHE.put(sess.session_id, sess)
|
|
883
|
+
else:
|
|
884
|
+
last_seen_seq = int(state.get("max_seq", 0))
|
|
885
|
+
if cur_max_seq > last_seen_seq:
|
|
886
|
+
affected = affected_session_keys(cache_conn, last_seen_seq)
|
|
887
|
+
if affected:
|
|
888
|
+
# `reaggregate` re-fetches each affected session's FULL in-window
|
|
889
|
+
# entry set (seq-keyed too — Codex-2c), so the aggregate is
|
|
890
|
+
# byte-identical to from-scratch even for an id-stable update.
|
|
891
|
+
for sess in reaggregate(last_seen_seq, affected):
|
|
892
|
+
_SESSION_CACHE.put(sess.session_id, sess)
|
|
893
|
+
_SESSION_LAST_SEEN.clear()
|
|
894
|
+
_SESSION_LAST_SEEN["max_id"] = cur_max_id
|
|
895
|
+
_SESSION_LAST_SEEN["max_seq"] = cur_max_seq
|
|
896
|
+
_SESSION_LAST_SEEN["extra"] = extra_signature
|
|
897
|
+
return list(_SESSION_CACHE.get_all().values())
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
# === Task 4.2 — doctor payload TTL memo (spec §6) ==========================
|
|
901
|
+
#
|
|
902
|
+
# The dashboard envelope used to re-fork the `security` keychain subprocess
|
|
903
|
+
# (via `doctor_gather_state`) once PER SSE CLIENT PER TICK. §6 moves the
|
|
904
|
+
# doctor gather onto the sync-thread `DataSnapshot` (precomputed once per
|
|
905
|
+
# rebuild). This short-TTL memo further guards against back-to-back WARM
|
|
906
|
+
# rebuilds re-forking `security` every tick — the keychain/symlink/log state
|
|
907
|
+
# it reads changes rarely. The `compute` callable is INJECTED so this module
|
|
908
|
+
# stays decoupled from the doctor I/O layer (no `_cctally_doctor` import).
|
|
909
|
+
# The lazy `GET /api/doctor` endpoint deliberately does NOT route through
|
|
910
|
+
# this memo — an explicit user refresh must be live.
|
|
911
|
+
|
|
912
|
+
DOCTOR_MEMO_TTL_S = 30.0
|
|
913
|
+
|
|
914
|
+
_DOCTOR_MEMO_LOCK = threading.Lock()
|
|
915
|
+
_DOCTOR_MEMO: "dict" = {}
|
|
916
|
+
|
|
917
|
+
|
|
918
|
+
def doctor_payload_memo(
|
|
919
|
+
now_utc: dt.datetime,
|
|
920
|
+
runtime_bind: "str | None",
|
|
921
|
+
*,
|
|
922
|
+
ttl_s: float,
|
|
923
|
+
compute: "Callable[[dt.datetime, str | None], dict]",
|
|
924
|
+
) -> dict:
|
|
925
|
+
"""Return the doctor envelope payload, recomputing via ``compute`` only
|
|
926
|
+
when the memo is cold — never computed, older than ``ttl_s``, a clock
|
|
927
|
+
regression (``now_utc`` before the cached instant), or a ``runtime_bind``
|
|
928
|
+
change (the bind feeds ``safety.dashboard_bind``).
|
|
929
|
+
|
|
930
|
+
``compute(now_utc, runtime_bind) -> dict`` is the injected
|
|
931
|
+
gather→checks→envelope-dict step; it runs OUTSIDE the lock so the
|
|
932
|
+
`security` subprocess fork never serializes other readers. In practice
|
|
933
|
+
only the sync thread calls this, so the (harmless) double-compute a
|
|
934
|
+
concurrent caller could trigger never happens.
|
|
935
|
+
"""
|
|
936
|
+
with _DOCTOR_MEMO_LOCK:
|
|
937
|
+
cached = _DOCTOR_MEMO
|
|
938
|
+
computed_at = cached.get("computed_at")
|
|
939
|
+
fresh = (
|
|
940
|
+
bool(cached)
|
|
941
|
+
and cached.get("runtime_bind") == runtime_bind
|
|
942
|
+
and computed_at is not None
|
|
943
|
+
and now_utc >= computed_at
|
|
944
|
+
and (now_utc - computed_at).total_seconds() < ttl_s
|
|
945
|
+
)
|
|
946
|
+
if fresh:
|
|
947
|
+
return cached["payload"]
|
|
948
|
+
payload = compute(now_utc, runtime_bind)
|
|
949
|
+
with _DOCTOR_MEMO_LOCK:
|
|
950
|
+
_DOCTOR_MEMO.clear()
|
|
951
|
+
_DOCTOR_MEMO["payload"] = payload
|
|
952
|
+
_DOCTOR_MEMO["computed_at"] = now_utc
|
|
953
|
+
_DOCTOR_MEMO["runtime_bind"] = runtime_bind
|
|
954
|
+
return payload
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
def reset_doctor_memo() -> None:
|
|
958
|
+
"""Drop the memoized doctor payload (test hook + M5 invalidation entry)."""
|
|
959
|
+
with _DOCTOR_MEMO_LOCK:
|
|
960
|
+
_DOCTOR_MEMO.clear()
|
|
961
|
+
|
|
962
|
+
|
|
963
|
+
# === Task 5.1 — idle-path dispatch state (last signature + snapshot) ========
|
|
964
|
+
#
|
|
965
|
+
# The dashboard sync-thread rebuild computes the composite ``SnapshotSignature``
|
|
966
|
+
# at the top of every tick; when it is UNCHANGED versus the last published
|
|
967
|
+
# rebuild AND no wall-clock day/week/month boundary has rolled over, the rebuild
|
|
968
|
+
# takes the IDLE path (spec §3): it reuses the last published snapshot's heavy
|
|
969
|
+
# period/session rows and re-patches only the time-derived fields, skipping ALL
|
|
970
|
+
# re-aggregation — so an idle dashboard sits near 0% CPU. This module holds that
|
|
971
|
+
# last ``(signature, snapshot)`` pair.
|
|
972
|
+
#
|
|
973
|
+
# Sync-thread-only, single-writer — same discipline as the Group A / session
|
|
974
|
+
# caches (spec §7). The snapshot is stored as an OPAQUE object: this module
|
|
975
|
+
# never introspects it, keeping the "no dashboard/TUI import" design (the caller
|
|
976
|
+
# in bin/_cctally_tui.py owns the ``DataSnapshot`` type and all patching).
|
|
977
|
+
|
|
978
|
+
_LAST_DISPATCH_KEY: object = None
|
|
979
|
+
_LAST_PUBLISHED_SNAPSHOT: object = None
|
|
980
|
+
|
|
981
|
+
|
|
982
|
+
def dispatch_state() -> "tuple[object, object]":
|
|
983
|
+
"""Return the ``(last dispatch key, last published snapshot)`` pair (spec §3).
|
|
984
|
+
|
|
985
|
+
``(None, None)`` before the first rebuild or after a reset. The dispatch key
|
|
986
|
+
is the caller's opaque hashable — the composite ``SnapshotSignature`` bundled
|
|
987
|
+
with a render key (resolved display-tz + config) covering the render inputs
|
|
988
|
+
the DB signature does not; the caller compares it whole. The snapshot is an
|
|
989
|
+
opaque ``DataSnapshot`` reference the caller owns.
|
|
990
|
+
"""
|
|
991
|
+
return (_LAST_DISPATCH_KEY, _LAST_PUBLISHED_SNAPSHOT)
|
|
992
|
+
|
|
993
|
+
|
|
994
|
+
def store_dispatch_state(dispatch_key: object, snapshot: object) -> None:
|
|
995
|
+
"""Record the dispatch key + published snapshot for the next idle short-circuit.
|
|
996
|
+
|
|
997
|
+
Called once per rebuild (idle or full) so the next tick compares against the
|
|
998
|
+
key the just-published snapshot was built from.
|
|
999
|
+
"""
|
|
1000
|
+
global _LAST_DISPATCH_KEY, _LAST_PUBLISHED_SNAPSHOT
|
|
1001
|
+
_LAST_DISPATCH_KEY = dispatch_key
|
|
1002
|
+
_LAST_PUBLISHED_SNAPSHOT = snapshot
|
|
1003
|
+
|
|
1004
|
+
|
|
1005
|
+
def reset_dispatch_state() -> None:
|
|
1006
|
+
"""Drop the idle-path ``(key, snapshot)`` memo (test hook + isolation).
|
|
1007
|
+
|
|
1008
|
+
A fresh process starts with no memo; tests reset it between rebuilds so a
|
|
1009
|
+
prior test's leftover snapshot can't be idle-served under a matching key. Not
|
|
1010
|
+
part of the M5.2 prune invalidation — the generation bump (a signature leg)
|
|
1011
|
+
already forces the next rebuild off the idle path.
|
|
1012
|
+
"""
|
|
1013
|
+
global _LAST_DISPATCH_KEY, _LAST_PUBLISHED_SNAPSHOT
|
|
1014
|
+
_LAST_DISPATCH_KEY = None
|
|
1015
|
+
_LAST_PUBLISHED_SNAPSHOT = None
|
|
1016
|
+
|
|
1017
|
+
|
|
1018
|
+
# === #269 M0 — shared per-weekref immutable-cost cache (B1 trend + B3 forecast)
|
|
1019
|
+
#
|
|
1020
|
+
# A closed subscription week's cost is IMMUTABLE, so it is computed once and
|
|
1021
|
+
# reused until a signal invalidates it (spec §4). Both `build_trend_view`'s
|
|
1022
|
+
# reset-event weeks (`_compute_cost_for_weekref`) and forecast's trailing-4-week
|
|
1023
|
+
# fallback (`_select_dollars_per_percent` → `_sum_cost_for_range`) call the same
|
|
1024
|
+
# per-closed-week cost primitive from two sites; one shared cache keyed by the
|
|
1025
|
+
# week's `(week_start_at, week_end_at)` boundaries serves both. The OPEN (current)
|
|
1026
|
+
# week is never cached — it is decided per call from `week_end_at > now_utc`, so a
|
|
1027
|
+
# just-closed week caches on the next tick and the newly-opened week always
|
|
1028
|
+
# recomputes (no `_snapshot_period_rolled_over` dependence).
|
|
1029
|
+
#
|
|
1030
|
+
# Module-level state follows the Group A / session-cache precedent (spec §7): a
|
|
1031
|
+
# plain dict of immutable floats + a per-cache last-seen dict, mutated only on the
|
|
1032
|
+
# dashboard sync thread (single-writer). Every cached value is an immutable float;
|
|
1033
|
+
# each rebuild builds FRESH trend / forecast presentation objects and never
|
|
1034
|
+
# mutates a cached value (Codex F7).
|
|
1035
|
+
|
|
1036
|
+
_WEEKREF_COST_CACHE: dict = {} # {(week_start_iso, week_end_iso): cost_usd}
|
|
1037
|
+
_WEEKREF_COST_LAST_SEEN: dict = {} # {"max_id": int, "reset_sig": tuple}
|
|
1038
|
+
|
|
1039
|
+
|
|
1040
|
+
def _weekref_key(week_start_at, week_end_at):
|
|
1041
|
+
"""Canonical UTC-ISO key for a subscription week's cost.
|
|
1042
|
+
|
|
1043
|
+
Normalizes both boundaries to UTC before serializing, so two callers that
|
|
1044
|
+
resolve the same physical week in different tzinfos key identically.
|
|
1045
|
+
"""
|
|
1046
|
+
return (
|
|
1047
|
+
week_start_at.astimezone(dt.timezone.utc).isoformat(),
|
|
1048
|
+
week_end_at.astimezone(dt.timezone.utc).isoformat(),
|
|
1049
|
+
)
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
def reset_weekref_cost_state():
|
|
1053
|
+
"""Clear the weekref-cost cache + its watermark (full invalidation).
|
|
1054
|
+
|
|
1055
|
+
Called from the orphan-prune site (a prune deletes ``session_entries``
|
|
1056
|
+
possibly WITHOUT lowering ``MAX(id)``, so the reconcile's max-id-regression
|
|
1057
|
+
check cannot catch it — the explicit clear must) and as a test hook.
|
|
1058
|
+
"""
|
|
1059
|
+
_WEEKREF_COST_CACHE.clear()
|
|
1060
|
+
_WEEKREF_COST_LAST_SEEN.clear()
|
|
1061
|
+
|
|
1062
|
+
|
|
1063
|
+
def cached_weekref_cost(*, week_start_at, week_end_at, now_utc, compute):
|
|
1064
|
+
"""Get-or-compute a subscription week's cost (spec §4).
|
|
1065
|
+
|
|
1066
|
+
The OPEN week (``week_end_at > now_utc``) is always recomputed and never
|
|
1067
|
+
cached — open-vs-closed is decided per call so a just-closed week caches on
|
|
1068
|
+
the next tick and the newly-opened week always recomputes. A CLOSED week is
|
|
1069
|
+
served from ``_WEEKREF_COST_CACHE`` on a hit, else computed via ``compute``
|
|
1070
|
+
and stored. ``compute`` is the caller's from-scratch closure
|
|
1071
|
+
(``_compute_cost_for_weekref`` for B1, ``_sum_cost_for_range`` for B3), so
|
|
1072
|
+
the returned float is bit-identical to today's.
|
|
1073
|
+
"""
|
|
1074
|
+
if week_end_at > now_utc:
|
|
1075
|
+
return compute()
|
|
1076
|
+
key = _weekref_key(week_start_at, week_end_at)
|
|
1077
|
+
hit = _WEEKREF_COST_CACHE.get(key)
|
|
1078
|
+
if hit is not None: # 0.0 is a legitimate cached value, not a miss
|
|
1079
|
+
return hit
|
|
1080
|
+
val = compute()
|
|
1081
|
+
_WEEKREF_COST_CACHE[key] = val
|
|
1082
|
+
return val
|
|
1083
|
+
|
|
1084
|
+
|
|
1085
|
+
def reconcile_weekref_cache(cache_conn, *, max_entry_id, max_mutation_seq, reset_sig):
|
|
1086
|
+
"""Once-per-non-idle-rebuild invalidation for the weekref-cost cache (spec §4).
|
|
1087
|
+
|
|
1088
|
+
Driven by ``_tui_build_snapshot`` after the idle-path check, before the
|
|
1089
|
+
builders run, using the already-computed dispatch-signature legs
|
|
1090
|
+
(``max_entry_id`` + ``max_mutation_seq`` + ``reset_sig`` are passed in — no
|
|
1091
|
+
extra query for those):
|
|
1092
|
+
|
|
1093
|
+
- Cold (no last-seen): record last-seen, return — no eviction.
|
|
1094
|
+
- ``reset_sig`` changed OR ``max_entry_id < last_seen`` (cache.db rebuilt
|
|
1095
|
+
out-of-process) OR ``max_mutation_seq < last_seen_seq``: full ``clear()``.
|
|
1096
|
+
A credit/reset re-shapes a past week's cost; reset events are rare, so a
|
|
1097
|
+
conservative full clear is correct and cheap. A max-id / seq regression
|
|
1098
|
+
means the ids no longer map to the same rows.
|
|
1099
|
+
- ``max_mutation_seq > last_seen_seq`` (#270 §7c — the seq gate, so an
|
|
1100
|
+
id-stable in-place finalization with a flat ``max_entry_id`` still
|
|
1101
|
+
evicts): evict cached weeks whose ``week_end_at`` is
|
|
1102
|
+
``>= changed_min_timestamp(cache_conn, last_seen_seq)`` — a
|
|
1103
|
+
genuinely-changed row (new OR finalized-in-place) could fall inside them
|
|
1104
|
+
(F1 late-ingest / #270 in-place). The bound is ``>=``, NOT ``>``, because
|
|
1105
|
+
``_sum_cost_for_range`` / ``iter_entries`` sum an inclusive
|
|
1106
|
+
``[start, end]`` window, so a row whose timestamp lands exactly on
|
|
1107
|
+
``week_end_at`` belongs to that week (Codex-1). Over-eviction is byte-safe
|
|
1108
|
+
(forces a recompute). Normally ``wm`` is recent and no closed week drops.
|
|
1109
|
+
|
|
1110
|
+
Idempotent within a tick: after the first call updates last-seen, a later
|
|
1111
|
+
call in the same tick sees no delta (``max_mutation_seq == last_seen_seq``,
|
|
1112
|
+
``reset_sig`` unchanged) and no-ops — never re-running the watermark query.
|
|
1113
|
+
|
|
1114
|
+
Connection lifecycle (Codex-4): the ``changed_min_timestamp`` watermark query
|
|
1115
|
+
is the only use of ``cache_conn`` and runs only on the
|
|
1116
|
+
``max_mutation_seq > last_seen_seq`` branch; the caller passes a short-lived
|
|
1117
|
+
cache connection, opened for that query.
|
|
1118
|
+
"""
|
|
1119
|
+
ls = _WEEKREF_COST_LAST_SEEN
|
|
1120
|
+
if not ls: # cold
|
|
1121
|
+
ls["max_id"] = max_entry_id
|
|
1122
|
+
ls["max_seq"] = max_mutation_seq
|
|
1123
|
+
ls["reset_sig"] = reset_sig
|
|
1124
|
+
return
|
|
1125
|
+
if (
|
|
1126
|
+
reset_sig != ls["reset_sig"]
|
|
1127
|
+
or max_entry_id < ls["max_id"]
|
|
1128
|
+
or max_mutation_seq < ls["max_seq"]
|
|
1129
|
+
):
|
|
1130
|
+
_WEEKREF_COST_CACHE.clear() # reset/credit, or cache.db rebuilt
|
|
1131
|
+
ls["max_id"] = max_entry_id
|
|
1132
|
+
ls["max_seq"] = max_mutation_seq
|
|
1133
|
+
ls["reset_sig"] = reset_sig
|
|
1134
|
+
return
|
|
1135
|
+
if max_mutation_seq > ls["max_seq"]:
|
|
1136
|
+
wm = changed_min_timestamp(cache_conn, ls["max_seq"])
|
|
1137
|
+
if wm is not None:
|
|
1138
|
+
for key in list(_WEEKREF_COST_CACHE):
|
|
1139
|
+
# key = (start_iso, end_iso); inclusive [start,end] window, so
|
|
1140
|
+
# evict when the week's end >= the earliest changed event time.
|
|
1141
|
+
if dt.datetime.fromisoformat(key[1]) >= wm:
|
|
1142
|
+
del _WEEKREF_COST_CACHE[key]
|
|
1143
|
+
ls["max_id"] = max_entry_id
|
|
1144
|
+
ls["max_seq"] = max_mutation_seq
|
|
1145
|
+
ls["reset_sig"] = reset_sig
|
|
1146
|
+
|
|
1147
|
+
|
|
1148
|
+
# === #269 M4 — projects-envelope per-(project, week) incremental cache =======
|
|
1149
|
+
#
|
|
1150
|
+
# `_build_projects_envelope` re-iterates all ~190K window entries every warm
|
|
1151
|
+
# tick, doing a per-entry `_resolve_project_key` + cost + per-(project, week)
|
|
1152
|
+
# aggregation. At scale that builder DOMINATES the warm rebuild (spec §13). A
|
|
1153
|
+
# CLOSED week's per-project aggregate is IMMUTABLE, so cache it and recompute
|
|
1154
|
+
# only the CURRENT week each warm tick (spec §14 Win 2).
|
|
1155
|
+
#
|
|
1156
|
+
# Storage (opaque — this module never introspects the aggregate object, keeping
|
|
1157
|
+
# the "no dashboard/TUI import" design, exactly like `BucketCache`):
|
|
1158
|
+
# - `_PROJECTS_ENV_WEEK_CACHE`: `{(bucket_path, week_iso) -> agg}` per closed
|
|
1159
|
+
# week. The dashboard packs a `(cost_usd, sessions_count, first_seen,
|
|
1160
|
+
# last_seen, first_order, first_id, first_key)` record; here it is opaque.
|
|
1161
|
+
# - `_PROJECTS_ENV_WEEK_TOTALS`: `{week_iso -> total_cost}` cached as its OWN
|
|
1162
|
+
# entry-order aggregate (spec §14(a) — never re-summed from project buckets,
|
|
1163
|
+
# which would re-associate the float sum). ALSO the "week computed" registry:
|
|
1164
|
+
# a week is a cache HIT iff its `week_iso` is present here (an empty computed
|
|
1165
|
+
# week is stored with total 0.0 and no bucket rows, so cold empty weeks are
|
|
1166
|
+
# not re-queried every tick).
|
|
1167
|
+
# - `_PROJECTS_ENV_LAST_SEEN`: `{max_id, max_wus_id, sf_sig}` this cache last
|
|
1168
|
+
# reconciled against.
|
|
1169
|
+
#
|
|
1170
|
+
# Single-writer (sync thread only), immutable values, fresh presentation each
|
|
1171
|
+
# tick — the Group A / weekref discipline (spec §6, Codex F7).
|
|
1172
|
+
|
|
1173
|
+
_PROJECTS_ENV_WEEK_CACHE: dict = {} # {(bucket_path, week_iso): agg}
|
|
1174
|
+
_PROJECTS_ENV_WEEK_TOTALS: dict = {} # {week_iso: total_cost} (also the registry)
|
|
1175
|
+
_PROJECTS_ENV_LAST_SEEN: dict = {} # {"max_id", "max_wus_id", "sf_sig"}
|
|
1176
|
+
|
|
1177
|
+
# #271 M4 (spec §20): the CURRENT-week per-project aggregate is re-folded from
|
|
1178
|
+
# scratch every warm tick (the closed weeks are already cache-served). This
|
|
1179
|
+
# single-slot accumulator folds only the new-by-id delta each warm tick,
|
|
1180
|
+
# byte-identically — the exact Item-1 incremental single-left-fold, but per
|
|
1181
|
+
# project and simpler (the current-week window `[cw_start, cw_end]` is FIXED, not
|
|
1182
|
+
# `now`-clamped, so the delta predicate is purely `id > reconciled_max_id`; no
|
|
1183
|
+
# `last_now` machinery). Single-writer (sync thread only), module state, never
|
|
1184
|
+
# reachable from a published DataSnapshot (F7 — the snapshot holds the finalized
|
|
1185
|
+
# buckets, not the running `mut`). The `mut` is opaque here (the dashboard
|
|
1186
|
+
# packs/unpacks it — same "no dashboard import" discipline as `BucketCache`).
|
|
1187
|
+
_PROJECTS_ENV_CURRENT: dict = {"state": None} # single-slot current-week fold
|
|
1188
|
+
|
|
1189
|
+
|
|
1190
|
+
def projects_env_week_key(week_start):
|
|
1191
|
+
"""Canonical UTC-ISO key for a Monday-anchored subscription week start.
|
|
1192
|
+
|
|
1193
|
+
The dashboard resolves week starts as aware-UTC datetimes; normalizing to
|
|
1194
|
+
UTC before serializing keeps the key stable and parseable back by
|
|
1195
|
+
``reconcile_projects_env_cache`` (for the ``week_end`` watermark compare).
|
|
1196
|
+
"""
|
|
1197
|
+
return week_start.astimezone(dt.timezone.utc).isoformat()
|
|
1198
|
+
|
|
1199
|
+
|
|
1200
|
+
def reset_projects_env_state():
|
|
1201
|
+
"""Clear the projects-envelope week cache + totals + watermark + the
|
|
1202
|
+
CURRENT-week accumulator slot (#271 M4).
|
|
1203
|
+
|
|
1204
|
+
Called from the orphan-prune site (a prune deletes ``session_entries``
|
|
1205
|
+
possibly WITHOUT lowering ``MAX(id)``, so the reconcile's regression check
|
|
1206
|
+
cannot catch it — the explicit clear must) and as a test hook. The
|
|
1207
|
+
current-week slot rides this same clear (spec §20): a prune can re-key a
|
|
1208
|
+
current-week bucket_path, so it must cold-refold next tick.
|
|
1209
|
+
"""
|
|
1210
|
+
_PROJECTS_ENV_WEEK_CACHE.clear()
|
|
1211
|
+
_PROJECTS_ENV_WEEK_TOTALS.clear()
|
|
1212
|
+
_PROJECTS_ENV_LAST_SEEN.clear()
|
|
1213
|
+
reset_projects_env_current_state()
|
|
1214
|
+
|
|
1215
|
+
|
|
1216
|
+
def reset_projects_env_current_state():
|
|
1217
|
+
"""Drop the projects-envelope CURRENT-week accumulator slot (#271 §20).
|
|
1218
|
+
|
|
1219
|
+
A standalone hook (mirrors ``reset_group_a_current_state``); also invoked by
|
|
1220
|
+
``reset_projects_env_state`` (prune site + test reset) and by
|
|
1221
|
+
``reconcile_projects_env_cache``'s full-clear branch, so a project-key remap
|
|
1222
|
+
/ prune / cache.db rebuild cold-refolds the current week the same tick the
|
|
1223
|
+
closed-week cache clears.
|
|
1224
|
+
"""
|
|
1225
|
+
_PROJECTS_ENV_CURRENT["state"] = None
|
|
1226
|
+
|
|
1227
|
+
|
|
1228
|
+
def session_files_sig(cache_conn) -> "tuple[int, int]":
|
|
1229
|
+
"""`(COUNT(*), COALESCE(MAX(rowid), 0))` over ``session_files`` (Codex-M4 P2).
|
|
1230
|
+
|
|
1231
|
+
``sync_cache`` lazily backfills ``session_files.session_id`` /
|
|
1232
|
+
``project_path`` for OLD entries — moving a closed week's row from
|
|
1233
|
+
``(unknown)`` to a project, or changing a per-week session count — WITHOUT
|
|
1234
|
+
advancing ``MAX(session_entries.id)`` / ``MAX(weekly_usage_snapshots.id)``.
|
|
1235
|
+
So the envelope cache keys this cheap change-signal and full-clears when it
|
|
1236
|
+
moves. Returns ``(0, 0)`` on a missing table (fresh DB) so callers never
|
|
1237
|
+
raise.
|
|
1238
|
+
|
|
1239
|
+
#271 §9d rider (from the #269 final review): this ``(COUNT(*), MAX(rowid))``
|
|
1240
|
+
leg does NOT by itself catch the in-place ``ON CONFLICT(path) DO UPDATE SET
|
|
1241
|
+
project_path = COALESCE(...)`` attribution backfill — that UPDATE preserves
|
|
1242
|
+
the rowid and the row count, so both legs are unmoved. It is covered
|
|
1243
|
+
belt-and-suspenders, though: the backfill lands in the SAME ``sync_cache``
|
|
1244
|
+
ingest-loop iteration as the file's new ``session_entries`` rows, which bump
|
|
1245
|
+
``max_entry_id`` — caught by the watermark eviction path. So a pure
|
|
1246
|
+
attribution move never both slips this signal and leaves the cache stale.
|
|
1247
|
+
"""
|
|
1248
|
+
try:
|
|
1249
|
+
row = cache_conn.execute(
|
|
1250
|
+
"SELECT COUNT(*), COALESCE(MAX(rowid), 0) FROM session_files"
|
|
1251
|
+
).fetchone()
|
|
1252
|
+
return (int(row[0]), int(row[1]))
|
|
1253
|
+
except sqlite3.Error:
|
|
1254
|
+
return (0, 0)
|
|
1255
|
+
|
|
1256
|
+
|
|
1257
|
+
def projects_env_week_get(week_iso):
|
|
1258
|
+
"""Return ``(buckets_by_bucket_path, total_cost)`` for a cached week, or
|
|
1259
|
+
``None`` on a miss.
|
|
1260
|
+
|
|
1261
|
+
``_PROJECTS_ENV_WEEK_TOTALS`` is the presence registry: a week is a HIT iff
|
|
1262
|
+
its key is present (an empty computed week is stored with total 0.0 and no
|
|
1263
|
+
bucket rows, so it is a HIT with an empty bucket map). The bucket rows are
|
|
1264
|
+
reassembled by scanning ``_PROJECTS_ENV_WEEK_CACHE`` for this week's keys —
|
|
1265
|
+
the distinct-bucket_path count is small (project count), so the scan is
|
|
1266
|
+
cheap.
|
|
1267
|
+
"""
|
|
1268
|
+
if week_iso not in _PROJECTS_ENV_WEEK_TOTALS:
|
|
1269
|
+
return None
|
|
1270
|
+
total = _PROJECTS_ENV_WEEK_TOTALS[week_iso]
|
|
1271
|
+
by_bp = {
|
|
1272
|
+
bp: agg
|
|
1273
|
+
for (bp, wk), agg in _PROJECTS_ENV_WEEK_CACHE.items()
|
|
1274
|
+
if wk == week_iso
|
|
1275
|
+
}
|
|
1276
|
+
return by_bp, total
|
|
1277
|
+
|
|
1278
|
+
|
|
1279
|
+
def projects_env_week_put(week_iso, buckets_by_bp, total) -> None:
|
|
1280
|
+
"""Store one CLOSED week's per-bucket aggregates + entry-order total.
|
|
1281
|
+
|
|
1282
|
+
``total`` is registered even when ``buckets_by_bp`` is empty, so a
|
|
1283
|
+
computed-empty week is a HIT (not a perpetual miss). Values are stored
|
|
1284
|
+
as-is (immutable); this never mutates a stored aggregate in place.
|
|
1285
|
+
"""
|
|
1286
|
+
_PROJECTS_ENV_WEEK_TOTALS[week_iso] = total
|
|
1287
|
+
for bp, agg in buckets_by_bp.items():
|
|
1288
|
+
_PROJECTS_ENV_WEEK_CACHE[(bp, week_iso)] = agg
|
|
1289
|
+
|
|
1290
|
+
|
|
1291
|
+
def accumulate_projects_current_week(*, week_key, cur_max_id, cur_max_seq,
|
|
1292
|
+
fetch_all_raw, fetch_delta_rows,
|
|
1293
|
+
finalize, fold):
|
|
1294
|
+
"""Single-slot incremental fold of the projects-envelope CURRENT week (#271 §20).
|
|
1295
|
+
|
|
1296
|
+
Returns ``(finalized_buckets, week_total)`` — the exact
|
|
1297
|
+
``_aggregate_projects_week`` public shape — folding only the changed-row delta
|
|
1298
|
+
each warm tick instead of re-folding the whole ~12K-entry current week
|
|
1299
|
+
(the closed weeks are already cache-served). The dashboard injects the
|
|
1300
|
+
fold/finalize/fetch callables so this module keeps no dashboard import
|
|
1301
|
+
(the ``BucketCache`` / ``build_cached_group_a`` discipline).
|
|
1302
|
+
|
|
1303
|
+
Injected closures (all capture the current-week window ``[cw_start, cw_end]``
|
|
1304
|
+
and the live conn on the dashboard side):
|
|
1305
|
+
``fetch_all_raw() -> (mut_with_sessions_sets, week_total, tail)`` — the
|
|
1306
|
+
full-window raw fold (cold seed / fold-order fallback).
|
|
1307
|
+
``fetch_delta_rows(after_seq) -> list[row]`` — rows with
|
|
1308
|
+
``mutation_seq > after_seq`` in the current-week window (#270 §8 re-key),
|
|
1309
|
+
already membership/``<synthetic>``-filtered AND sorted
|
|
1310
|
+
``(timestamp_utc, id)``; ``row[0]`` is the ``id``, ``row[1]`` the
|
|
1311
|
+
``timestamp_utc``, so ``rows[0]`` is the minimum genuine current-week entry.
|
|
1312
|
+
``fold(mut, row) -> entry_cost | None`` — the shared ``_fold_projects_entry``
|
|
1313
|
+
(byte-identical arithmetic to the cold path).
|
|
1314
|
+
``finalize(mut) -> {bucket_path: agg}`` — the public finalize.
|
|
1315
|
+
|
|
1316
|
+
Simpler than the Group A accumulator (spec §20): the current-week aggregate
|
|
1317
|
+
is a pure function of ``(conn, cw_start)`` over the FIXED ``[cw_start, cw_end]``
|
|
1318
|
+
window (no moving-``now`` clamp), so there is no ``last_now`` field / empty-
|
|
1319
|
+
delta-by-time machinery.
|
|
1320
|
+
|
|
1321
|
+
#270 §8: the delta is re-keyed from ``id > reconciled_max_id`` to
|
|
1322
|
+
``mutation_seq > reconciled_max_seq`` — a strict superset (every insert
|
|
1323
|
+
carries a fresh seq monotone with id, so on a pure-insert tick the delta row
|
|
1324
|
+
set is byte-identical to today), so an id-stable in-place finalization of a
|
|
1325
|
+
current-week row now appears in the delta. But the incremental fold cannot
|
|
1326
|
+
un-fold that row's already-folded stale partial: any delta row that is a
|
|
1327
|
+
PRE-EXISTING row (``id <= reconciled_max_id``) discards the delta and forces a
|
|
1328
|
+
cold refold (checked BEFORE the fold-order gate, so a timestamp-non-decreasing
|
|
1329
|
+
finalization that sorts past ``tail`` is still caught). Genuine new inserts
|
|
1330
|
+
(``id > reconciled_max_id``) proceed through the unchanged fold-order gate +
|
|
1331
|
+
append.
|
|
1332
|
+
|
|
1333
|
+
- **Cold** (``prior is None``, ``label`` changed = Monday rollover / window
|
|
1334
|
+
slide, or ``cur_max_id < reconciled_max_id`` = cache.db rebuilt): seed the
|
|
1335
|
+
slot from ``fetch_all_raw()``; finalize and return.
|
|
1336
|
+
- **Warm**: fetch the ``mutation_seq > reconciled_max_seq`` delta (empty when
|
|
1337
|
+
``cur_max_seq`` did not advance — the fast path, no fetch). **Pre-existing-row
|
|
1338
|
+
cold-refold (#270 §8):** any delta ``row`` with ``row[0] <= reconciled_max_id``
|
|
1339
|
+
→ discard + cold refold. **Fold-order gate:** the delta is ``(ts_iso, id)``-
|
|
1340
|
+
sorted, so ``rows[0]`` is the minimum; if it sorts ``(ts_iso, id) <= tail`` a
|
|
1341
|
+
late older-timestamp backfill landed mid-week → discard the delta and cold-
|
|
1342
|
+
refold (byte-safe; first-row-only is sufficient under the total order).
|
|
1343
|
+
``tail is None`` ⇒ prior folded nothing ⇒ the delta is a pure suffix, so no
|
|
1344
|
+
gate. Otherwise ``fold`` each delta row onto ``prior.mut`` in order.
|
|
1345
|
+
- ``reconciled_max_id`` / ``reconciled_max_seq`` advance to
|
|
1346
|
+
``cur_max_id`` / ``cur_max_seq`` on EVERY path (cold, empty delta, append,
|
|
1347
|
+
fallback). They are the tick's GLOBAL ``MAX(session_entries.id)`` /
|
|
1348
|
+
``MAX(mutation_seq)``, DECOUPLED from ``tail`` (Codex-P2a): a quiet current
|
|
1349
|
+
week has a folded-max far below the global max because high ids/seqs land in
|
|
1350
|
+
OTHER weeks (recent backfills), so keying the delta on the folded-max would
|
|
1351
|
+
re-scan every tick and let the ~63ms floor creep back.
|
|
1352
|
+
|
|
1353
|
+
F7: the running ``mut`` is module state, never reachable from a published
|
|
1354
|
+
snapshot — ``finalize`` builds a FRESH bucket map each tick, so mutating
|
|
1355
|
+
``mut`` next tick cannot tear a prior snapshot.
|
|
1356
|
+
"""
|
|
1357
|
+
prior = _PROJECTS_ENV_CURRENT["state"]
|
|
1358
|
+
cold = (
|
|
1359
|
+
prior is None
|
|
1360
|
+
or prior["label"] != week_key
|
|
1361
|
+
or cur_max_id < prior["reconciled_max_id"]
|
|
1362
|
+
)
|
|
1363
|
+
rows: list = []
|
|
1364
|
+
if not cold:
|
|
1365
|
+
if cur_max_seq > prior["reconciled_max_seq"]:
|
|
1366
|
+
rows = fetch_delta_rows(prior["reconciled_max_seq"])
|
|
1367
|
+
# #270 §8: any PRE-EXISTING row (id <= reconciled_max_id) is an id-stable
|
|
1368
|
+
# in-place finalization the incremental fold cannot reconcile → cold
|
|
1369
|
+
# refold. Checked BEFORE the fold-order gate (a timestamp-non-decreasing
|
|
1370
|
+
# finalization sorts PAST tail and would evade that gate).
|
|
1371
|
+
if rows and any(r[0] <= prior["reconciled_max_id"] for r in rows):
|
|
1372
|
+
cold = True
|
|
1373
|
+
# Fold-order gate: rows are (ts_iso, id)-sorted so rows[0] is the
|
|
1374
|
+
# minimum; if it sorts <= tail, some genuinely-new row is out of the
|
|
1375
|
+
# from-scratch fold order → cold refold. (tail None ⇒ prior folded
|
|
1376
|
+
# nothing ⇒ the delta is a pure suffix; never gate against None.)
|
|
1377
|
+
elif (
|
|
1378
|
+
rows
|
|
1379
|
+
and prior["tail"] is not None
|
|
1380
|
+
and (rows[0][1], rows[0][0]) <= prior["tail"]
|
|
1381
|
+
):
|
|
1382
|
+
cold = True
|
|
1383
|
+
if cold:
|
|
1384
|
+
mut, week_total, tail = fetch_all_raw()
|
|
1385
|
+
_PROJECTS_ENV_CURRENT["state"] = {
|
|
1386
|
+
"label": week_key,
|
|
1387
|
+
"mut": mut,
|
|
1388
|
+
"week_total": week_total,
|
|
1389
|
+
"tail": tail,
|
|
1390
|
+
"reconciled_max_id": cur_max_id,
|
|
1391
|
+
"reconciled_max_seq": cur_max_seq,
|
|
1392
|
+
}
|
|
1393
|
+
return finalize(mut), week_total
|
|
1394
|
+
# Warm: append the delta onto the running mut in (ts_iso, id) order. The
|
|
1395
|
+
# week_total is its OWN entry-order accumulator (never re-summed from the
|
|
1396
|
+
# per-bucket costs — spec §14(a) non-association rule).
|
|
1397
|
+
mut = prior["mut"]
|
|
1398
|
+
week_total = prior["week_total"]
|
|
1399
|
+
tail = prior["tail"]
|
|
1400
|
+
for row in rows:
|
|
1401
|
+
entry_cost = fold(mut, row)
|
|
1402
|
+
if entry_cost is None: # defensive: fetch_delta_rows pre-filters
|
|
1403
|
+
continue
|
|
1404
|
+
week_total += entry_cost
|
|
1405
|
+
tail = (row[1], row[0])
|
|
1406
|
+
prior["week_total"] = week_total
|
|
1407
|
+
prior["tail"] = tail
|
|
1408
|
+
prior["reconciled_max_id"] = cur_max_id
|
|
1409
|
+
prior["reconciled_max_seq"] = cur_max_seq
|
|
1410
|
+
return finalize(mut), week_total
|
|
1411
|
+
|
|
1412
|
+
|
|
1413
|
+
def reconcile_projects_env_cache(cache_conn, *, max_entry_id, max_mutation_seq,
|
|
1414
|
+
max_wus_id, sf_sig):
|
|
1415
|
+
"""Once-per-non-idle-rebuild invalidation for the projects-envelope cache.
|
|
1416
|
+
|
|
1417
|
+
Driven by ``_tui_build_snapshot`` after the idle-path check, before the
|
|
1418
|
+
envelope builder runs, using the dispatch-signature legs (``max_entry_id``,
|
|
1419
|
+
``max_mutation_seq``, ``max_wus_id``) + a ``session_files`` signal
|
|
1420
|
+
(``sf_sig``) passed in:
|
|
1421
|
+
|
|
1422
|
+
- Cold (no last-seen): record last-seen, return — no eviction.
|
|
1423
|
+
- ``sf_sig`` changed (attribution backfill, Codex-M4 P2) OR
|
|
1424
|
+
``max_entry_id < last_seen`` (cache.db rebuilt) OR
|
|
1425
|
+
``max_mutation_seq < last_seen_seq``: full clear. Conservative but
|
|
1426
|
+
byte-safe (recompute). ``max_wus_id`` stays tracked in last-seen
|
|
1427
|
+
but is deliberately NOT a full-clear trigger (#271 §9) — a `record-usage`
|
|
1428
|
+
write reuses this cost cache; the whole-envelope memo refreshes attribution.
|
|
1429
|
+
- ``max_mutation_seq > last_seen_seq`` (#270 §7c — the seq gate, so an
|
|
1430
|
+
id-stable in-place finalization with a flat ``max_entry_id`` still evicts):
|
|
1431
|
+
evict cached weeks (and their bucket rows + week total) whose
|
|
1432
|
+
``week_end (= parse(week_iso) + 7d)`` is
|
|
1433
|
+
``>= changed_min_timestamp(cache_conn, last_seen_seq)`` — a
|
|
1434
|
+
genuinely-changed row could fall inside them (F1 late-ingest / #270
|
|
1435
|
+
in-place). The bound is ``>=`` (Codex-1); over-eviction is byte-safe.
|
|
1436
|
+
|
|
1437
|
+
Idempotent within a tick: after the first call updates last-seen, a later
|
|
1438
|
+
call with the same signature sees no delta and no-ops (never re-running the
|
|
1439
|
+
watermark query). The short-lived ``cache_conn`` is used only for the
|
|
1440
|
+
watermark query on the ``max_mutation_seq > last_seen_seq`` branch (Codex-4).
|
|
1441
|
+
"""
|
|
1442
|
+
ls = _PROJECTS_ENV_LAST_SEEN
|
|
1443
|
+
if not ls: # cold
|
|
1444
|
+
ls["max_id"] = max_entry_id
|
|
1445
|
+
ls["max_seq"] = max_mutation_seq
|
|
1446
|
+
ls["max_wus_id"] = max_wus_id
|
|
1447
|
+
ls["sf_sig"] = sf_sig
|
|
1448
|
+
return
|
|
1449
|
+
if (
|
|
1450
|
+
sf_sig != ls["sf_sig"]
|
|
1451
|
+
or max_entry_id < ls["max_id"]
|
|
1452
|
+
or max_mutation_seq < ls["max_seq"]
|
|
1453
|
+
):
|
|
1454
|
+
# NOTE (#271 §9): max_wus_id is deliberately NOT a full-clear trigger. The
|
|
1455
|
+
# cached per-(project, week) aggregates are session_entries-only; a WUS
|
|
1456
|
+
# bump (a `record-usage` write) changes only the attribution denominator,
|
|
1457
|
+
# which the whole-envelope memo (_PROJECTS_ENV_MEMO, still keyed on
|
|
1458
|
+
# max_wus_id) recomputes fresh on its own miss. Reusing this cost cache
|
|
1459
|
+
# across a WUS bump is byte-identical. Do NOT re-add
|
|
1460
|
+
# `max_wus_id != ls["max_wus_id"]` here.
|
|
1461
|
+
_PROJECTS_ENV_WEEK_CACHE.clear()
|
|
1462
|
+
_PROJECTS_ENV_WEEK_TOTALS.clear()
|
|
1463
|
+
# #271 M4 (spec §20): the current-week accumulator rides the SAME
|
|
1464
|
+
# full-clear signals (sf_sig attribution remap / max_entry_id regression)
|
|
1465
|
+
# — an sf_sig move can re-key a current-week bucket_path, so cold-refold.
|
|
1466
|
+
reset_projects_env_current_state()
|
|
1467
|
+
ls["max_id"] = max_entry_id
|
|
1468
|
+
ls["max_seq"] = max_mutation_seq
|
|
1469
|
+
ls["max_wus_id"] = max_wus_id
|
|
1470
|
+
ls["sf_sig"] = sf_sig
|
|
1471
|
+
return
|
|
1472
|
+
if max_mutation_seq > ls["max_seq"]:
|
|
1473
|
+
wm = changed_min_timestamp(cache_conn, ls["max_seq"])
|
|
1474
|
+
if wm is not None:
|
|
1475
|
+
dirty = [
|
|
1476
|
+
wk
|
|
1477
|
+
for wk in list(_PROJECTS_ENV_WEEK_TOTALS)
|
|
1478
|
+
if dt.datetime.fromisoformat(wk) + dt.timedelta(days=7) >= wm
|
|
1479
|
+
]
|
|
1480
|
+
for wk in dirty:
|
|
1481
|
+
_PROJECTS_ENV_WEEK_TOTALS.pop(wk, None)
|
|
1482
|
+
for key in [k for k in _PROJECTS_ENV_WEEK_CACHE if k[1] == wk]:
|
|
1483
|
+
del _PROJECTS_ENV_WEEK_CACHE[key]
|
|
1484
|
+
ls["max_id"] = max_entry_id
|
|
1485
|
+
ls["max_seq"] = max_mutation_seq
|
|
1486
|
+
ls["max_wus_id"] = max_wus_id
|
|
1487
|
+
ls["sf_sig"] = sf_sig
|
|
1488
|
+
|
|
1489
|
+
|
|
1490
|
+
# === #271 M3 — Bug-K pre-credit segment cache (spec §18) =====================
|
|
1491
|
+
#
|
|
1492
|
+
# The dashboard Weekly panel's Bug-K synthesis (`_dashboard_build_weekly_periods`)
|
|
1493
|
+
# rebuilds, per in-place credit event, the pre-credit aggregate over the CLOSED
|
|
1494
|
+
# past interval `[original_start, effective)` on EVERY warm tick — a wide
|
|
1495
|
+
# `get_entries` re-fetch + re-cost (~100ms wall / 5 windows / ~42.6K entries on
|
|
1496
|
+
# the 314K-entry prod copy). Because `effective` is a historical credit moment
|
|
1497
|
+
# (always <= now), that aggregate is IMMUTABLE — the SAME "re-aggregate immutable
|
|
1498
|
+
# history every tick" pattern the #269 weekref cost cache (above, §4) fixed. Cache
|
|
1499
|
+
# it and recompute only when a genuine data change reaches the window.
|
|
1500
|
+
#
|
|
1501
|
+
# Module-level state mirrors the weekref cost cache exactly: a plain dict of
|
|
1502
|
+
# immutable `BugKSegment` values + a per-cache last-seen dict, mutated only on the
|
|
1503
|
+
# dashboard sync thread (single-writer). Every cached value is immutable and each
|
|
1504
|
+
# rebuild builds a FRESH `WeeklyPeriodRow` from the cached segment, never mutating
|
|
1505
|
+
# it (F7). The `models` payload is a FROZEN TUPLE of (model, cost) in first-seen
|
|
1506
|
+
# order (NOT a live dict — Codex-BK-4), so the row's stable cost-desc sort
|
|
1507
|
+
# tie-order (which depends on first-seen order) can never be mutated after caching.
|
|
1508
|
+
#
|
|
1509
|
+
# Id-stable in-place mutation (Codex-BK-1) — RESOLVED by #270. `sync_cache`'s
|
|
1510
|
+
# `ON CONFLICT(msg_id, req_id) DO UPDATE` can finalize a streaming-intermediate
|
|
1511
|
+
# entry in place (same `id`, changed tokens/cost/timestamp) inside a closed
|
|
1512
|
+
# pre-credit window, advancing NEITHER `max_entry_id` NOR the old
|
|
1513
|
+
# `new_min_timestamp` watermark → a stale segment. This WAS the same exposure the
|
|
1514
|
+
# weekref cost cache and the Group A past-bucket caches carried (spec §6 /
|
|
1515
|
+
# first-review Codex-2). #270 closed it across every #268/#269/#271 cache at once
|
|
1516
|
+
# with the durable `session_entries.mutation_seq` change stamp: it is folded into
|
|
1517
|
+
# `compute_signature` (the `entry_mutation_seq` leg, so an id-stable finalization
|
|
1518
|
+
# leaves the idle path) AND drives `reconcile_bugk_cache`'s seq gate + the
|
|
1519
|
+
# `changed_min_timestamp(mutation_min_ts)` watermark below (so the affected CLOSED
|
|
1520
|
+
# segment — including one the finalization's timestamp MOVED the row across — is
|
|
1521
|
+
# recomputed). Regression: `test_reconcile_bugk_idstable_update_evicts`.
|
|
1522
|
+
#
|
|
1523
|
+
# One accepted trade-off remains, identical to the shipped #269 weekref cost cache
|
|
1524
|
+
# (deliberately NOT closed — closing it would be inconsistent with the already-
|
|
1525
|
+
# shipped caches):
|
|
1526
|
+
# - Pricing-at-cache-time (Codex-BK-3). Caching the folded `pre_cost` means an
|
|
1527
|
+
# in-process embedded-pricing edit is not reflected in the Bug-K pre-credit cost
|
|
1528
|
+
# until the segment invalidates — the SAME dashboard-only trade-off the weekref
|
|
1529
|
+
# cost cache already makes (see its module note above). Acceptable because
|
|
1530
|
+
# embedded-pricing edits require a code change + process restart regardless.
|
|
1531
|
+
|
|
1532
|
+
|
|
1533
|
+
class BugKSegment(NamedTuple):
|
|
1534
|
+
"""Immutable folded pre-credit aggregate over a closed `[original_start,
|
|
1535
|
+
effective)` window (spec §18).
|
|
1536
|
+
|
|
1537
|
+
``models`` is a frozen tuple of ``(display_model, cost)`` in FIRST-SEEN order
|
|
1538
|
+
(Codex-BK-4); the caller re-derives the cost-desc-sorted ``model_breakdowns``
|
|
1539
|
+
fresh each tick from it, and the stable sort preserves that first-seen tie
|
|
1540
|
+
order byte-for-byte.
|
|
1541
|
+
"""
|
|
1542
|
+
|
|
1543
|
+
input: int
|
|
1544
|
+
output: int
|
|
1545
|
+
cache_create: int
|
|
1546
|
+
cache_read: int
|
|
1547
|
+
cost: float
|
|
1548
|
+
models: tuple # ((display_model, cost), ...) in first-seen order
|
|
1549
|
+
entry_count: int
|
|
1550
|
+
|
|
1551
|
+
|
|
1552
|
+
_BUGK_SEGMENT_CACHE: dict = {} # {(orig_start_utc_iso, eff_utc_iso): BugKSegment}
|
|
1553
|
+
_BUGK_SEGMENT_LAST_SEEN: dict = {} # {"max_id": int, "reset_sig": tuple}
|
|
1554
|
+
|
|
1555
|
+
|
|
1556
|
+
def _bugk_key(original_start_at, effective_at):
|
|
1557
|
+
"""Canonical UTC-ISO key for a pre-credit segment window (spec §18).
|
|
1558
|
+
|
|
1559
|
+
Normalizes both boundaries to UTC before serializing (exactly like
|
|
1560
|
+
``_weekref_key``), so two spellings of one window can't create duplicate
|
|
1561
|
+
entries. The RAW ISO strings are still used for the row output — this key is
|
|
1562
|
+
cache identity only.
|
|
1563
|
+
"""
|
|
1564
|
+
return (
|
|
1565
|
+
original_start_at.astimezone(dt.timezone.utc).isoformat(),
|
|
1566
|
+
effective_at.astimezone(dt.timezone.utc).isoformat(),
|
|
1567
|
+
)
|
|
1568
|
+
|
|
1569
|
+
|
|
1570
|
+
def reset_bugk_segment_state():
|
|
1571
|
+
"""Clear the Bug-K segment cache + its watermark (full invalidation).
|
|
1572
|
+
|
|
1573
|
+
Called from the orphan-prune site (a prune deletes ``session_entries``
|
|
1574
|
+
possibly WITHOUT lowering ``MAX(id)``, so the reconcile's max-id-regression
|
|
1575
|
+
check cannot catch it — the explicit clear must) and as a test hook.
|
|
1576
|
+
"""
|
|
1577
|
+
_BUGK_SEGMENT_CACHE.clear()
|
|
1578
|
+
_BUGK_SEGMENT_LAST_SEEN.clear()
|
|
1579
|
+
|
|
1580
|
+
|
|
1581
|
+
def cached_bugk_segment(*, key, compute):
|
|
1582
|
+
"""Get-or-compute one pre-credit segment aggregate (spec §18).
|
|
1583
|
+
|
|
1584
|
+
The ``[original_start, effective)`` window is ALWAYS a closed past interval
|
|
1585
|
+
(``effective`` is a historical credit moment), so it is always cacheable: a
|
|
1586
|
+
cache hit returns the stored ``BugKSegment``; a miss calls ``compute()`` — the
|
|
1587
|
+
caller's exact from-scratch fetch+fold closure, a ``(timestamp_utc, id)``-
|
|
1588
|
+
ordered fetch so the left-fold ``cost`` / first-seen ``models`` order is
|
|
1589
|
+
bit-identical to today's — and stores it. ``key`` is the canonical
|
|
1590
|
+
``_bugk_key``.
|
|
1591
|
+
"""
|
|
1592
|
+
hit = _BUGK_SEGMENT_CACHE.get(key)
|
|
1593
|
+
if hit is not None:
|
|
1594
|
+
return hit
|
|
1595
|
+
val = compute()
|
|
1596
|
+
_BUGK_SEGMENT_CACHE[key] = val
|
|
1597
|
+
return val
|
|
1598
|
+
|
|
1599
|
+
|
|
1600
|
+
def reconcile_bugk_cache(cache_conn, *, max_entry_id, max_mutation_seq, reset_sig):
|
|
1601
|
+
"""Once-per-non-idle-rebuild invalidation for the Bug-K segment cache (§18).
|
|
1602
|
+
|
|
1603
|
+
Driven by ``_tui_build_snapshot`` after the idle-path check, before the
|
|
1604
|
+
builders run, ALONGSIDE ``reconcile_weekref_cache``, using the dispatch-
|
|
1605
|
+
signature legs already computed for the idle decision (``max_entry_id`` +
|
|
1606
|
+
``max_mutation_seq`` + ``reset_sig`` passed in — no extra query for those):
|
|
1607
|
+
|
|
1608
|
+
- Cold (no last-seen): record last-seen, return — no eviction.
|
|
1609
|
+
- ``reset_sig`` changed (credit events / their ``effective`` moments moved)
|
|
1610
|
+
OR ``max_entry_id < last_seen`` (cache.db rebuilt out-of-process) OR
|
|
1611
|
+
``max_mutation_seq < last_seen_seq``: full ``clear()``. Credit events are
|
|
1612
|
+
rare, so a conservative full clear is correct and cheap; a max-id / seq
|
|
1613
|
+
regression means the ids no longer map to the same rows.
|
|
1614
|
+
- ``max_mutation_seq > last_seen_seq`` (#270 §7c — the seq gate, so an
|
|
1615
|
+
id-stable in-place finalization with a flat ``max_entry_id`` still evicts):
|
|
1616
|
+
evict segments whose ``effective`` is
|
|
1617
|
+
``> changed_min_timestamp(cache_conn, last_seen_seq)`` — a genuinely-changed
|
|
1618
|
+
row could fall inside them (F1 late-ingest / #270 in-place). The bound is
|
|
1619
|
+
``>`` (STRICT), NOT ``>=``, because the segment window is HALF-OPEN
|
|
1620
|
+
``[original_start, effective)`` (Codex-BK-5): a row EXACTLY at ``effective``
|
|
1621
|
+
is OUTSIDE the segment (never contributes), while a row at
|
|
1622
|
+
``original_start`` .. just-below ``effective`` evicts. This is the ONE
|
|
1623
|
+
semantic difference from the weekref cache's inclusive ``>=`` (that window
|
|
1624
|
+
is ``[start, end]``). Over-eviction is byte-safe (forces a recompute);
|
|
1625
|
+
normally ``wm`` is recent and nothing drops.
|
|
1626
|
+
|
|
1627
|
+
Idempotent within a tick: after the first call updates last-seen, a later call
|
|
1628
|
+
with the same signature sees no delta and no-ops (never re-running the
|
|
1629
|
+
watermark query). The short-lived ``cache_conn`` is used only for the
|
|
1630
|
+
``changed_min_timestamp`` query on the ``max_mutation_seq > last_seen_seq``
|
|
1631
|
+
branch (Codex-4 lifecycle from §4).
|
|
1632
|
+
"""
|
|
1633
|
+
ls = _BUGK_SEGMENT_LAST_SEEN
|
|
1634
|
+
if not ls: # cold
|
|
1635
|
+
ls["max_id"] = max_entry_id
|
|
1636
|
+
ls["max_seq"] = max_mutation_seq
|
|
1637
|
+
ls["reset_sig"] = reset_sig
|
|
1638
|
+
return
|
|
1639
|
+
if (
|
|
1640
|
+
reset_sig != ls["reset_sig"]
|
|
1641
|
+
or max_entry_id < ls["max_id"]
|
|
1642
|
+
or max_mutation_seq < ls["max_seq"]
|
|
1643
|
+
):
|
|
1644
|
+
_BUGK_SEGMENT_CACHE.clear() # a credit event moved, or cache.db rebuilt
|
|
1645
|
+
ls["max_id"] = max_entry_id
|
|
1646
|
+
ls["max_seq"] = max_mutation_seq
|
|
1647
|
+
ls["reset_sig"] = reset_sig
|
|
1648
|
+
return
|
|
1649
|
+
if max_mutation_seq > ls["max_seq"]:
|
|
1650
|
+
wm = changed_min_timestamp(cache_conn, ls["max_seq"])
|
|
1651
|
+
if wm is not None:
|
|
1652
|
+
for key in list(_BUGK_SEGMENT_CACHE):
|
|
1653
|
+
# key = (orig_start_iso, eff_iso); half-open [start, eff) window,
|
|
1654
|
+
# so evict when eff > the earliest changed event time (a row AT
|
|
1655
|
+
# eff is outside the segment; a row < eff could be inside it).
|
|
1656
|
+
if dt.datetime.fromisoformat(key[1]) > wm:
|
|
1657
|
+
del _BUGK_SEGMENT_CACHE[key]
|
|
1658
|
+
ls["max_id"] = max_entry_id
|
|
1659
|
+
ls["max_seq"] = max_mutation_seq
|
|
1660
|
+
ls["reset_sig"] = reset_sig
|
|
1661
|
+
|
|
1662
|
+
|
|
1663
|
+
# === #272 — cache-report per-day cache =====================================
|
|
1664
|
+
#
|
|
1665
|
+
# A per-day cache in front of the dashboard's `build_cache_report_snapshot`
|
|
1666
|
+
# warm-tick leg. Each closed day's raw aggregate + per-project partials is
|
|
1667
|
+
# an immutable `CachedCacheReportDay` (bin/_lib_cache_report.py §5); the
|
|
1668
|
+
# reconcile below invalidates it on the #270/#271 `mutation_seq` / `max_id`
|
|
1669
|
+
# / `reset_sig` / `sf_sig` / display-tz signals, mirroring
|
|
1670
|
+
# `reconcile_bugk_cache` + the projects-env `session_files_sig` leg.
|
|
1671
|
+
|
|
1672
|
+
_CACHE_REPORT_DAY_CACHE: dict = {} # {date_key: CachedCacheReportDay}
|
|
1673
|
+
_CACHE_REPORT_LAST_SEEN: dict = {} # {max_id, max_seq, reset_sig, sf_sig, tz_key}
|
|
1674
|
+
|
|
1675
|
+
|
|
1676
|
+
def reset_cache_report_state():
|
|
1677
|
+
"""Clear the cache-report per-day cache + its watermark (full invalidation).
|
|
1678
|
+
|
|
1679
|
+
Called from the orphan-prune site (a prune can delete ``session_entries``
|
|
1680
|
+
WITHOUT lowering ``MAX(id)`` / advancing ``mutation_seq``, which the
|
|
1681
|
+
reconcile's regression check cannot catch — the explicit clear must) and
|
|
1682
|
+
as a test hook. Mirrors ``reset_bugk_segment_state`` /
|
|
1683
|
+
``reset_projects_env_state``.
|
|
1684
|
+
"""
|
|
1685
|
+
_CACHE_REPORT_DAY_CACHE.clear()
|
|
1686
|
+
_CACHE_REPORT_LAST_SEEN.clear()
|
|
1687
|
+
|
|
1688
|
+
|
|
1689
|
+
def cache_report_day_get(date_key):
|
|
1690
|
+
"""Return the cached ``CachedCacheReportDay`` for ``date_key``, or ``None``."""
|
|
1691
|
+
return _CACHE_REPORT_DAY_CACHE.get(date_key)
|
|
1692
|
+
|
|
1693
|
+
|
|
1694
|
+
def cache_report_day_store(date_key, value) -> None:
|
|
1695
|
+
"""Store one CLOSED day's immutable ``CachedCacheReportDay`` (never mutated)."""
|
|
1696
|
+
_CACHE_REPORT_DAY_CACHE[date_key] = value
|
|
1697
|
+
|
|
1698
|
+
|
|
1699
|
+
def cache_report_day_evict_before(oldest_key) -> None:
|
|
1700
|
+
"""Drop cached days strictly older than ``oldest_key`` — the window-rolloff
|
|
1701
|
+
tail (#275).
|
|
1702
|
+
|
|
1703
|
+
``reconcile_cache_report_cache`` only evicts CLOSED days that CHANGED (``>=``
|
|
1704
|
+
the seq-gated change watermark); a day that simply rolls off the trailing edge
|
|
1705
|
+
of the ``[since, now]`` window is never touched by it and would linger in the
|
|
1706
|
+
module dict until a reset / tz-change / ``sf_sig`` / regression full-clear or
|
|
1707
|
+
``reset_cache_report_state()``. On a long-uptime dashboard that accretes ~1
|
|
1708
|
+
frozen ``CachedCacheReportDay`` per day. ``build_cache_report_snapshot`` calls
|
|
1709
|
+
this on its rare cold/rollover store tick with the window's oldest still-needed
|
|
1710
|
+
closed date, keeping the dict bounded to the live window. Over-eviction is
|
|
1711
|
+
byte-safe (a re-needed day just re-populates on the next cold tick), so the
|
|
1712
|
+
predicate is a plain lexical ``<`` on the ``YYYY-MM-DD`` keys.
|
|
1713
|
+
"""
|
|
1714
|
+
for date_key in [d for d in _CACHE_REPORT_DAY_CACHE if d < oldest_key]:
|
|
1715
|
+
del _CACHE_REPORT_DAY_CACHE[date_key]
|
|
1716
|
+
|
|
1717
|
+
|
|
1718
|
+
def reconcile_cache_report_cache(
|
|
1719
|
+
cache_conn, *, max_entry_id, max_mutation_seq, reset_sig, sf_sig, bucket_tz,
|
|
1720
|
+
tz_key,
|
|
1721
|
+
):
|
|
1722
|
+
"""Once-per-non-idle-rebuild invalidation for the cache-report cache (#272 §5).
|
|
1723
|
+
|
|
1724
|
+
The canonical four-step shape (mirrors ``reconcile_bugk_cache`` +
|
|
1725
|
+
``reconcile_weekref_cache``, plus the projects-env cache's
|
|
1726
|
+
``session_files_sig`` leg and a display-tz leg):
|
|
1727
|
+
|
|
1728
|
+
1. **Cold** (empty last-seen) → record
|
|
1729
|
+
``{max_id, max_seq, reset_sig, sf_sig, tz_key}``, return, no eviction.
|
|
1730
|
+
2. **Full-clear** on any of: ``reset_sig`` changed, ``sf_sig`` changed
|
|
1731
|
+
(Codex-1 — lazy ``session_files.project_path`` backfill can re-attribute
|
|
1732
|
+
a CLOSED day's by_project rows with no ``session_entries`` / seq change).
|
|
1733
|
+
INHERITED LIMITATION (accepted; matches the projects-env cache, same
|
|
1734
|
+
``sf_sig`` leg): ``sf_sig`` is ``(COUNT(*), MAX(rowid))``, so an in-place
|
|
1735
|
+
``ON CONFLICT(path) DO UPDATE SET project_path = COALESCE(...)`` backfill
|
|
1736
|
+
(rowid + count both stable — the actual lazy-backfill shape) does NOT
|
|
1737
|
+
move it. Such an UPDATE is only watermark-covered via the co-ingested new
|
|
1738
|
+
``session_entries`` rows, which carry the NEW day — so the seq-gated
|
|
1739
|
+
eviction reaches only ``>= that day`` and a re-attributed CLOSED day can
|
|
1740
|
+
stale-serve ``(unknown)`` until it rolls out of the window (or a reset
|
|
1741
|
+
full-clears). A framework-wide fix belongs with the shared ``sf_sig``
|
|
1742
|
+
leg, not #272.
|
|
1743
|
+
``tz_key`` changed (a display-tz change shifts every calendar-day
|
|
1744
|
+
boundary → all cached day keys invalid), ``max_entry_id`` regressed, or
|
|
1745
|
+
``max_mutation_seq`` regressed (cache.db rebuilt out-of-process) →
|
|
1746
|
+
``clear()``, update last-seen, return.
|
|
1747
|
+
3. **Seq-gated eviction** (``max_mutation_seq > last_seen_seq`` — the #270
|
|
1748
|
+
§7c seq gate, so an id-stable in-place finalization with a flat
|
|
1749
|
+
``max_entry_id`` still evicts): ``wm = changed_min_timestamp(cache_conn,
|
|
1750
|
+
last_seen_seq)`` (the one query on this branch; #270
|
|
1751
|
+
``min(old_ts, new_ts)`` change-time). Evict every cached day whose
|
|
1752
|
+
``date_key >= wm.astimezone(bucket_tz).strftime("%Y-%m-%d")``. The bound
|
|
1753
|
+
is ``>=`` (INCLUSIVE), NOT ``>`` — a day bucket is the CLOSED interval
|
|
1754
|
+
``[start, end]``, so a changed row landing anywhere on ``wm_day`` (or a
|
|
1755
|
+
later day) is inside that day and must evict it; and a cross-day
|
|
1756
|
+
finalization pulls ``mutation_min_ts`` back to ``min(old, new)``, i.e.
|
|
1757
|
+
the OLD day, so the OLD day (exactly at the watermark) evicts too. This
|
|
1758
|
+
is the ONE semantic difference from ``reconcile_bugk_cache``'s strict
|
|
1759
|
+
``>`` (its segment window is HALF-OPEN ``[start, effective)``).
|
|
1760
|
+
Over-eviction is byte-safe. Update last-seen.
|
|
1761
|
+
4. **Idempotent within a tick**: after the first call updates last-seen, a
|
|
1762
|
+
same-signature second call sees ``max_mutation_seq == last_seen_seq`` and
|
|
1763
|
+
no sig delta → no watermark re-query, no eviction.
|
|
1764
|
+
"""
|
|
1765
|
+
ls = _CACHE_REPORT_LAST_SEEN
|
|
1766
|
+
if not ls: # cold
|
|
1767
|
+
ls.update(
|
|
1768
|
+
max_id=max_entry_id, max_seq=max_mutation_seq,
|
|
1769
|
+
reset_sig=reset_sig, sf_sig=sf_sig, tz_key=tz_key,
|
|
1770
|
+
)
|
|
1771
|
+
return
|
|
1772
|
+
if (
|
|
1773
|
+
reset_sig != ls["reset_sig"]
|
|
1774
|
+
or sf_sig != ls["sf_sig"]
|
|
1775
|
+
or tz_key != ls["tz_key"]
|
|
1776
|
+
or max_entry_id < ls["max_id"]
|
|
1777
|
+
or max_mutation_seq < ls["max_seq"]
|
|
1778
|
+
):
|
|
1779
|
+
_CACHE_REPORT_DAY_CACHE.clear()
|
|
1780
|
+
ls.update(
|
|
1781
|
+
max_id=max_entry_id, max_seq=max_mutation_seq,
|
|
1782
|
+
reset_sig=reset_sig, sf_sig=sf_sig, tz_key=tz_key,
|
|
1783
|
+
)
|
|
1784
|
+
return
|
|
1785
|
+
if max_mutation_seq > ls["max_seq"]:
|
|
1786
|
+
wm = changed_min_timestamp(cache_conn, ls["max_seq"])
|
|
1787
|
+
if wm is not None:
|
|
1788
|
+
wm_day = wm.astimezone(bucket_tz).strftime("%Y-%m-%d")
|
|
1789
|
+
for date_key in list(_CACHE_REPORT_DAY_CACHE):
|
|
1790
|
+
# Inclusive [start, end] day bucket: a changed row on wm_day (or
|
|
1791
|
+
# any later day) falls inside that day, so evict date_key >= wm_day.
|
|
1792
|
+
if date_key >= wm_day:
|
|
1793
|
+
del _CACHE_REPORT_DAY_CACHE[date_key]
|
|
1794
|
+
ls.update(
|
|
1795
|
+
max_id=max_entry_id, max_seq=max_mutation_seq,
|
|
1796
|
+
reset_sig=reset_sig, sf_sig=sf_sig, tz_key=tz_key,
|
|
1797
|
+
)
|