cctally 1.61.0 → 1.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/bin/_cctally_cache.py +156 -6
- package/bin/_cctally_config.py +26 -0
- package/bin/_cctally_dashboard.py +670 -162
- package/bin/_cctally_db.py +27 -1
- package/bin/_cctally_parser.py +15 -0
- package/bin/_cctally_statusline.py +9 -1
- package/bin/_cctally_tui.py +74 -9
- package/bin/_cctally_weekrefs.py +9 -0
- package/bin/_lib_aggregators.py +98 -71
- package/bin/_lib_cache_report.py +455 -67
- package/bin/_lib_pricing.py +31 -4
- package/bin/_lib_snapshot_cache.py +869 -60
- package/bin/_lib_statusline.py +25 -13
- package/package.json +1 -1
|
@@ -48,10 +48,19 @@ import datetime as dt
|
|
|
48
48
|
import os
|
|
49
49
|
import sqlite3
|
|
50
50
|
import threading
|
|
51
|
+
from dataclasses import dataclass
|
|
51
52
|
from typing import TYPE_CHECKING, Callable, NamedTuple
|
|
52
53
|
|
|
53
54
|
from _cctally_core import parse_iso_datetime
|
|
54
55
|
|
|
56
|
+
# #271: the current-bucket accumulator (below) reuses the aggregators' single
|
|
57
|
+
# per-entry fold primitive so the incremental append is byte-identical to the
|
|
58
|
+
# full pass (spec §6/§7). This is the ONE deliberate runtime coupling to
|
|
59
|
+
# _lib_aggregators — the cache primitives above stay decoupled (BucketUsage is
|
|
60
|
+
# still a TYPE_CHECKING-only hint). _lib_aggregators imports only _cctally_core,
|
|
61
|
+
# so there is no import cycle.
|
|
62
|
+
from _lib_aggregators import _fold_entry, _finalize_bucket, _new_bucket_acc
|
|
63
|
+
|
|
55
64
|
if TYPE_CHECKING: # type hints only — no runtime coupling to the aggregators
|
|
56
65
|
from _lib_aggregators import BucketUsage, ClaudeSessionUsage
|
|
57
66
|
|
|
@@ -72,6 +81,10 @@ class SnapshotSignature(NamedTuple):
|
|
|
72
81
|
reset_sig: tuple[int, int]
|
|
73
82
|
max_codex_id: int
|
|
74
83
|
generation: int
|
|
84
|
+
# #270 §7a: the O(1) `cache_meta` mutation counter. An id-stable in-place
|
|
85
|
+
# finalization UPSERT advances this leg while `max_entry_id` stays flat, so
|
|
86
|
+
# the dashboard leaves the idle path and recomputes the affected bucket.
|
|
87
|
+
entry_mutation_seq: int = 0
|
|
75
88
|
|
|
76
89
|
|
|
77
90
|
def _max_id(conn: sqlite3.Connection, table: str) -> int:
|
|
@@ -88,6 +101,32 @@ def _max_id(conn: sqlite3.Connection, table: str) -> int:
|
|
|
88
101
|
return 0
|
|
89
102
|
|
|
90
103
|
|
|
104
|
+
def _entry_mutation_seq(conn: sqlite3.Connection) -> int:
|
|
105
|
+
"""Current ``session_entries`` mutation counter from ``cache_meta`` (§7a).
|
|
106
|
+
|
|
107
|
+
Reads the O(1) KV counter (key ``session_entries_mutation_seq``) that
|
|
108
|
+
``sync_cache`` bumps on every insert AND every WHERE-passing in-place UPSERT
|
|
109
|
+
(#270). This is the new ``entry_mutation_seq`` signature leg: an id-stable
|
|
110
|
+
finalization advances it even though ``MAX(session_entries.id)`` is flat, so
|
|
111
|
+
the dashboard leaves the idle path and recomputes the affected bucket.
|
|
112
|
+
|
|
113
|
+
An O(1) KV read — NOT a ``MAX(mutation_seq)`` full scan (that is exactly the
|
|
114
|
+
per-tick cost #268 removed). Degrades to 0 on a fresh / partially-migrated DB
|
|
115
|
+
where the ``cache_meta`` table or the key is absent (like ``_max_id`` /
|
|
116
|
+
``_reset_sig``), so the signature never raises.
|
|
117
|
+
"""
|
|
118
|
+
try:
|
|
119
|
+
row = conn.execute(
|
|
120
|
+
"SELECT value FROM cache_meta "
|
|
121
|
+
"WHERE key='session_entries_mutation_seq'"
|
|
122
|
+
).fetchone()
|
|
123
|
+
except sqlite3.Error:
|
|
124
|
+
return 0
|
|
125
|
+
if row is None or row[0] is None:
|
|
126
|
+
return 0
|
|
127
|
+
return int(row[0])
|
|
128
|
+
|
|
129
|
+
|
|
91
130
|
def _reset_sig(conn: sqlite3.Connection) -> tuple[int, int]:
|
|
92
131
|
"""Change-signal over the two reset-event tables combined (spec §3).
|
|
93
132
|
|
|
@@ -133,6 +172,7 @@ def compute_signature(
|
|
|
133
172
|
reset_sig=_reset_sig(stats_conn),
|
|
134
173
|
max_codex_id=_max_id(cache_conn, "codex_session_entries"),
|
|
135
174
|
generation=int(generation),
|
|
175
|
+
entry_mutation_seq=_entry_mutation_seq(cache_conn),
|
|
136
176
|
)
|
|
137
177
|
|
|
138
178
|
|
|
@@ -176,6 +216,53 @@ def new_min_timestamp(
|
|
|
176
216
|
return parsed.astimezone(dt.timezone.utc)
|
|
177
217
|
|
|
178
218
|
|
|
219
|
+
def changed_min_timestamp(
|
|
220
|
+
cache_conn: sqlite3.Connection,
|
|
221
|
+
last_seen_seq: int,
|
|
222
|
+
) -> "dt.datetime | None":
|
|
223
|
+
"""Earliest event time among rows CHANGED since ``last_seen_seq`` (§7b).
|
|
224
|
+
|
|
225
|
+
Returns ``MIN(mutation_min_ts) WHERE mutation_seq > last_seen_seq`` as an
|
|
226
|
+
aware UTC datetime, or ``None`` when there are no such rows. Generalizes
|
|
227
|
+
``new_min_timestamp`` from the ``id`` watermark to the #270 ``mutation_seq``
|
|
228
|
+
watermark; the same parse / normalize-to-UTC path and the same
|
|
229
|
+
``None``-on-missing-table degrade.
|
|
230
|
+
|
|
231
|
+
Load-bearing byte-identity invariant: on a PURE-INSERT interval every changed
|
|
232
|
+
row is a fresh insert carrying a new ``id`` (> last_seen_id), a new
|
|
233
|
+
``mutation_seq`` (> last_seen_seq), and ``mutation_min_ts == timestamp_utc``,
|
|
234
|
+
and no existing row was re-stamped — so ``{mutation_seq > last_seen_seq}`` ==
|
|
235
|
+
``{id > last_seen_id}`` and ``MIN(mutation_min_ts)`` == ``MIN(timestamp_utc)``,
|
|
236
|
+
i.e. this equals ``new_min_timestamp(last_seen_id)`` (late-ingest reach-back
|
|
237
|
+
included). When an in-place update landed, the seq set additionally contains
|
|
238
|
+
the updated row, whose ``mutation_min_ts = min(all event times it has held)``
|
|
239
|
+
pulls the watermark back to the EARLIEST bucket it has touched, so both the
|
|
240
|
+
old bucket (stripped from) and the new bucket (added to) are recomputed;
|
|
241
|
+
over-marking the between-buckets is byte-safe. ``MIN()`` ignores the NULL
|
|
242
|
+
``mutation_min_ts`` of any legacy seq-0 row (which this ``> last_seen_seq``
|
|
243
|
+
predicate never selects anyway).
|
|
244
|
+
|
|
245
|
+
Backed by ``idx_entries_mutation_seq(mutation_seq, mutation_min_ts)`` — an
|
|
246
|
+
index-only range-min over the handful of rows changed since the last tick, so
|
|
247
|
+
#268's per-tick full-scan is not reintroduced.
|
|
248
|
+
"""
|
|
249
|
+
try:
|
|
250
|
+
row = cache_conn.execute(
|
|
251
|
+
"SELECT MIN(mutation_min_ts) FROM session_entries "
|
|
252
|
+
"WHERE mutation_seq > ?",
|
|
253
|
+
(last_seen_seq,),
|
|
254
|
+
).fetchone()
|
|
255
|
+
except sqlite3.Error:
|
|
256
|
+
return None
|
|
257
|
+
if row is None or row[0] is None:
|
|
258
|
+
return None
|
|
259
|
+
# Parse via the repo's canonical ISO helper (both ``...Z`` and ``...+00:00``
|
|
260
|
+
# stored forms), then normalize to a UTC-tzinfo instant so downstream
|
|
261
|
+
# comparisons against UTC bucket boundaries are exact.
|
|
262
|
+
parsed = parse_iso_datetime(row[0], "session_entries.mutation_min_ts")
|
|
263
|
+
return parsed.astimezone(dt.timezone.utc)
|
|
264
|
+
|
|
265
|
+
|
|
179
266
|
# === Task 0.3 — cache-generation counter ===================================
|
|
180
267
|
#
|
|
181
268
|
# A monotonic counter bumped by any path that deletes/rewrites history in
|
|
@@ -346,6 +433,148 @@ def reset_group_a_state() -> None:
|
|
|
346
433
|
"""
|
|
347
434
|
_GROUP_A_CACHE.clear()
|
|
348
435
|
_GROUP_A_LAST_SEEN.clear()
|
|
436
|
+
reset_group_a_current_state()
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
# === #271 — incremental current-bucket accumulator =========================
|
|
440
|
+
#
|
|
441
|
+
# The Group A builders (daily / monthly / weekly) re-fold the whole OPEN
|
|
442
|
+
# bucket every warm tick (`cached_buckets` recomputes `current_label` from a
|
|
443
|
+
# full window fetch). On a recency-dense instance that open week/month/day
|
|
444
|
+
# holds tens of thousands of entries, so the recompute dominates the residual
|
|
445
|
+
# warm rebuild (#271 §1). This holds a persisted single-left-fold accumulator
|
|
446
|
+
# per builder key: each warm tick folds only the DELTA (new-by-id OR
|
|
447
|
+
# newly-in-window-because-`now`-advanced) into the running aggregate, with a
|
|
448
|
+
# full-recompute fallback when a late older-timestamp row lands mid-bucket.
|
|
449
|
+
#
|
|
450
|
+
# Byte-identity rests on the pinned `(timestamp_utc, id)` fold order (#271 §5)
|
|
451
|
+
# + the shared `_fold_entry` primitive (§6): the incremental append reproduces
|
|
452
|
+
# the full left-fold exactly. Single-writer (sync thread only), module state,
|
|
453
|
+
# never reachable from a published DataSnapshot (F7 — the snapshot holds the
|
|
454
|
+
# finalized BucketUsage, not `acc`).
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
@dataclass
|
|
458
|
+
class CurrentBucketAccumulator:
|
|
459
|
+
"""Persisted running fold of one Group A builder's CURRENT bucket (§7a)."""
|
|
460
|
+
label: str
|
|
461
|
+
acc: dict # running _new_bucket_acc() shape
|
|
462
|
+
tail: "tuple | None" # (timestamp_utc, id) of the last folded entry
|
|
463
|
+
last_seen_id: int # max session_entries.id reconciled (= cur_max_id)
|
|
464
|
+
last_seen_seq: int # #270 §8: the mutation_seq watermark (= cur_max_seq)
|
|
465
|
+
last_now: dt.datetime # now_utc upper bound used to clamp the last fold
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
_GROUP_A_CURRENT: "dict[str, CurrentBucketAccumulator]" = {}
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def reset_group_a_current_state() -> None:
|
|
472
|
+
"""Drop every builder's current-bucket accumulator (prune-site + full-invalidate)."""
|
|
473
|
+
_GROUP_A_CURRENT.clear()
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _finalize_or_none(label, acc, tail):
|
|
477
|
+
"""Finalize ``acc`` into a ``BucketUsage``, or ``None`` when nothing was
|
|
478
|
+
folded (``tail is None`` ⇒ an empty/gap bucket, matching a from-scratch
|
|
479
|
+
``aggregate_one`` returning ``None``)."""
|
|
480
|
+
return None if tail is None else _finalize_bucket(label, acc)
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def accumulate_current_bucket(prior, *, current_label, cur_now, cur_max_id,
|
|
484
|
+
cur_max_seq, fetch_all, fetch_delta, membership,
|
|
485
|
+
mode="auto"):
|
|
486
|
+
"""Pure §7b tick algorithm. Returns ``(BucketUsage | None, CurrentBucketAccumulator)``.
|
|
487
|
+
|
|
488
|
+
``fetch_all() -> list[(id, UsageEntry)]`` over the whole current-bucket
|
|
489
|
+
window (cold / fallback); ``fetch_delta(after_seq, after_ts) ->
|
|
490
|
+
list[(id, UsageEntry)]`` the ``(mutation_seq > after_seq OR ts > after_ts)``
|
|
491
|
+
delta (#270 §8 re-key); both ordered ``(timestamp_utc, id)``.
|
|
492
|
+
``membership(entry) -> bool`` keeps exactly the entries the full pass assigns
|
|
493
|
+
to ``current_label``.
|
|
494
|
+
|
|
495
|
+
#270 §8: the delta is keyed on ``mutation_seq`` (a strict superset of the old
|
|
496
|
+
``id`` leg — every insert carries a fresh seq, so the insert path stays
|
|
497
|
+
byte-identical), so an id-stable in-place finalization of a current-bucket
|
|
498
|
+
row now appears in the delta. But the incremental fold cannot un-fold a stale
|
|
499
|
+
contribution: a finalization that keeps or increases its timestamp sorts
|
|
500
|
+
AT-OR-AFTER ``tail`` and the ``(ts,id) <= tail`` gate would append it,
|
|
501
|
+
double-counting the already-folded partial. So any delta row that is a
|
|
502
|
+
PRE-EXISTING row (``id <= reconciled_max_id``) forces a cold refold of the
|
|
503
|
+
whole current bucket instead. Genuine new inserts (``id > reconciled_max_id``)
|
|
504
|
+
proceed through the unchanged ``(ts,id)``-vs-``tail`` gate + append.
|
|
505
|
+
"""
|
|
506
|
+
def _cold():
|
|
507
|
+
acc = _new_bucket_acc()
|
|
508
|
+
tail = None
|
|
509
|
+
for eid, e in fetch_all():
|
|
510
|
+
if e.model == "<synthetic>" or not membership(e):
|
|
511
|
+
continue
|
|
512
|
+
_fold_entry(acc, e, mode)
|
|
513
|
+
tail = (e.timestamp, eid)
|
|
514
|
+
return acc, tail
|
|
515
|
+
|
|
516
|
+
# Cold refold on: no prior, a bucket rollover (label change), OR a backward
|
|
517
|
+
# wall-clock step (`cur_now < prior.last_now`, e.g. an NTP adjustment). The
|
|
518
|
+
# current-bucket fetch window is clamped to `now_utc`; if `now` moves
|
|
519
|
+
# backward with no data change, the empty-delta fast path would reuse the
|
|
520
|
+
# larger prior fold set and OVER-count vs a from-scratch pass over the
|
|
521
|
+
# shrunken `[start, cur_now]` window. A graceful cold refold matches
|
|
522
|
+
# from-scratch byte-for-byte — never `assert`, a clock step must not crash
|
|
523
|
+
# the dashboard (#271 M1 review).
|
|
524
|
+
if (
|
|
525
|
+
prior is None
|
|
526
|
+
or prior.label != current_label
|
|
527
|
+
or cur_now < prior.last_now
|
|
528
|
+
):
|
|
529
|
+
acc, tail = _cold()
|
|
530
|
+
new = CurrentBucketAccumulator(current_label, acc, tail, cur_max_id,
|
|
531
|
+
cur_max_seq, cur_now)
|
|
532
|
+
return _finalize_or_none(current_label, acc, tail), new
|
|
533
|
+
|
|
534
|
+
delta = [
|
|
535
|
+
(eid, e) for eid, e in fetch_delta(prior.last_seen_seq, prior.last_now)
|
|
536
|
+
if e.model != "<synthetic>" and membership(e)
|
|
537
|
+
]
|
|
538
|
+
if not delta:
|
|
539
|
+
# From-scratch fold set unchanged since last tick → byte-identical.
|
|
540
|
+
new = CurrentBucketAccumulator(current_label, prior.acc, prior.tail,
|
|
541
|
+
cur_max_id, cur_max_seq, cur_now)
|
|
542
|
+
return _finalize_or_none(current_label, prior.acc, prior.tail), new
|
|
543
|
+
|
|
544
|
+
# #270 §8: any PRE-EXISTING row in the seq-delta (id <= reconciled_max_id) is
|
|
545
|
+
# an in-place update (a finalization UPSERT re-stamped its seq) — or the
|
|
546
|
+
# near-impossible clock-advance re-entry — that the incremental fold cannot
|
|
547
|
+
# safely reconcile (appending would double-count the already-folded partial).
|
|
548
|
+
# Discard the delta and cold-refold the whole current bucket (always
|
|
549
|
+
# byte-correct, bounded to this bucket). Checked BEFORE the fold-order gate so
|
|
550
|
+
# a timestamp-non-decreasing finalization (which sorts past `tail`, evading
|
|
551
|
+
# that gate) is caught here.
|
|
552
|
+
if any(eid <= prior.last_seen_id for eid, _e in delta):
|
|
553
|
+
acc, tail = _cold()
|
|
554
|
+
new = CurrentBucketAccumulator(current_label, acc, tail, cur_max_id,
|
|
555
|
+
cur_max_seq, cur_now)
|
|
556
|
+
return _finalize_or_none(current_label, acc, tail), new
|
|
557
|
+
|
|
558
|
+
# Guard: every folded delta row must sort AFTER the tail (a strict suffix).
|
|
559
|
+
# tail None ⇒ prior folded nothing ⇒ the delta IS the full member set (each
|
|
560
|
+
# current member has mutation_seq>last_seen_seq OR ts>last_now), so appending
|
|
561
|
+
# == full fold.
|
|
562
|
+
if prior.tail is not None and any(
|
|
563
|
+
(e.timestamp, eid) <= prior.tail for eid, e in delta
|
|
564
|
+
):
|
|
565
|
+
acc, tail = _cold() # late-ingest mid-bucket fallback
|
|
566
|
+
new = CurrentBucketAccumulator(current_label, acc, tail, cur_max_id,
|
|
567
|
+
cur_max_seq, cur_now)
|
|
568
|
+
return _finalize_or_none(current_label, acc, tail), new
|
|
569
|
+
|
|
570
|
+
acc = prior.acc # mutate in place (module state, never published)
|
|
571
|
+
tail = prior.tail
|
|
572
|
+
for eid, e in delta: # fetch_delta returns (timestamp_utc, id) ascending
|
|
573
|
+
_fold_entry(acc, e, mode)
|
|
574
|
+
tail = (e.timestamp, eid)
|
|
575
|
+
new = CurrentBucketAccumulator(current_label, acc, tail, cur_max_id,
|
|
576
|
+
cur_max_seq, cur_now)
|
|
577
|
+
return _finalize_or_none(current_label, acc, tail), new
|
|
349
578
|
|
|
350
579
|
|
|
351
580
|
def cached_buckets(
|
|
@@ -357,37 +586,48 @@ def cached_buckets(
|
|
|
357
586
|
dirty_predicate: "Callable[[str], bool]",
|
|
358
587
|
fetch_bucket_entries: "Callable[[str], list]",
|
|
359
588
|
aggregate_one: "Callable[[str, list], object | None]",
|
|
589
|
+
current_override: "Callable[[], object | None] | None" = None,
|
|
360
590
|
) -> "list[object]":
|
|
361
591
|
"""Assemble one Group A builder's per-bucket aggregates (spec §5.1).
|
|
362
592
|
|
|
363
593
|
For each label in ``all_bucket_labels`` (caller order — pass oldest→newest
|
|
364
594
|
so the assembled list matches ``_aggregate_*``'s ascending-key output):
|
|
365
595
|
|
|
366
|
-
- If the label is ``current_label``
|
|
367
|
-
|
|
368
|
-
|
|
596
|
+
- If the label is ``current_label`` and ``current_override`` is supplied
|
|
597
|
+
(#271 §8a), the current bucket is produced by ``current_override()`` — the
|
|
598
|
+
incremental accumulator — INSTEAD of the full ``aggregate_one`` recompute.
|
|
599
|
+
The override returns the finalized ``BucketUsage`` (or ``None`` for a
|
|
600
|
+
no-data current bucket, handled by the same gap-drop below).
|
|
601
|
+
- Else if the label is ``current_label`` or ``dirty_predicate`` returns True
|
|
602
|
+
(the watermark reached it, or a forced recompute), recompute it WHOLE via
|
|
603
|
+
``aggregate_one(label, fetch_bucket_entries(label))`` — a
|
|
369
604
|
timestamp-ordered fetch, so ``_aggregate_buckets`` first-seen model order
|
|
370
605
|
reproduces the full-history pass byte-for-byte (Codex F3).
|
|
371
606
|
- Otherwise serve the cached ``BucketUsage``; on a cache MISS (cold start /
|
|
372
607
|
post-invalidation) recompute it the same way.
|
|
373
608
|
|
|
374
|
-
``aggregate_one``
|
|
375
|
-
day/month/week): the bucket is omitted from the result and any
|
|
376
|
-
entry for that label is evicted. The returned list therefore
|
|
377
|
-
buckets-with-data — exactly what
|
|
378
|
-
the full entry set returns — in
|
|
609
|
+
``aggregate_one`` / ``current_override`` return ``None`` for a label with no
|
|
610
|
+
data (a gap day/month/week): the bucket is omitted from the result and any
|
|
611
|
+
stale cache entry for that label is evicted. The returned list therefore
|
|
612
|
+
contains only buckets-with-data — exactly what
|
|
613
|
+
``_aggregate_daily/_monthly/_weekly`` over the full entry set returns — in
|
|
614
|
+
``all_bucket_labels`` order.
|
|
379
615
|
|
|
380
616
|
Values are stored/served as-is (immutable ``BucketUsage``); this loop never
|
|
381
|
-
mutates a bucket in place (spec §7).
|
|
617
|
+
mutates a bucket in place (spec §7). ``current_override`` stays a pure hook:
|
|
618
|
+
the assembly order, past-bucket serve/cold-miss, and gap-drop are untouched.
|
|
382
619
|
"""
|
|
383
620
|
result: "list[object]" = []
|
|
384
621
|
for label in all_bucket_labels:
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
622
|
+
if label == current_label and current_override is not None:
|
|
623
|
+
bucket = current_override()
|
|
624
|
+
else:
|
|
625
|
+
recompute = (label == current_label) or dirty_predicate(label)
|
|
626
|
+
bucket = None
|
|
627
|
+
if not recompute:
|
|
628
|
+
bucket = cache.get(builder_key, label)
|
|
629
|
+
if bucket is None: # forced recompute OR cold miss
|
|
630
|
+
bucket = aggregate_one(label, fetch_bucket_entries(label))
|
|
391
631
|
if bucket is None:
|
|
392
632
|
# Gap bucket (no data): drop any stale cache entry, omit from output.
|
|
393
633
|
cache.drop_from(builder_key, lambda lbl, _t=label: lbl == _t)
|
|
@@ -407,6 +647,11 @@ def build_cached_group_a(
|
|
|
407
647
|
fetch_bucket_entries: "Callable[[str], list]",
|
|
408
648
|
aggregate_one: "Callable[[str, list], object | None]",
|
|
409
649
|
extra_signature: object = None,
|
|
650
|
+
use_current_accumulator: bool = False,
|
|
651
|
+
now_utc: "dt.datetime | None" = None,
|
|
652
|
+
current_all_fetch: "Callable[[str], list] | None" = None,
|
|
653
|
+
current_delta_fetch: "Callable[[str, int, dt.datetime], list] | None" = None,
|
|
654
|
+
membership_of: "Callable[[object], bool] | None" = None,
|
|
410
655
|
) -> "list[object]":
|
|
411
656
|
"""Stateful Group A assembly: invalidation + watermark + ``cached_buckets``.
|
|
412
657
|
|
|
@@ -427,13 +672,23 @@ def build_cached_group_a(
|
|
|
427
672
|
order, and updates this builder's last-seen state.
|
|
428
673
|
"""
|
|
429
674
|
cur_max_id = _max_id(cache_conn, "session_entries")
|
|
675
|
+
# #270 §7c: split max_id's two duties — the incremental watermark + "any
|
|
676
|
+
# new work?" gate move to the O(1) `cache_meta` mutation counter (an
|
|
677
|
+
# id-stable in-place finalization advances the seq while `max_id` stays
|
|
678
|
+
# flat), while `max_id` stays the regression backstop below.
|
|
679
|
+
cur_max_seq = _entry_mutation_seq(cache_conn)
|
|
430
680
|
state = _GROUP_A_LAST_SEEN.get(builder_key)
|
|
431
681
|
full_invalidate = state is not None and (
|
|
432
682
|
state.get("extra") != extra_signature
|
|
433
683
|
or cur_max_id < int(state.get("max_id", 0))
|
|
684
|
+
or cur_max_seq < int(state.get("max_seq", 0)) # belt-and-suspenders
|
|
434
685
|
)
|
|
435
686
|
if full_invalidate:
|
|
436
687
|
_GROUP_A_CACHE.drop_from(builder_key, lambda _lbl: True)
|
|
688
|
+
# The current bucket's boundaries may have shifted (a weekly signature
|
|
689
|
+
# move / cache.db rebuild) → discard the accumulator so the override
|
|
690
|
+
# cold-refolds this tick (#271 §8c).
|
|
691
|
+
_GROUP_A_CURRENT.pop(builder_key, None)
|
|
437
692
|
|
|
438
693
|
if state is None or full_invalidate:
|
|
439
694
|
# Cold start OR post-invalidation: recompute every label. (Cold already
|
|
@@ -442,7 +697,11 @@ def build_cached_group_a(
|
|
|
442
697
|
def dirty(_label: str) -> bool:
|
|
443
698
|
return True
|
|
444
699
|
else:
|
|
445
|
-
|
|
700
|
+
# #270 §7b: the change-aware watermark over `mutation_min_ts` (the
|
|
701
|
+
# earliest event time any row CHANGED-since-last-tick has held), so an
|
|
702
|
+
# id-stable finalization that landed in / moved to a closed bucket marks
|
|
703
|
+
# it dirty. Reduces to `new_min_timestamp(max_id)` on a pure-insert tick.
|
|
704
|
+
new_min_ts = changed_min_timestamp(cache_conn, int(state.get("max_seq", 0)))
|
|
446
705
|
|
|
447
706
|
def dirty(label: str) -> bool:
|
|
448
707
|
if new_min_ts is None:
|
|
@@ -450,6 +709,34 @@ def build_cached_group_a(
|
|
|
450
709
|
end = bucket_end_of(label)
|
|
451
710
|
return end is not None and end > new_min_ts
|
|
452
711
|
|
|
712
|
+
# #271 §8a/§8b: when the incremental accumulator is enabled, the current
|
|
713
|
+
# bucket is produced by folding only the delta each tick instead of a full
|
|
714
|
+
# re-aggregate. #270 §8: the delta's lower bound is now the mutation_seq
|
|
715
|
+
# watermark `cur_max_seq` (the same O(1) `cache_meta` counter the signature
|
|
716
|
+
# + Group A dirty-predicate read), reused via prior.last_seen_seq inside the
|
|
717
|
+
# accumulator, so an id-stable in-place finalization of a current-bucket row
|
|
718
|
+
# is caught (and cold-refolded). Gated ON only by the sync-thread
|
|
719
|
+
# `_group_a_*_buckets` closures; every other caller leaves it off
|
|
720
|
+
# (byte-identical to today).
|
|
721
|
+
current_override = None
|
|
722
|
+
if use_current_accumulator and current_label is not None:
|
|
723
|
+
def current_override():
|
|
724
|
+
prior = _GROUP_A_CURRENT.get(builder_key)
|
|
725
|
+
bucket, new_state = accumulate_current_bucket(
|
|
726
|
+
prior,
|
|
727
|
+
current_label=current_label,
|
|
728
|
+
cur_now=now_utc,
|
|
729
|
+
cur_max_id=cur_max_id,
|
|
730
|
+
cur_max_seq=cur_max_seq,
|
|
731
|
+
fetch_all=lambda: current_all_fetch(current_label),
|
|
732
|
+
fetch_delta=lambda aseq, ats: current_delta_fetch(
|
|
733
|
+
current_label, aseq, ats),
|
|
734
|
+
membership=membership_of,
|
|
735
|
+
mode="auto",
|
|
736
|
+
)
|
|
737
|
+
_GROUP_A_CURRENT[builder_key] = new_state
|
|
738
|
+
return bucket
|
|
739
|
+
|
|
453
740
|
buckets = cached_buckets(
|
|
454
741
|
builder_key,
|
|
455
742
|
cache=_GROUP_A_CACHE,
|
|
@@ -458,8 +745,11 @@ def build_cached_group_a(
|
|
|
458
745
|
dirty_predicate=dirty,
|
|
459
746
|
fetch_bucket_entries=fetch_bucket_entries,
|
|
460
747
|
aggregate_one=aggregate_one,
|
|
748
|
+
current_override=current_override,
|
|
461
749
|
)
|
|
462
|
-
_GROUP_A_LAST_SEEN[builder_key] = {
|
|
750
|
+
_GROUP_A_LAST_SEEN[builder_key] = {
|
|
751
|
+
"max_id": cur_max_id, "max_seq": cur_max_seq, "extra": extra_signature,
|
|
752
|
+
}
|
|
463
753
|
return buckets
|
|
464
754
|
|
|
465
755
|
|
|
@@ -468,9 +758,9 @@ def build_cached_group_a(
|
|
|
468
758
|
|
|
469
759
|
def affected_session_keys(
|
|
470
760
|
cache_conn: sqlite3.Connection,
|
|
471
|
-
|
|
761
|
+
last_seen_seq: int,
|
|
472
762
|
) -> "set[str]":
|
|
473
|
-
"""Resolved session identities for entries
|
|
763
|
+
"""Resolved session identities for entries CHANGED since ``last_seen_seq`` (§5.2).
|
|
474
764
|
|
|
475
765
|
Mirrors ``_aggregate_claude_sessions`` grouping EXACTLY: identity is
|
|
476
766
|
``session_files.session_id`` when the ``LEFT JOIN`` on ``source_path``
|
|
@@ -485,14 +775,21 @@ def affected_session_keys(
|
|
|
485
775
|
the join to ``session_files`` — so the returned keys key IDENTICALLY to
|
|
486
776
|
the aggregator's session grouping (Codex F5). Returns an empty set on a
|
|
487
777
|
missing table (fresh / partially-migrated DB) so callers never raise.
|
|
778
|
+
|
|
779
|
+
#270 (§7d, Codex-2c): keyed on ``mutation_seq > last_seen_seq``, NOT
|
|
780
|
+
``id > last_seen`` — so an id-stable in-place finalization of an EXISTING
|
|
781
|
+
session's row (same id, advanced seq) surfaces that session as affected and
|
|
782
|
+
it re-aggregates. On a pure-insert interval every changed row carries a
|
|
783
|
+
fresh seq (> last_seen_seq) exactly when its id > last_seen, so the affected
|
|
784
|
+
set is byte-identical to the old id-keyed one.
|
|
488
785
|
"""
|
|
489
786
|
try:
|
|
490
787
|
rows = cache_conn.execute(
|
|
491
788
|
"SELECT se.source_path, sf.session_id "
|
|
492
789
|
"FROM session_entries se "
|
|
493
790
|
"LEFT JOIN session_files sf ON sf.path = se.source_path "
|
|
494
|
-
"WHERE se.
|
|
495
|
-
(
|
|
791
|
+
"WHERE se.mutation_seq > ? AND se.model != '<synthetic>'",
|
|
792
|
+
(last_seen_seq,),
|
|
496
793
|
).fetchall()
|
|
497
794
|
except sqlite3.Error:
|
|
498
795
|
return set()
|
|
@@ -567,25 +864,35 @@ def build_cached_sessions(
|
|
|
567
864
|
fallback sessions), so the cache keys match ``affected_session_keys``.
|
|
568
865
|
"""
|
|
569
866
|
cur_max_id = _max_id(cache_conn, "session_entries")
|
|
867
|
+
# #270 §7c/§7d: the warm "any new work?" gate + the affected-set query key
|
|
868
|
+
# on the `mutation_seq` counter (an id-stable in-place finalization of an
|
|
869
|
+
# EXISTING session advances the seq while `max_id` stays flat); `max_id`
|
|
870
|
+
# remains the cache.db-rebuild regression backstop.
|
|
871
|
+
cur_max_seq = _entry_mutation_seq(cache_conn)
|
|
570
872
|
state = _SESSION_LAST_SEEN
|
|
571
873
|
cold = (
|
|
572
874
|
not state
|
|
573
875
|
or state.get("extra") != extra_signature
|
|
574
876
|
or cur_max_id < int(state.get("max_id", 0))
|
|
877
|
+
or cur_max_seq < int(state.get("max_seq", 0)) # belt-and-suspenders
|
|
575
878
|
)
|
|
576
879
|
if cold:
|
|
577
880
|
_SESSION_CACHE.clear()
|
|
578
881
|
for sess in aggregate_all():
|
|
579
882
|
_SESSION_CACHE.put(sess.session_id, sess)
|
|
580
883
|
else:
|
|
581
|
-
|
|
582
|
-
if
|
|
583
|
-
affected = affected_session_keys(cache_conn,
|
|
884
|
+
last_seen_seq = int(state.get("max_seq", 0))
|
|
885
|
+
if cur_max_seq > last_seen_seq:
|
|
886
|
+
affected = affected_session_keys(cache_conn, last_seen_seq)
|
|
584
887
|
if affected:
|
|
585
|
-
|
|
888
|
+
# `reaggregate` re-fetches each affected session's FULL in-window
|
|
889
|
+
# entry set (seq-keyed too — Codex-2c), so the aggregate is
|
|
890
|
+
# byte-identical to from-scratch even for an id-stable update.
|
|
891
|
+
for sess in reaggregate(last_seen_seq, affected):
|
|
586
892
|
_SESSION_CACHE.put(sess.session_id, sess)
|
|
587
893
|
_SESSION_LAST_SEEN.clear()
|
|
588
894
|
_SESSION_LAST_SEEN["max_id"] = cur_max_id
|
|
895
|
+
_SESSION_LAST_SEEN["max_seq"] = cur_max_seq
|
|
589
896
|
_SESSION_LAST_SEEN["extra"] = extra_signature
|
|
590
897
|
return list(_SESSION_CACHE.get_all().values())
|
|
591
898
|
|
|
@@ -775,54 +1082,66 @@ def cached_weekref_cost(*, week_start_at, week_end_at, now_utc, compute):
|
|
|
775
1082
|
return val
|
|
776
1083
|
|
|
777
1084
|
|
|
778
|
-
def reconcile_weekref_cache(cache_conn, *, max_entry_id, reset_sig):
|
|
1085
|
+
def reconcile_weekref_cache(cache_conn, *, max_entry_id, max_mutation_seq, reset_sig):
|
|
779
1086
|
"""Once-per-non-idle-rebuild invalidation for the weekref-cost cache (spec §4).
|
|
780
1087
|
|
|
781
1088
|
Driven by ``_tui_build_snapshot`` after the idle-path check, before the
|
|
782
1089
|
builders run, using the already-computed dispatch-signature legs
|
|
783
|
-
(``max_entry_id`` + ``reset_sig`` are passed in — no
|
|
1090
|
+
(``max_entry_id`` + ``max_mutation_seq`` + ``reset_sig`` are passed in — no
|
|
1091
|
+
extra query for those):
|
|
784
1092
|
|
|
785
1093
|
- Cold (no last-seen): record last-seen, return — no eviction.
|
|
786
1094
|
- ``reset_sig`` changed OR ``max_entry_id < last_seen`` (cache.db rebuilt
|
|
787
|
-
out-of-process)
|
|
788
|
-
cost; reset events are rare, so a
|
|
789
|
-
cheap. A max-id
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
1095
|
+
out-of-process) OR ``max_mutation_seq < last_seen_seq``: full ``clear()``.
|
|
1096
|
+
A credit/reset re-shapes a past week's cost; reset events are rare, so a
|
|
1097
|
+
conservative full clear is correct and cheap. A max-id / seq regression
|
|
1098
|
+
means the ids no longer map to the same rows.
|
|
1099
|
+
- ``max_mutation_seq > last_seen_seq`` (#270 §7c — the seq gate, so an
|
|
1100
|
+
id-stable in-place finalization with a flat ``max_entry_id`` still
|
|
1101
|
+
evicts): evict cached weeks whose ``week_end_at`` is
|
|
1102
|
+
``>= changed_min_timestamp(cache_conn, last_seen_seq)`` — a
|
|
1103
|
+
genuinely-changed row (new OR finalized-in-place) could fall inside them
|
|
1104
|
+
(F1 late-ingest / #270 in-place). The bound is ``>=``, NOT ``>``, because
|
|
1105
|
+
``_sum_cost_for_range`` / ``iter_entries`` sum an inclusive
|
|
794
1106
|
``[start, end]`` window, so a row whose timestamp lands exactly on
|
|
795
1107
|
``week_end_at`` belongs to that week (Codex-1). Over-eviction is byte-safe
|
|
796
1108
|
(forces a recompute). Normally ``wm`` is recent and no closed week drops.
|
|
797
1109
|
|
|
798
1110
|
Idempotent within a tick: after the first call updates last-seen, a later
|
|
799
|
-
call in the same tick sees no delta (``
|
|
1111
|
+
call in the same tick sees no delta (``max_mutation_seq == last_seen_seq``,
|
|
800
1112
|
``reset_sig`` unchanged) and no-ops — never re-running the watermark query.
|
|
801
1113
|
|
|
802
|
-
Connection lifecycle (Codex-4): the ``
|
|
803
|
-
the only use of ``cache_conn`` and runs only on the
|
|
804
|
-
``
|
|
805
|
-
connection, opened for that query.
|
|
1114
|
+
Connection lifecycle (Codex-4): the ``changed_min_timestamp`` watermark query
|
|
1115
|
+
is the only use of ``cache_conn`` and runs only on the
|
|
1116
|
+
``max_mutation_seq > last_seen_seq`` branch; the caller passes a short-lived
|
|
1117
|
+
cache connection, opened for that query.
|
|
806
1118
|
"""
|
|
807
1119
|
ls = _WEEKREF_COST_LAST_SEEN
|
|
808
1120
|
if not ls: # cold
|
|
809
1121
|
ls["max_id"] = max_entry_id
|
|
1122
|
+
ls["max_seq"] = max_mutation_seq
|
|
810
1123
|
ls["reset_sig"] = reset_sig
|
|
811
1124
|
return
|
|
812
|
-
if
|
|
1125
|
+
if (
|
|
1126
|
+
reset_sig != ls["reset_sig"]
|
|
1127
|
+
or max_entry_id < ls["max_id"]
|
|
1128
|
+
or max_mutation_seq < ls["max_seq"]
|
|
1129
|
+
):
|
|
813
1130
|
_WEEKREF_COST_CACHE.clear() # reset/credit, or cache.db rebuilt
|
|
814
1131
|
ls["max_id"] = max_entry_id
|
|
1132
|
+
ls["max_seq"] = max_mutation_seq
|
|
815
1133
|
ls["reset_sig"] = reset_sig
|
|
816
1134
|
return
|
|
817
|
-
if
|
|
818
|
-
wm =
|
|
1135
|
+
if max_mutation_seq > ls["max_seq"]:
|
|
1136
|
+
wm = changed_min_timestamp(cache_conn, ls["max_seq"])
|
|
819
1137
|
if wm is not None:
|
|
820
1138
|
for key in list(_WEEKREF_COST_CACHE):
|
|
821
1139
|
# key = (start_iso, end_iso); inclusive [start,end] window, so
|
|
822
|
-
# evict when the week's end >= the earliest
|
|
1140
|
+
# evict when the week's end >= the earliest changed event time.
|
|
823
1141
|
if dt.datetime.fromisoformat(key[1]) >= wm:
|
|
824
1142
|
del _WEEKREF_COST_CACHE[key]
|
|
825
1143
|
ls["max_id"] = max_entry_id
|
|
1144
|
+
ls["max_seq"] = max_mutation_seq
|
|
826
1145
|
ls["reset_sig"] = reset_sig
|
|
827
1146
|
|
|
828
1147
|
|
|
@@ -855,6 +1174,18 @@ _PROJECTS_ENV_WEEK_CACHE: dict = {} # {(bucket_path, week_iso): agg}
|
|
|
855
1174
|
_PROJECTS_ENV_WEEK_TOTALS: dict = {} # {week_iso: total_cost} (also the registry)
|
|
856
1175
|
_PROJECTS_ENV_LAST_SEEN: dict = {} # {"max_id", "max_wus_id", "sf_sig"}
|
|
857
1176
|
|
|
1177
|
+
# #271 M4 (spec §20): the CURRENT-week per-project aggregate is re-folded from
|
|
1178
|
+
# scratch every warm tick (the closed weeks are already cache-served). This
|
|
1179
|
+
# single-slot accumulator folds only the new-by-id delta each warm tick,
|
|
1180
|
+
# byte-identically — the exact Item-1 incremental single-left-fold, but per
|
|
1181
|
+
# project and simpler (the current-week window `[cw_start, cw_end]` is FIXED, not
|
|
1182
|
+
# `now`-clamped, so the delta predicate is purely `id > reconciled_max_id`; no
|
|
1183
|
+
# `last_now` machinery). Single-writer (sync thread only), module state, never
|
|
1184
|
+
# reachable from a published DataSnapshot (F7 — the snapshot holds the finalized
|
|
1185
|
+
# buckets, not the running `mut`). The `mut` is opaque here (the dashboard
|
|
1186
|
+
# packs/unpacks it — same "no dashboard import" discipline as `BucketCache`).
|
|
1187
|
+
_PROJECTS_ENV_CURRENT: dict = {"state": None} # single-slot current-week fold
|
|
1188
|
+
|
|
858
1189
|
|
|
859
1190
|
def projects_env_week_key(week_start):
|
|
860
1191
|
"""Canonical UTC-ISO key for a Monday-anchored subscription week start.
|
|
@@ -867,15 +1198,31 @@ def projects_env_week_key(week_start):
|
|
|
867
1198
|
|
|
868
1199
|
|
|
869
1200
|
def reset_projects_env_state():
|
|
870
|
-
"""Clear the projects-envelope week cache + totals + watermark
|
|
1201
|
+
"""Clear the projects-envelope week cache + totals + watermark + the
|
|
1202
|
+
CURRENT-week accumulator slot (#271 M4).
|
|
871
1203
|
|
|
872
1204
|
Called from the orphan-prune site (a prune deletes ``session_entries``
|
|
873
1205
|
possibly WITHOUT lowering ``MAX(id)``, so the reconcile's regression check
|
|
874
|
-
cannot catch it — the explicit clear must) and as a test hook.
|
|
1206
|
+
cannot catch it — the explicit clear must) and as a test hook. The
|
|
1207
|
+
current-week slot rides this same clear (spec §20): a prune can re-key a
|
|
1208
|
+
current-week bucket_path, so it must cold-refold next tick.
|
|
875
1209
|
"""
|
|
876
1210
|
_PROJECTS_ENV_WEEK_CACHE.clear()
|
|
877
1211
|
_PROJECTS_ENV_WEEK_TOTALS.clear()
|
|
878
1212
|
_PROJECTS_ENV_LAST_SEEN.clear()
|
|
1213
|
+
reset_projects_env_current_state()
|
|
1214
|
+
|
|
1215
|
+
|
|
1216
|
+
def reset_projects_env_current_state():
|
|
1217
|
+
"""Drop the projects-envelope CURRENT-week accumulator slot (#271 §20).
|
|
1218
|
+
|
|
1219
|
+
A standalone hook (mirrors ``reset_group_a_current_state``); also invoked by
|
|
1220
|
+
``reset_projects_env_state`` (prune site + test reset) and by
|
|
1221
|
+
``reconcile_projects_env_cache``'s full-clear branch, so a project-key remap
|
|
1222
|
+
/ prune / cache.db rebuild cold-refolds the current week the same tick the
|
|
1223
|
+
closed-week cache clears.
|
|
1224
|
+
"""
|
|
1225
|
+
_PROJECTS_ENV_CURRENT["state"] = None
|
|
879
1226
|
|
|
880
1227
|
|
|
881
1228
|
def session_files_sig(cache_conn) -> "tuple[int, int]":
|
|
@@ -888,6 +1235,15 @@ def session_files_sig(cache_conn) -> "tuple[int, int]":
|
|
|
888
1235
|
So the envelope cache keys this cheap change-signal and full-clears when it
|
|
889
1236
|
moves. Returns ``(0, 0)`` on a missing table (fresh DB) so callers never
|
|
890
1237
|
raise.
|
|
1238
|
+
|
|
1239
|
+
#271 §9d rider (from the #269 final review): this ``(COUNT(*), MAX(rowid))``
|
|
1240
|
+
leg does NOT by itself catch the in-place ``ON CONFLICT(path) DO UPDATE SET
|
|
1241
|
+
project_path = COALESCE(...)`` attribution backfill — that UPDATE preserves
|
|
1242
|
+
the rowid and the row count, so both legs are unmoved. It is covered
|
|
1243
|
+
belt-and-suspenders, though: the backfill lands in the SAME ``sync_cache``
|
|
1244
|
+
ingest-loop iteration as the file's new ``session_entries`` rows, which bump
|
|
1245
|
+
``max_entry_id`` — caught by the watermark eviction path. So a pure
|
|
1246
|
+
attribution move never both slips this signal and leaves the cache stale.
|
|
891
1247
|
"""
|
|
892
1248
|
try:
|
|
893
1249
|
row = cache_conn.execute(
|
|
@@ -932,47 +1288,189 @@ def projects_env_week_put(week_iso, buckets_by_bp, total) -> None:
|
|
|
932
1288
|
_PROJECTS_ENV_WEEK_CACHE[(bp, week_iso)] = agg
|
|
933
1289
|
|
|
934
1290
|
|
|
935
|
-
def
|
|
1291
|
+
def accumulate_projects_current_week(*, week_key, cur_max_id, cur_max_seq,
|
|
1292
|
+
fetch_all_raw, fetch_delta_rows,
|
|
1293
|
+
finalize, fold):
|
|
1294
|
+
"""Single-slot incremental fold of the projects-envelope CURRENT week (#271 §20).
|
|
1295
|
+
|
|
1296
|
+
Returns ``(finalized_buckets, week_total)`` — the exact
|
|
1297
|
+
``_aggregate_projects_week`` public shape — folding only the changed-row delta
|
|
1298
|
+
each warm tick instead of re-folding the whole ~12K-entry current week
|
|
1299
|
+
(the closed weeks are already cache-served). The dashboard injects the
|
|
1300
|
+
fold/finalize/fetch callables so this module keeps no dashboard import
|
|
1301
|
+
(the ``BucketCache`` / ``build_cached_group_a`` discipline).
|
|
1302
|
+
|
|
1303
|
+
Injected closures (all capture the current-week window ``[cw_start, cw_end]``
|
|
1304
|
+
and the live conn on the dashboard side):
|
|
1305
|
+
``fetch_all_raw() -> (mut_with_sessions_sets, week_total, tail)`` — the
|
|
1306
|
+
full-window raw fold (cold seed / fold-order fallback).
|
|
1307
|
+
``fetch_delta_rows(after_seq) -> list[row]`` — rows with
|
|
1308
|
+
``mutation_seq > after_seq`` in the current-week window (#270 §8 re-key),
|
|
1309
|
+
already membership/``<synthetic>``-filtered AND sorted
|
|
1310
|
+
``(timestamp_utc, id)``; ``row[0]`` is the ``id``, ``row[1]`` the
|
|
1311
|
+
``timestamp_utc``, so ``rows[0]`` is the minimum genuine current-week entry.
|
|
1312
|
+
``fold(mut, row) -> entry_cost | None`` — the shared ``_fold_projects_entry``
|
|
1313
|
+
(byte-identical arithmetic to the cold path).
|
|
1314
|
+
``finalize(mut) -> {bucket_path: agg}`` — the public finalize.
|
|
1315
|
+
|
|
1316
|
+
Simpler than the Group A accumulator (spec §20): the current-week aggregate
|
|
1317
|
+
is a pure function of ``(conn, cw_start)`` over the FIXED ``[cw_start, cw_end]``
|
|
1318
|
+
window (no moving-``now`` clamp), so there is no ``last_now`` field / empty-
|
|
1319
|
+
delta-by-time machinery.
|
|
1320
|
+
|
|
1321
|
+
#270 §8: the delta is re-keyed from ``id > reconciled_max_id`` to
|
|
1322
|
+
``mutation_seq > reconciled_max_seq`` — a strict superset (every insert
|
|
1323
|
+
carries a fresh seq monotone with id, so on a pure-insert tick the delta row
|
|
1324
|
+
set is byte-identical to today), so an id-stable in-place finalization of a
|
|
1325
|
+
current-week row now appears in the delta. But the incremental fold cannot
|
|
1326
|
+
un-fold that row's already-folded stale partial: any delta row that is a
|
|
1327
|
+
PRE-EXISTING row (``id <= reconciled_max_id``) discards the delta and forces a
|
|
1328
|
+
cold refold (checked BEFORE the fold-order gate, so a timestamp-non-decreasing
|
|
1329
|
+
finalization that sorts past ``tail`` is still caught). Genuine new inserts
|
|
1330
|
+
(``id > reconciled_max_id``) proceed through the unchanged fold-order gate +
|
|
1331
|
+
append.
|
|
1332
|
+
|
|
1333
|
+
- **Cold** (``prior is None``, ``label`` changed = Monday rollover / window
|
|
1334
|
+
slide, or ``cur_max_id < reconciled_max_id`` = cache.db rebuilt): seed the
|
|
1335
|
+
slot from ``fetch_all_raw()``; finalize and return.
|
|
1336
|
+
- **Warm**: fetch the ``mutation_seq > reconciled_max_seq`` delta (empty when
|
|
1337
|
+
``cur_max_seq`` did not advance — the fast path, no fetch). **Pre-existing-row
|
|
1338
|
+
cold-refold (#270 §8):** any delta ``row`` with ``row[0] <= reconciled_max_id``
|
|
1339
|
+
→ discard + cold refold. **Fold-order gate:** the delta is ``(ts_iso, id)``-
|
|
1340
|
+
sorted, so ``rows[0]`` is the minimum; if it sorts ``(ts_iso, id) <= tail`` a
|
|
1341
|
+
late older-timestamp backfill landed mid-week → discard the delta and cold-
|
|
1342
|
+
refold (byte-safe; first-row-only is sufficient under the total order).
|
|
1343
|
+
``tail is None`` ⇒ prior folded nothing ⇒ the delta is a pure suffix, so no
|
|
1344
|
+
gate. Otherwise ``fold`` each delta row onto ``prior.mut`` in order.
|
|
1345
|
+
- ``reconciled_max_id`` / ``reconciled_max_seq`` advance to
|
|
1346
|
+
``cur_max_id`` / ``cur_max_seq`` on EVERY path (cold, empty delta, append,
|
|
1347
|
+
fallback). They are the tick's GLOBAL ``MAX(session_entries.id)`` /
|
|
1348
|
+
``MAX(mutation_seq)``, DECOUPLED from ``tail`` (Codex-P2a): a quiet current
|
|
1349
|
+
week has a folded-max far below the global max because high ids/seqs land in
|
|
1350
|
+
OTHER weeks (recent backfills), so keying the delta on the folded-max would
|
|
1351
|
+
re-scan every tick and let the ~63ms floor creep back.
|
|
1352
|
+
|
|
1353
|
+
F7: the running ``mut`` is module state, never reachable from a published
|
|
1354
|
+
snapshot — ``finalize`` builds a FRESH bucket map each tick, so mutating
|
|
1355
|
+
``mut`` next tick cannot tear a prior snapshot.
|
|
1356
|
+
"""
|
|
1357
|
+
prior = _PROJECTS_ENV_CURRENT["state"]
|
|
1358
|
+
cold = (
|
|
1359
|
+
prior is None
|
|
1360
|
+
or prior["label"] != week_key
|
|
1361
|
+
or cur_max_id < prior["reconciled_max_id"]
|
|
1362
|
+
)
|
|
1363
|
+
rows: list = []
|
|
1364
|
+
if not cold:
|
|
1365
|
+
if cur_max_seq > prior["reconciled_max_seq"]:
|
|
1366
|
+
rows = fetch_delta_rows(prior["reconciled_max_seq"])
|
|
1367
|
+
# #270 §8: any PRE-EXISTING row (id <= reconciled_max_id) is an id-stable
|
|
1368
|
+
# in-place finalization the incremental fold cannot reconcile → cold
|
|
1369
|
+
# refold. Checked BEFORE the fold-order gate (a timestamp-non-decreasing
|
|
1370
|
+
# finalization sorts PAST tail and would evade that gate).
|
|
1371
|
+
if rows and any(r[0] <= prior["reconciled_max_id"] for r in rows):
|
|
1372
|
+
cold = True
|
|
1373
|
+
# Fold-order gate: rows are (ts_iso, id)-sorted so rows[0] is the
|
|
1374
|
+
# minimum; if it sorts <= tail, some genuinely-new row is out of the
|
|
1375
|
+
# from-scratch fold order → cold refold. (tail None ⇒ prior folded
|
|
1376
|
+
# nothing ⇒ the delta is a pure suffix; never gate against None.)
|
|
1377
|
+
elif (
|
|
1378
|
+
rows
|
|
1379
|
+
and prior["tail"] is not None
|
|
1380
|
+
and (rows[0][1], rows[0][0]) <= prior["tail"]
|
|
1381
|
+
):
|
|
1382
|
+
cold = True
|
|
1383
|
+
if cold:
|
|
1384
|
+
mut, week_total, tail = fetch_all_raw()
|
|
1385
|
+
_PROJECTS_ENV_CURRENT["state"] = {
|
|
1386
|
+
"label": week_key,
|
|
1387
|
+
"mut": mut,
|
|
1388
|
+
"week_total": week_total,
|
|
1389
|
+
"tail": tail,
|
|
1390
|
+
"reconciled_max_id": cur_max_id,
|
|
1391
|
+
"reconciled_max_seq": cur_max_seq,
|
|
1392
|
+
}
|
|
1393
|
+
return finalize(mut), week_total
|
|
1394
|
+
# Warm: append the delta onto the running mut in (ts_iso, id) order. The
|
|
1395
|
+
# week_total is its OWN entry-order accumulator (never re-summed from the
|
|
1396
|
+
# per-bucket costs — spec §14(a) non-association rule).
|
|
1397
|
+
mut = prior["mut"]
|
|
1398
|
+
week_total = prior["week_total"]
|
|
1399
|
+
tail = prior["tail"]
|
|
1400
|
+
for row in rows:
|
|
1401
|
+
entry_cost = fold(mut, row)
|
|
1402
|
+
if entry_cost is None: # defensive: fetch_delta_rows pre-filters
|
|
1403
|
+
continue
|
|
1404
|
+
week_total += entry_cost
|
|
1405
|
+
tail = (row[1], row[0])
|
|
1406
|
+
prior["week_total"] = week_total
|
|
1407
|
+
prior["tail"] = tail
|
|
1408
|
+
prior["reconciled_max_id"] = cur_max_id
|
|
1409
|
+
prior["reconciled_max_seq"] = cur_max_seq
|
|
1410
|
+
return finalize(mut), week_total
|
|
1411
|
+
|
|
1412
|
+
|
|
1413
|
+
def reconcile_projects_env_cache(cache_conn, *, max_entry_id, max_mutation_seq,
|
|
1414
|
+
max_wus_id, sf_sig):
|
|
936
1415
|
"""Once-per-non-idle-rebuild invalidation for the projects-envelope cache.
|
|
937
1416
|
|
|
938
1417
|
Driven by ``_tui_build_snapshot`` after the idle-path check, before the
|
|
939
1418
|
envelope builder runs, using the dispatch-signature legs (``max_entry_id``,
|
|
940
|
-
``max_wus_id``) + a ``session_files`` signal
|
|
1419
|
+
``max_mutation_seq``, ``max_wus_id``) + a ``session_files`` signal
|
|
1420
|
+
(``sf_sig``) passed in:
|
|
941
1421
|
|
|
942
1422
|
- Cold (no last-seen): record last-seen, return — no eviction.
|
|
943
|
-
- ``
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
1423
|
+
- ``sf_sig`` changed (attribution backfill, Codex-M4 P2) OR
|
|
1424
|
+
``max_entry_id < last_seen`` (cache.db rebuilt) OR
|
|
1425
|
+
``max_mutation_seq < last_seen_seq``: full clear. Conservative but
|
|
1426
|
+
byte-safe (recompute). ``max_wus_id`` stays tracked in last-seen
|
|
1427
|
+
but is deliberately NOT a full-clear trigger (#271 §9) — a `record-usage`
|
|
1428
|
+
write reuses this cost cache; the whole-envelope memo refreshes attribution.
|
|
1429
|
+
- ``max_mutation_seq > last_seen_seq`` (#270 §7c — the seq gate, so an
|
|
1430
|
+
id-stable in-place finalization with a flat ``max_entry_id`` still evicts):
|
|
1431
|
+
evict cached weeks (and their bucket rows + week total) whose
|
|
1432
|
+
``week_end (= parse(week_iso) + 7d)`` is
|
|
1433
|
+
``>= changed_min_timestamp(cache_conn, last_seen_seq)`` — a
|
|
1434
|
+
genuinely-changed row could fall inside them (F1 late-ingest / #270
|
|
1435
|
+
in-place). The bound is ``>=`` (Codex-1); over-eviction is byte-safe.
|
|
951
1436
|
|
|
952
1437
|
Idempotent within a tick: after the first call updates last-seen, a later
|
|
953
1438
|
call with the same signature sees no delta and no-ops (never re-running the
|
|
954
1439
|
watermark query). The short-lived ``cache_conn`` is used only for the
|
|
955
|
-
watermark query on the ``
|
|
1440
|
+
watermark query on the ``max_mutation_seq > last_seen_seq`` branch (Codex-4).
|
|
956
1441
|
"""
|
|
957
1442
|
ls = _PROJECTS_ENV_LAST_SEEN
|
|
958
1443
|
if not ls: # cold
|
|
959
1444
|
ls["max_id"] = max_entry_id
|
|
1445
|
+
ls["max_seq"] = max_mutation_seq
|
|
960
1446
|
ls["max_wus_id"] = max_wus_id
|
|
961
1447
|
ls["sf_sig"] = sf_sig
|
|
962
1448
|
return
|
|
963
1449
|
if (
|
|
964
|
-
|
|
965
|
-
or sf_sig != ls["sf_sig"]
|
|
1450
|
+
sf_sig != ls["sf_sig"]
|
|
966
1451
|
or max_entry_id < ls["max_id"]
|
|
1452
|
+
or max_mutation_seq < ls["max_seq"]
|
|
967
1453
|
):
|
|
1454
|
+
# NOTE (#271 §9): max_wus_id is deliberately NOT a full-clear trigger. The
|
|
1455
|
+
# cached per-(project, week) aggregates are session_entries-only; a WUS
|
|
1456
|
+
# bump (a `record-usage` write) changes only the attribution denominator,
|
|
1457
|
+
# which the whole-envelope memo (_PROJECTS_ENV_MEMO, still keyed on
|
|
1458
|
+
# max_wus_id) recomputes fresh on its own miss. Reusing this cost cache
|
|
1459
|
+
# across a WUS bump is byte-identical. Do NOT re-add
|
|
1460
|
+
# `max_wus_id != ls["max_wus_id"]` here.
|
|
968
1461
|
_PROJECTS_ENV_WEEK_CACHE.clear()
|
|
969
1462
|
_PROJECTS_ENV_WEEK_TOTALS.clear()
|
|
1463
|
+
# #271 M4 (spec §20): the current-week accumulator rides the SAME
|
|
1464
|
+
# full-clear signals (sf_sig attribution remap / max_entry_id regression)
|
|
1465
|
+
# — an sf_sig move can re-key a current-week bucket_path, so cold-refold.
|
|
1466
|
+
reset_projects_env_current_state()
|
|
970
1467
|
ls["max_id"] = max_entry_id
|
|
1468
|
+
ls["max_seq"] = max_mutation_seq
|
|
971
1469
|
ls["max_wus_id"] = max_wus_id
|
|
972
1470
|
ls["sf_sig"] = sf_sig
|
|
973
1471
|
return
|
|
974
|
-
if
|
|
975
|
-
wm =
|
|
1472
|
+
if max_mutation_seq > ls["max_seq"]:
|
|
1473
|
+
wm = changed_min_timestamp(cache_conn, ls["max_seq"])
|
|
976
1474
|
if wm is not None:
|
|
977
1475
|
dirty = [
|
|
978
1476
|
wk
|
|
@@ -984,5 +1482,316 @@ def reconcile_projects_env_cache(cache_conn, *, max_entry_id, max_wus_id, sf_sig
|
|
|
984
1482
|
for key in [k for k in _PROJECTS_ENV_WEEK_CACHE if k[1] == wk]:
|
|
985
1483
|
del _PROJECTS_ENV_WEEK_CACHE[key]
|
|
986
1484
|
ls["max_id"] = max_entry_id
|
|
1485
|
+
ls["max_seq"] = max_mutation_seq
|
|
987
1486
|
ls["max_wus_id"] = max_wus_id
|
|
988
1487
|
ls["sf_sig"] = sf_sig
|
|
1488
|
+
|
|
1489
|
+
|
|
1490
|
+
# === #271 M3 — Bug-K pre-credit segment cache (spec §18) =====================
|
|
1491
|
+
#
|
|
1492
|
+
# The dashboard Weekly panel's Bug-K synthesis (`_dashboard_build_weekly_periods`)
|
|
1493
|
+
# rebuilds, per in-place credit event, the pre-credit aggregate over the CLOSED
|
|
1494
|
+
# past interval `[original_start, effective)` on EVERY warm tick — a wide
|
|
1495
|
+
# `get_entries` re-fetch + re-cost (~100ms wall / 5 windows / ~42.6K entries on
|
|
1496
|
+
# the 314K-entry prod copy). Because `effective` is a historical credit moment
|
|
1497
|
+
# (always <= now), that aggregate is IMMUTABLE — the SAME "re-aggregate immutable
|
|
1498
|
+
# history every tick" pattern the #269 weekref cost cache (above, §4) fixed. Cache
|
|
1499
|
+
# it and recompute only when a genuine data change reaches the window.
|
|
1500
|
+
#
|
|
1501
|
+
# Module-level state mirrors the weekref cost cache exactly: a plain dict of
|
|
1502
|
+
# immutable `BugKSegment` values + a per-cache last-seen dict, mutated only on the
|
|
1503
|
+
# dashboard sync thread (single-writer). Every cached value is immutable and each
|
|
1504
|
+
# rebuild builds a FRESH `WeeklyPeriodRow` from the cached segment, never mutating
|
|
1505
|
+
# it (F7). The `models` payload is a FROZEN TUPLE of (model, cost) in first-seen
|
|
1506
|
+
# order (NOT a live dict — Codex-BK-4), so the row's stable cost-desc sort
|
|
1507
|
+
# tie-order (which depends on first-seen order) can never be mutated after caching.
|
|
1508
|
+
#
|
|
1509
|
+
# Id-stable in-place mutation (Codex-BK-1) — RESOLVED by #270. `sync_cache`'s
|
|
1510
|
+
# `ON CONFLICT(msg_id, req_id) DO UPDATE` can finalize a streaming-intermediate
|
|
1511
|
+
# entry in place (same `id`, changed tokens/cost/timestamp) inside a closed
|
|
1512
|
+
# pre-credit window, advancing NEITHER `max_entry_id` NOR the old
|
|
1513
|
+
# `new_min_timestamp` watermark → a stale segment. This WAS the same exposure the
|
|
1514
|
+
# weekref cost cache and the Group A past-bucket caches carried (spec §6 /
|
|
1515
|
+
# first-review Codex-2). #270 closed it across every #268/#269/#271 cache at once
|
|
1516
|
+
# with the durable `session_entries.mutation_seq` change stamp: it is folded into
|
|
1517
|
+
# `compute_signature` (the `entry_mutation_seq` leg, so an id-stable finalization
|
|
1518
|
+
# leaves the idle path) AND drives `reconcile_bugk_cache`'s seq gate + the
|
|
1519
|
+
# `changed_min_timestamp(mutation_min_ts)` watermark below (so the affected CLOSED
|
|
1520
|
+
# segment — including one the finalization's timestamp MOVED the row across — is
|
|
1521
|
+
# recomputed). Regression: `test_reconcile_bugk_idstable_update_evicts`.
|
|
1522
|
+
#
|
|
1523
|
+
# One accepted trade-off remains, identical to the shipped #269 weekref cost cache
|
|
1524
|
+
# (deliberately NOT closed — closing it would be inconsistent with the already-
|
|
1525
|
+
# shipped caches):
|
|
1526
|
+
# - Pricing-at-cache-time (Codex-BK-3). Caching the folded `pre_cost` means an
|
|
1527
|
+
# in-process embedded-pricing edit is not reflected in the Bug-K pre-credit cost
|
|
1528
|
+
# until the segment invalidates — the SAME dashboard-only trade-off the weekref
|
|
1529
|
+
# cost cache already makes (see its module note above). Acceptable because
|
|
1530
|
+
# embedded-pricing edits require a code change + process restart regardless.
|
|
1531
|
+
|
|
1532
|
+
|
|
1533
|
+
class BugKSegment(NamedTuple):
|
|
1534
|
+
"""Immutable folded pre-credit aggregate over a closed `[original_start,
|
|
1535
|
+
effective)` window (spec §18).
|
|
1536
|
+
|
|
1537
|
+
``models`` is a frozen tuple of ``(display_model, cost)`` in FIRST-SEEN order
|
|
1538
|
+
(Codex-BK-4); the caller re-derives the cost-desc-sorted ``model_breakdowns``
|
|
1539
|
+
fresh each tick from it, and the stable sort preserves that first-seen tie
|
|
1540
|
+
order byte-for-byte.
|
|
1541
|
+
"""
|
|
1542
|
+
|
|
1543
|
+
input: int
|
|
1544
|
+
output: int
|
|
1545
|
+
cache_create: int
|
|
1546
|
+
cache_read: int
|
|
1547
|
+
cost: float
|
|
1548
|
+
models: tuple # ((display_model, cost), ...) in first-seen order
|
|
1549
|
+
entry_count: int
|
|
1550
|
+
|
|
1551
|
+
|
|
1552
|
+
_BUGK_SEGMENT_CACHE: dict = {} # {(orig_start_utc_iso, eff_utc_iso): BugKSegment}
|
|
1553
|
+
_BUGK_SEGMENT_LAST_SEEN: dict = {} # {"max_id": int, "reset_sig": tuple}
|
|
1554
|
+
|
|
1555
|
+
|
|
1556
|
+
def _bugk_key(original_start_at, effective_at):
|
|
1557
|
+
"""Canonical UTC-ISO key for a pre-credit segment window (spec §18).
|
|
1558
|
+
|
|
1559
|
+
Normalizes both boundaries to UTC before serializing (exactly like
|
|
1560
|
+
``_weekref_key``), so two spellings of one window can't create duplicate
|
|
1561
|
+
entries. The RAW ISO strings are still used for the row output — this key is
|
|
1562
|
+
cache identity only.
|
|
1563
|
+
"""
|
|
1564
|
+
return (
|
|
1565
|
+
original_start_at.astimezone(dt.timezone.utc).isoformat(),
|
|
1566
|
+
effective_at.astimezone(dt.timezone.utc).isoformat(),
|
|
1567
|
+
)
|
|
1568
|
+
|
|
1569
|
+
|
|
1570
|
+
def reset_bugk_segment_state():
|
|
1571
|
+
"""Clear the Bug-K segment cache + its watermark (full invalidation).
|
|
1572
|
+
|
|
1573
|
+
Called from the orphan-prune site (a prune deletes ``session_entries``
|
|
1574
|
+
possibly WITHOUT lowering ``MAX(id)``, so the reconcile's max-id-regression
|
|
1575
|
+
check cannot catch it — the explicit clear must) and as a test hook.
|
|
1576
|
+
"""
|
|
1577
|
+
_BUGK_SEGMENT_CACHE.clear()
|
|
1578
|
+
_BUGK_SEGMENT_LAST_SEEN.clear()
|
|
1579
|
+
|
|
1580
|
+
|
|
1581
|
+
def cached_bugk_segment(*, key, compute):
|
|
1582
|
+
"""Get-or-compute one pre-credit segment aggregate (spec §18).
|
|
1583
|
+
|
|
1584
|
+
The ``[original_start, effective)`` window is ALWAYS a closed past interval
|
|
1585
|
+
(``effective`` is a historical credit moment), so it is always cacheable: a
|
|
1586
|
+
cache hit returns the stored ``BugKSegment``; a miss calls ``compute()`` — the
|
|
1587
|
+
caller's exact from-scratch fetch+fold closure, a ``(timestamp_utc, id)``-
|
|
1588
|
+
ordered fetch so the left-fold ``cost`` / first-seen ``models`` order is
|
|
1589
|
+
bit-identical to today's — and stores it. ``key`` is the canonical
|
|
1590
|
+
``_bugk_key``.
|
|
1591
|
+
"""
|
|
1592
|
+
hit = _BUGK_SEGMENT_CACHE.get(key)
|
|
1593
|
+
if hit is not None:
|
|
1594
|
+
return hit
|
|
1595
|
+
val = compute()
|
|
1596
|
+
_BUGK_SEGMENT_CACHE[key] = val
|
|
1597
|
+
return val
|
|
1598
|
+
|
|
1599
|
+
|
|
1600
|
+
def reconcile_bugk_cache(cache_conn, *, max_entry_id, max_mutation_seq, reset_sig):
|
|
1601
|
+
"""Once-per-non-idle-rebuild invalidation for the Bug-K segment cache (§18).
|
|
1602
|
+
|
|
1603
|
+
Driven by ``_tui_build_snapshot`` after the idle-path check, before the
|
|
1604
|
+
builders run, ALONGSIDE ``reconcile_weekref_cache``, using the dispatch-
|
|
1605
|
+
signature legs already computed for the idle decision (``max_entry_id`` +
|
|
1606
|
+
``max_mutation_seq`` + ``reset_sig`` passed in — no extra query for those):
|
|
1607
|
+
|
|
1608
|
+
- Cold (no last-seen): record last-seen, return — no eviction.
|
|
1609
|
+
- ``reset_sig`` changed (credit events / their ``effective`` moments moved)
|
|
1610
|
+
OR ``max_entry_id < last_seen`` (cache.db rebuilt out-of-process) OR
|
|
1611
|
+
``max_mutation_seq < last_seen_seq``: full ``clear()``. Credit events are
|
|
1612
|
+
rare, so a conservative full clear is correct and cheap; a max-id / seq
|
|
1613
|
+
regression means the ids no longer map to the same rows.
|
|
1614
|
+
- ``max_mutation_seq > last_seen_seq`` (#270 §7c — the seq gate, so an
|
|
1615
|
+
id-stable in-place finalization with a flat ``max_entry_id`` still evicts):
|
|
1616
|
+
evict segments whose ``effective`` is
|
|
1617
|
+
``> changed_min_timestamp(cache_conn, last_seen_seq)`` — a genuinely-changed
|
|
1618
|
+
row could fall inside them (F1 late-ingest / #270 in-place). The bound is
|
|
1619
|
+
``>`` (STRICT), NOT ``>=``, because the segment window is HALF-OPEN
|
|
1620
|
+
``[original_start, effective)`` (Codex-BK-5): a row EXACTLY at ``effective``
|
|
1621
|
+
is OUTSIDE the segment (never contributes), while a row at
|
|
1622
|
+
``original_start`` .. just-below ``effective`` evicts. This is the ONE
|
|
1623
|
+
semantic difference from the weekref cache's inclusive ``>=`` (that window
|
|
1624
|
+
is ``[start, end]``). Over-eviction is byte-safe (forces a recompute);
|
|
1625
|
+
normally ``wm`` is recent and nothing drops.
|
|
1626
|
+
|
|
1627
|
+
Idempotent within a tick: after the first call updates last-seen, a later call
|
|
1628
|
+
with the same signature sees no delta and no-ops (never re-running the
|
|
1629
|
+
watermark query). The short-lived ``cache_conn`` is used only for the
|
|
1630
|
+
``changed_min_timestamp`` query on the ``max_mutation_seq > last_seen_seq``
|
|
1631
|
+
branch (Codex-4 lifecycle from §4).
|
|
1632
|
+
"""
|
|
1633
|
+
ls = _BUGK_SEGMENT_LAST_SEEN
|
|
1634
|
+
if not ls: # cold
|
|
1635
|
+
ls["max_id"] = max_entry_id
|
|
1636
|
+
ls["max_seq"] = max_mutation_seq
|
|
1637
|
+
ls["reset_sig"] = reset_sig
|
|
1638
|
+
return
|
|
1639
|
+
if (
|
|
1640
|
+
reset_sig != ls["reset_sig"]
|
|
1641
|
+
or max_entry_id < ls["max_id"]
|
|
1642
|
+
or max_mutation_seq < ls["max_seq"]
|
|
1643
|
+
):
|
|
1644
|
+
_BUGK_SEGMENT_CACHE.clear() # a credit event moved, or cache.db rebuilt
|
|
1645
|
+
ls["max_id"] = max_entry_id
|
|
1646
|
+
ls["max_seq"] = max_mutation_seq
|
|
1647
|
+
ls["reset_sig"] = reset_sig
|
|
1648
|
+
return
|
|
1649
|
+
if max_mutation_seq > ls["max_seq"]:
|
|
1650
|
+
wm = changed_min_timestamp(cache_conn, ls["max_seq"])
|
|
1651
|
+
if wm is not None:
|
|
1652
|
+
for key in list(_BUGK_SEGMENT_CACHE):
|
|
1653
|
+
# key = (orig_start_iso, eff_iso); half-open [start, eff) window,
|
|
1654
|
+
# so evict when eff > the earliest changed event time (a row AT
|
|
1655
|
+
# eff is outside the segment; a row < eff could be inside it).
|
|
1656
|
+
if dt.datetime.fromisoformat(key[1]) > wm:
|
|
1657
|
+
del _BUGK_SEGMENT_CACHE[key]
|
|
1658
|
+
ls["max_id"] = max_entry_id
|
|
1659
|
+
ls["max_seq"] = max_mutation_seq
|
|
1660
|
+
ls["reset_sig"] = reset_sig
|
|
1661
|
+
|
|
1662
|
+
|
|
1663
|
+
# === #272 — cache-report per-day cache =====================================
|
|
1664
|
+
#
|
|
1665
|
+
# A per-day cache in front of the dashboard's `build_cache_report_snapshot`
|
|
1666
|
+
# warm-tick leg. Each closed day's raw aggregate + per-project partials is
|
|
1667
|
+
# an immutable `CachedCacheReportDay` (bin/_lib_cache_report.py §5); the
|
|
1668
|
+
# reconcile below invalidates it on the #270/#271 `mutation_seq` / `max_id`
|
|
1669
|
+
# / `reset_sig` / `sf_sig` / display-tz signals, mirroring
|
|
1670
|
+
# `reconcile_bugk_cache` + the projects-env `session_files_sig` leg.
|
|
1671
|
+
|
|
1672
|
+
_CACHE_REPORT_DAY_CACHE: dict = {} # {date_key: CachedCacheReportDay}
|
|
1673
|
+
_CACHE_REPORT_LAST_SEEN: dict = {} # {max_id, max_seq, reset_sig, sf_sig, tz_key}
|
|
1674
|
+
|
|
1675
|
+
|
|
1676
|
+
def reset_cache_report_state():
|
|
1677
|
+
"""Clear the cache-report per-day cache + its watermark (full invalidation).
|
|
1678
|
+
|
|
1679
|
+
Called from the orphan-prune site (a prune can delete ``session_entries``
|
|
1680
|
+
WITHOUT lowering ``MAX(id)`` / advancing ``mutation_seq``, which the
|
|
1681
|
+
reconcile's regression check cannot catch — the explicit clear must) and
|
|
1682
|
+
as a test hook. Mirrors ``reset_bugk_segment_state`` /
|
|
1683
|
+
``reset_projects_env_state``.
|
|
1684
|
+
"""
|
|
1685
|
+
_CACHE_REPORT_DAY_CACHE.clear()
|
|
1686
|
+
_CACHE_REPORT_LAST_SEEN.clear()
|
|
1687
|
+
|
|
1688
|
+
|
|
1689
|
+
def cache_report_day_get(date_key):
|
|
1690
|
+
"""Return the cached ``CachedCacheReportDay`` for ``date_key``, or ``None``."""
|
|
1691
|
+
return _CACHE_REPORT_DAY_CACHE.get(date_key)
|
|
1692
|
+
|
|
1693
|
+
|
|
1694
|
+
def cache_report_day_store(date_key, value) -> None:
|
|
1695
|
+
"""Store one CLOSED day's immutable ``CachedCacheReportDay`` (never mutated)."""
|
|
1696
|
+
_CACHE_REPORT_DAY_CACHE[date_key] = value
|
|
1697
|
+
|
|
1698
|
+
|
|
1699
|
+
def cache_report_day_evict_before(oldest_key) -> None:
|
|
1700
|
+
"""Drop cached days strictly older than ``oldest_key`` — the window-rolloff
|
|
1701
|
+
tail (#275).
|
|
1702
|
+
|
|
1703
|
+
``reconcile_cache_report_cache`` only evicts CLOSED days that CHANGED (``>=``
|
|
1704
|
+
the seq-gated change watermark); a day that simply rolls off the trailing edge
|
|
1705
|
+
of the ``[since, now]`` window is never touched by it and would linger in the
|
|
1706
|
+
module dict until a reset / tz-change / ``sf_sig`` / regression full-clear or
|
|
1707
|
+
``reset_cache_report_state()``. On a long-uptime dashboard that accretes ~1
|
|
1708
|
+
frozen ``CachedCacheReportDay`` per day. ``build_cache_report_snapshot`` calls
|
|
1709
|
+
this on its rare cold/rollover store tick with the window's oldest still-needed
|
|
1710
|
+
closed date, keeping the dict bounded to the live window. Over-eviction is
|
|
1711
|
+
byte-safe (a re-needed day just re-populates on the next cold tick), so the
|
|
1712
|
+
predicate is a plain lexical ``<`` on the ``YYYY-MM-DD`` keys.
|
|
1713
|
+
"""
|
|
1714
|
+
for date_key in [d for d in _CACHE_REPORT_DAY_CACHE if d < oldest_key]:
|
|
1715
|
+
del _CACHE_REPORT_DAY_CACHE[date_key]
|
|
1716
|
+
|
|
1717
|
+
|
|
1718
|
+
def reconcile_cache_report_cache(
|
|
1719
|
+
cache_conn, *, max_entry_id, max_mutation_seq, reset_sig, sf_sig, bucket_tz,
|
|
1720
|
+
tz_key,
|
|
1721
|
+
):
|
|
1722
|
+
"""Once-per-non-idle-rebuild invalidation for the cache-report cache (#272 §5).
|
|
1723
|
+
|
|
1724
|
+
The canonical four-step shape (mirrors ``reconcile_bugk_cache`` +
|
|
1725
|
+
``reconcile_weekref_cache``, plus the projects-env cache's
|
|
1726
|
+
``session_files_sig`` leg and a display-tz leg):
|
|
1727
|
+
|
|
1728
|
+
1. **Cold** (empty last-seen) → record
|
|
1729
|
+
``{max_id, max_seq, reset_sig, sf_sig, tz_key}``, return, no eviction.
|
|
1730
|
+
2. **Full-clear** on any of: ``reset_sig`` changed, ``sf_sig`` changed
|
|
1731
|
+
(Codex-1 — lazy ``session_files.project_path`` backfill can re-attribute
|
|
1732
|
+
a CLOSED day's by_project rows with no ``session_entries`` / seq change).
|
|
1733
|
+
INHERITED LIMITATION (accepted; matches the projects-env cache, same
|
|
1734
|
+
``sf_sig`` leg): ``sf_sig`` is ``(COUNT(*), MAX(rowid))``, so an in-place
|
|
1735
|
+
``ON CONFLICT(path) DO UPDATE SET project_path = COALESCE(...)`` backfill
|
|
1736
|
+
(rowid + count both stable — the actual lazy-backfill shape) does NOT
|
|
1737
|
+
move it. Such an UPDATE is only watermark-covered via the co-ingested new
|
|
1738
|
+
``session_entries`` rows, which carry the NEW day — so the seq-gated
|
|
1739
|
+
eviction reaches only ``>= that day`` and a re-attributed CLOSED day can
|
|
1740
|
+
stale-serve ``(unknown)`` until it rolls out of the window (or a reset
|
|
1741
|
+
full-clears). A framework-wide fix belongs with the shared ``sf_sig``
|
|
1742
|
+
leg, not #272.
|
|
1743
|
+
``tz_key`` changed (a display-tz change shifts every calendar-day
|
|
1744
|
+
boundary → all cached day keys invalid), ``max_entry_id`` regressed, or
|
|
1745
|
+
``max_mutation_seq`` regressed (cache.db rebuilt out-of-process) →
|
|
1746
|
+
``clear()``, update last-seen, return.
|
|
1747
|
+
3. **Seq-gated eviction** (``max_mutation_seq > last_seen_seq`` — the #270
|
|
1748
|
+
§7c seq gate, so an id-stable in-place finalization with a flat
|
|
1749
|
+
``max_entry_id`` still evicts): ``wm = changed_min_timestamp(cache_conn,
|
|
1750
|
+
last_seen_seq)`` (the one query on this branch; #270
|
|
1751
|
+
``min(old_ts, new_ts)`` change-time). Evict every cached day whose
|
|
1752
|
+
``date_key >= wm.astimezone(bucket_tz).strftime("%Y-%m-%d")``. The bound
|
|
1753
|
+
is ``>=`` (INCLUSIVE), NOT ``>`` — a day bucket is the CLOSED interval
|
|
1754
|
+
``[start, end]``, so a changed row landing anywhere on ``wm_day`` (or a
|
|
1755
|
+
later day) is inside that day and must evict it; and a cross-day
|
|
1756
|
+
finalization pulls ``mutation_min_ts`` back to ``min(old, new)``, i.e.
|
|
1757
|
+
the OLD day, so the OLD day (exactly at the watermark) evicts too. This
|
|
1758
|
+
is the ONE semantic difference from ``reconcile_bugk_cache``'s strict
|
|
1759
|
+
``>`` (its segment window is HALF-OPEN ``[start, effective)``).
|
|
1760
|
+
Over-eviction is byte-safe. Update last-seen.
|
|
1761
|
+
4. **Idempotent within a tick**: after the first call updates last-seen, a
|
|
1762
|
+
same-signature second call sees ``max_mutation_seq == last_seen_seq`` and
|
|
1763
|
+
no sig delta → no watermark re-query, no eviction.
|
|
1764
|
+
"""
|
|
1765
|
+
ls = _CACHE_REPORT_LAST_SEEN
|
|
1766
|
+
if not ls: # cold
|
|
1767
|
+
ls.update(
|
|
1768
|
+
max_id=max_entry_id, max_seq=max_mutation_seq,
|
|
1769
|
+
reset_sig=reset_sig, sf_sig=sf_sig, tz_key=tz_key,
|
|
1770
|
+
)
|
|
1771
|
+
return
|
|
1772
|
+
if (
|
|
1773
|
+
reset_sig != ls["reset_sig"]
|
|
1774
|
+
or sf_sig != ls["sf_sig"]
|
|
1775
|
+
or tz_key != ls["tz_key"]
|
|
1776
|
+
or max_entry_id < ls["max_id"]
|
|
1777
|
+
or max_mutation_seq < ls["max_seq"]
|
|
1778
|
+
):
|
|
1779
|
+
_CACHE_REPORT_DAY_CACHE.clear()
|
|
1780
|
+
ls.update(
|
|
1781
|
+
max_id=max_entry_id, max_seq=max_mutation_seq,
|
|
1782
|
+
reset_sig=reset_sig, sf_sig=sf_sig, tz_key=tz_key,
|
|
1783
|
+
)
|
|
1784
|
+
return
|
|
1785
|
+
if max_mutation_seq > ls["max_seq"]:
|
|
1786
|
+
wm = changed_min_timestamp(cache_conn, ls["max_seq"])
|
|
1787
|
+
if wm is not None:
|
|
1788
|
+
wm_day = wm.astimezone(bucket_tz).strftime("%Y-%m-%d")
|
|
1789
|
+
for date_key in list(_CACHE_REPORT_DAY_CACHE):
|
|
1790
|
+
# Inclusive [start, end] day bucket: a changed row on wm_day (or
|
|
1791
|
+
# any later day) falls inside that day, so evict date_key >= wm_day.
|
|
1792
|
+
if date_key >= wm_day:
|
|
1793
|
+
del _CACHE_REPORT_DAY_CACHE[date_key]
|
|
1794
|
+
ls.update(
|
|
1795
|
+
max_id=max_entry_id, max_seq=max_mutation_seq,
|
|
1796
|
+
reset_sig=reset_sig, sf_sig=sf_sig, tz_key=tz_key,
|
|
1797
|
+
)
|