cctally 1.61.0 → 1.63.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -48,10 +48,19 @@ import datetime as dt
48
48
  import os
49
49
  import sqlite3
50
50
  import threading
51
+ from dataclasses import dataclass
51
52
  from typing import TYPE_CHECKING, Callable, NamedTuple
52
53
 
53
54
  from _cctally_core import parse_iso_datetime
54
55
 
56
+ # #271: the current-bucket accumulator (below) reuses the aggregators' single
57
+ # per-entry fold primitive so the incremental append is byte-identical to the
58
+ # full pass (spec §6/§7). This is the ONE deliberate runtime coupling to
59
+ # _lib_aggregators — the cache primitives above stay decoupled (BucketUsage is
60
+ # still a TYPE_CHECKING-only hint). _lib_aggregators imports only _cctally_core,
61
+ # so there is no import cycle.
62
+ from _lib_aggregators import _fold_entry, _finalize_bucket, _new_bucket_acc
63
+
55
64
  if TYPE_CHECKING: # type hints only — no runtime coupling to the aggregators
56
65
  from _lib_aggregators import BucketUsage, ClaudeSessionUsage
57
66
 
@@ -72,6 +81,10 @@ class SnapshotSignature(NamedTuple):
72
81
  reset_sig: tuple[int, int]
73
82
  max_codex_id: int
74
83
  generation: int
84
+ # #270 §7a: the O(1) `cache_meta` mutation counter. An id-stable in-place
85
+ # finalization UPSERT advances this leg while `max_entry_id` stays flat, so
86
+ # the dashboard leaves the idle path and recomputes the affected bucket.
87
+ entry_mutation_seq: int = 0
75
88
 
76
89
 
77
90
  def _max_id(conn: sqlite3.Connection, table: str) -> int:
@@ -88,6 +101,32 @@ def _max_id(conn: sqlite3.Connection, table: str) -> int:
88
101
  return 0
89
102
 
90
103
 
104
+ def _entry_mutation_seq(conn: sqlite3.Connection) -> int:
105
+ """Current ``session_entries`` mutation counter from ``cache_meta`` (§7a).
106
+
107
+ Reads the O(1) KV counter (key ``session_entries_mutation_seq``) that
108
+ ``sync_cache`` bumps on every insert AND every WHERE-passing in-place UPSERT
109
+ (#270). This is the new ``entry_mutation_seq`` signature leg: an id-stable
110
+ finalization advances it even though ``MAX(session_entries.id)`` is flat, so
111
+ the dashboard leaves the idle path and recomputes the affected bucket.
112
+
113
+ An O(1) KV read — NOT a ``MAX(mutation_seq)`` full scan (that is exactly the
114
+ per-tick cost #268 removed). Degrades to 0 on a fresh / partially-migrated DB
115
+ where the ``cache_meta`` table or the key is absent (like ``_max_id`` /
116
+ ``_reset_sig``), so the signature never raises.
117
+ """
118
+ try:
119
+ row = conn.execute(
120
+ "SELECT value FROM cache_meta "
121
+ "WHERE key='session_entries_mutation_seq'"
122
+ ).fetchone()
123
+ except sqlite3.Error:
124
+ return 0
125
+ if row is None or row[0] is None:
126
+ return 0
127
+ return int(row[0])
128
+
129
+
91
130
  def _reset_sig(conn: sqlite3.Connection) -> tuple[int, int]:
92
131
  """Change-signal over the two reset-event tables combined (spec §3).
93
132
 
@@ -133,6 +172,7 @@ def compute_signature(
133
172
  reset_sig=_reset_sig(stats_conn),
134
173
  max_codex_id=_max_id(cache_conn, "codex_session_entries"),
135
174
  generation=int(generation),
175
+ entry_mutation_seq=_entry_mutation_seq(cache_conn),
136
176
  )
137
177
 
138
178
 
@@ -176,6 +216,53 @@ def new_min_timestamp(
176
216
  return parsed.astimezone(dt.timezone.utc)
177
217
 
178
218
 
219
+ def changed_min_timestamp(
220
+ cache_conn: sqlite3.Connection,
221
+ last_seen_seq: int,
222
+ ) -> "dt.datetime | None":
223
+ """Earliest event time among rows CHANGED since ``last_seen_seq`` (§7b).
224
+
225
+ Returns ``MIN(mutation_min_ts) WHERE mutation_seq > last_seen_seq`` as an
226
+ aware UTC datetime, or ``None`` when there are no such rows. Generalizes
227
+ ``new_min_timestamp`` from the ``id`` watermark to the #270 ``mutation_seq``
228
+ watermark; the same parse / normalize-to-UTC path and the same
229
+ ``None``-on-missing-table degrade.
230
+
231
+ Load-bearing byte-identity invariant: on a PURE-INSERT interval every changed
232
+ row is a fresh insert carrying a new ``id`` (> last_seen_id), a new
233
+ ``mutation_seq`` (> last_seen_seq), and ``mutation_min_ts == timestamp_utc``,
234
+ and no existing row was re-stamped — so ``{mutation_seq > last_seen_seq}`` ==
235
+ ``{id > last_seen_id}`` and ``MIN(mutation_min_ts)`` == ``MIN(timestamp_utc)``,
236
+ i.e. this equals ``new_min_timestamp(last_seen_id)`` (late-ingest reach-back
237
+ included). When an in-place update landed, the seq set additionally contains
238
+ the updated row, whose ``mutation_min_ts = min(all event times it has held)``
239
+ pulls the watermark back to the EARLIEST bucket it has touched, so both the
240
+ old bucket (stripped from) and the new bucket (added to) are recomputed;
241
+ over-marking the between-buckets is byte-safe. ``MIN()`` ignores the NULL
242
+ ``mutation_min_ts`` of any legacy seq-0 row (which this ``> last_seen_seq``
243
+ predicate never selects anyway).
244
+
245
+ Backed by ``idx_entries_mutation_seq(mutation_seq, mutation_min_ts)`` — an
246
+ index-only range-min over the handful of rows changed since the last tick, so
247
+ #268's per-tick full-scan is not reintroduced.
248
+ """
249
+ try:
250
+ row = cache_conn.execute(
251
+ "SELECT MIN(mutation_min_ts) FROM session_entries "
252
+ "WHERE mutation_seq > ?",
253
+ (last_seen_seq,),
254
+ ).fetchone()
255
+ except sqlite3.Error:
256
+ return None
257
+ if row is None or row[0] is None:
258
+ return None
259
+ # Parse via the repo's canonical ISO helper (both ``...Z`` and ``...+00:00``
260
+ # stored forms), then normalize to a UTC-tzinfo instant so downstream
261
+ # comparisons against UTC bucket boundaries are exact.
262
+ parsed = parse_iso_datetime(row[0], "session_entries.mutation_min_ts")
263
+ return parsed.astimezone(dt.timezone.utc)
264
+
265
+
179
266
  # === Task 0.3 — cache-generation counter ===================================
180
267
  #
181
268
  # A monotonic counter bumped by any path that deletes/rewrites history in
@@ -346,6 +433,148 @@ def reset_group_a_state() -> None:
346
433
  """
347
434
  _GROUP_A_CACHE.clear()
348
435
  _GROUP_A_LAST_SEEN.clear()
436
+ reset_group_a_current_state()
437
+
438
+
439
+ # === #271 — incremental current-bucket accumulator =========================
440
+ #
441
+ # The Group A builders (daily / monthly / weekly) re-fold the whole OPEN
442
+ # bucket every warm tick (`cached_buckets` recomputes `current_label` from a
443
+ # full window fetch). On a recency-dense instance that open week/month/day
444
+ # holds tens of thousands of entries, so the recompute dominates the residual
445
+ # warm rebuild (#271 §1). This holds a persisted single-left-fold accumulator
446
+ # per builder key: each warm tick folds only the DELTA (new-by-id OR
447
+ # newly-in-window-because-`now`-advanced) into the running aggregate, with a
448
+ # full-recompute fallback when a late older-timestamp row lands mid-bucket.
449
+ #
450
+ # Byte-identity rests on the pinned `(timestamp_utc, id)` fold order (#271 §5)
451
+ # + the shared `_fold_entry` primitive (§6): the incremental append reproduces
452
+ # the full left-fold exactly. Single-writer (sync thread only), module state,
453
+ # never reachable from a published DataSnapshot (F7 — the snapshot holds the
454
+ # finalized BucketUsage, not `acc`).
455
+
456
+
457
+ @dataclass
458
+ class CurrentBucketAccumulator:
459
+ """Persisted running fold of one Group A builder's CURRENT bucket (§7a)."""
460
+ label: str
461
+ acc: dict # running _new_bucket_acc() shape
462
+ tail: "tuple | None" # (timestamp_utc, id) of the last folded entry
463
+ last_seen_id: int # max session_entries.id reconciled (= cur_max_id)
464
+ last_seen_seq: int # #270 §8: the mutation_seq watermark (= cur_max_seq)
465
+ last_now: dt.datetime # now_utc upper bound used to clamp the last fold
466
+
467
+
468
+ _GROUP_A_CURRENT: "dict[str, CurrentBucketAccumulator]" = {}
469
+
470
+
471
+ def reset_group_a_current_state() -> None:
472
+ """Drop every builder's current-bucket accumulator (prune-site + full-invalidate)."""
473
+ _GROUP_A_CURRENT.clear()
474
+
475
+
476
+ def _finalize_or_none(label, acc, tail):
477
+ """Finalize ``acc`` into a ``BucketUsage``, or ``None`` when nothing was
478
+ folded (``tail is None`` ⇒ an empty/gap bucket, matching a from-scratch
479
+ ``aggregate_one`` returning ``None``)."""
480
+ return None if tail is None else _finalize_bucket(label, acc)
481
+
482
+
483
+ def accumulate_current_bucket(prior, *, current_label, cur_now, cur_max_id,
484
+ cur_max_seq, fetch_all, fetch_delta, membership,
485
+ mode="auto"):
486
+ """Pure §7b tick algorithm. Returns ``(BucketUsage | None, CurrentBucketAccumulator)``.
487
+
488
+ ``fetch_all() -> list[(id, UsageEntry)]`` over the whole current-bucket
489
+ window (cold / fallback); ``fetch_delta(after_seq, after_ts) ->
490
+ list[(id, UsageEntry)]`` the ``(mutation_seq > after_seq OR ts > after_ts)``
491
+ delta (#270 §8 re-key); both ordered ``(timestamp_utc, id)``.
492
+ ``membership(entry) -> bool`` keeps exactly the entries the full pass assigns
493
+ to ``current_label``.
494
+
495
+ #270 §8: the delta is keyed on ``mutation_seq`` (a strict superset of the old
496
+ ``id`` leg — every insert carries a fresh seq, so the insert path stays
497
+ byte-identical), so an id-stable in-place finalization of a current-bucket
498
+ row now appears in the delta. But the incremental fold cannot un-fold a stale
499
+ contribution: a finalization that keeps or increases its timestamp sorts
500
+ AT-OR-AFTER ``tail`` and the ``(ts,id) <= tail`` gate would append it,
501
+ double-counting the already-folded partial. So any delta row that is a
502
+ PRE-EXISTING row (``id <= reconciled_max_id``) forces a cold refold of the
503
+ whole current bucket instead. Genuine new inserts (``id > reconciled_max_id``)
504
+ proceed through the unchanged ``(ts,id)``-vs-``tail`` gate + append.
505
+ """
506
+ def _cold():
507
+ acc = _new_bucket_acc()
508
+ tail = None
509
+ for eid, e in fetch_all():
510
+ if e.model == "<synthetic>" or not membership(e):
511
+ continue
512
+ _fold_entry(acc, e, mode)
513
+ tail = (e.timestamp, eid)
514
+ return acc, tail
515
+
516
+ # Cold refold on: no prior, a bucket rollover (label change), OR a backward
517
+ # wall-clock step (`cur_now < prior.last_now`, e.g. an NTP adjustment). The
518
+ # current-bucket fetch window is clamped to `now_utc`; if `now` moves
519
+ # backward with no data change, the empty-delta fast path would reuse the
520
+ # larger prior fold set and OVER-count vs a from-scratch pass over the
521
+ # shrunken `[start, cur_now]` window. A graceful cold refold matches
522
+ # from-scratch byte-for-byte — never `assert`, a clock step must not crash
523
+ # the dashboard (#271 M1 review).
524
+ if (
525
+ prior is None
526
+ or prior.label != current_label
527
+ or cur_now < prior.last_now
528
+ ):
529
+ acc, tail = _cold()
530
+ new = CurrentBucketAccumulator(current_label, acc, tail, cur_max_id,
531
+ cur_max_seq, cur_now)
532
+ return _finalize_or_none(current_label, acc, tail), new
533
+
534
+ delta = [
535
+ (eid, e) for eid, e in fetch_delta(prior.last_seen_seq, prior.last_now)
536
+ if e.model != "<synthetic>" and membership(e)
537
+ ]
538
+ if not delta:
539
+ # From-scratch fold set unchanged since last tick → byte-identical.
540
+ new = CurrentBucketAccumulator(current_label, prior.acc, prior.tail,
541
+ cur_max_id, cur_max_seq, cur_now)
542
+ return _finalize_or_none(current_label, prior.acc, prior.tail), new
543
+
544
+ # #270 §8: any PRE-EXISTING row in the seq-delta (id <= reconciled_max_id) is
545
+ # an in-place update (a finalization UPSERT re-stamped its seq) — or the
546
+ # near-impossible clock-advance re-entry — that the incremental fold cannot
547
+ # safely reconcile (appending would double-count the already-folded partial).
548
+ # Discard the delta and cold-refold the whole current bucket (always
549
+ # byte-correct, bounded to this bucket). Checked BEFORE the fold-order gate so
550
+ # a timestamp-non-decreasing finalization (which sorts past `tail`, evading
551
+ # that gate) is caught here.
552
+ if any(eid <= prior.last_seen_id for eid, _e in delta):
553
+ acc, tail = _cold()
554
+ new = CurrentBucketAccumulator(current_label, acc, tail, cur_max_id,
555
+ cur_max_seq, cur_now)
556
+ return _finalize_or_none(current_label, acc, tail), new
557
+
558
+ # Guard: every folded delta row must sort AFTER the tail (a strict suffix).
559
+ # tail None ⇒ prior folded nothing ⇒ the delta IS the full member set (each
560
+ # current member has mutation_seq>last_seen_seq OR ts>last_now), so appending
561
+ # == full fold.
562
+ if prior.tail is not None and any(
563
+ (e.timestamp, eid) <= prior.tail for eid, e in delta
564
+ ):
565
+ acc, tail = _cold() # late-ingest mid-bucket fallback
566
+ new = CurrentBucketAccumulator(current_label, acc, tail, cur_max_id,
567
+ cur_max_seq, cur_now)
568
+ return _finalize_or_none(current_label, acc, tail), new
569
+
570
+ acc = prior.acc # mutate in place (module state, never published)
571
+ tail = prior.tail
572
+ for eid, e in delta: # fetch_delta returns (timestamp_utc, id) ascending
573
+ _fold_entry(acc, e, mode)
574
+ tail = (e.timestamp, eid)
575
+ new = CurrentBucketAccumulator(current_label, acc, tail, cur_max_id,
576
+ cur_max_seq, cur_now)
577
+ return _finalize_or_none(current_label, acc, tail), new
349
578
 
350
579
 
351
580
  def cached_buckets(
@@ -357,37 +586,48 @@ def cached_buckets(
357
586
  dirty_predicate: "Callable[[str], bool]",
358
587
  fetch_bucket_entries: "Callable[[str], list]",
359
588
  aggregate_one: "Callable[[str, list], object | None]",
589
+ current_override: "Callable[[], object | None] | None" = None,
360
590
  ) -> "list[object]":
361
591
  """Assemble one Group A builder's per-bucket aggregates (spec §5.1).
362
592
 
363
593
  For each label in ``all_bucket_labels`` (caller order — pass oldest→newest
364
594
  so the assembled list matches ``_aggregate_*``'s ascending-key output):
365
595
 
366
- - If the label is ``current_label`` (the open bucket) or ``dirty_predicate``
367
- returns True (the watermark reached it, or a forced recompute), recompute
368
- it WHOLE via ``aggregate_one(label, fetch_bucket_entries(label))`` — a
596
+ - If the label is ``current_label`` and ``current_override`` is supplied
597
+ (#271 §8a), the current bucket is produced by ``current_override()`` — the
598
+ incremental accumulator INSTEAD of the full ``aggregate_one`` recompute.
599
+ The override returns the finalized ``BucketUsage`` (or ``None`` for a
600
+ no-data current bucket, handled by the same gap-drop below).
601
+ - Else if the label is ``current_label`` or ``dirty_predicate`` returns True
602
+ (the watermark reached it, or a forced recompute), recompute it WHOLE via
603
+ ``aggregate_one(label, fetch_bucket_entries(label))`` — a
369
604
  timestamp-ordered fetch, so ``_aggregate_buckets`` first-seen model order
370
605
  reproduces the full-history pass byte-for-byte (Codex F3).
371
606
  - Otherwise serve the cached ``BucketUsage``; on a cache MISS (cold start /
372
607
  post-invalidation) recompute it the same way.
373
608
 
374
- ``aggregate_one`` returns ``None`` for a label with no data (a gap
375
- day/month/week): the bucket is omitted from the result and any stale cache
376
- entry for that label is evicted. The returned list therefore contains only
377
- buckets-with-data — exactly what ``_aggregate_daily/_monthly/_weekly`` over
378
- the full entry set returns — in ``all_bucket_labels`` order.
609
+ ``aggregate_one`` / ``current_override`` return ``None`` for a label with no
610
+ data (a gap day/month/week): the bucket is omitted from the result and any
611
+ stale cache entry for that label is evicted. The returned list therefore
612
+ contains only buckets-with-data — exactly what
613
+ ``_aggregate_daily/_monthly/_weekly`` over the full entry set returns — in
614
+ ``all_bucket_labels`` order.
379
615
 
380
616
  Values are stored/served as-is (immutable ``BucketUsage``); this loop never
381
- mutates a bucket in place (spec §7).
617
+ mutates a bucket in place (spec §7). ``current_override`` stays a pure hook:
618
+ the assembly order, past-bucket serve/cold-miss, and gap-drop are untouched.
382
619
  """
383
620
  result: "list[object]" = []
384
621
  for label in all_bucket_labels:
385
- recompute = (label == current_label) or dirty_predicate(label)
386
- bucket = None
387
- if not recompute:
388
- bucket = cache.get(builder_key, label)
389
- if bucket is None: # forced recompute OR cold miss
390
- bucket = aggregate_one(label, fetch_bucket_entries(label))
622
+ if label == current_label and current_override is not None:
623
+ bucket = current_override()
624
+ else:
625
+ recompute = (label == current_label) or dirty_predicate(label)
626
+ bucket = None
627
+ if not recompute:
628
+ bucket = cache.get(builder_key, label)
629
+ if bucket is None: # forced recompute OR cold miss
630
+ bucket = aggregate_one(label, fetch_bucket_entries(label))
391
631
  if bucket is None:
392
632
  # Gap bucket (no data): drop any stale cache entry, omit from output.
393
633
  cache.drop_from(builder_key, lambda lbl, _t=label: lbl == _t)
@@ -407,6 +647,11 @@ def build_cached_group_a(
407
647
  fetch_bucket_entries: "Callable[[str], list]",
408
648
  aggregate_one: "Callable[[str, list], object | None]",
409
649
  extra_signature: object = None,
650
+ use_current_accumulator: bool = False,
651
+ now_utc: "dt.datetime | None" = None,
652
+ current_all_fetch: "Callable[[str], list] | None" = None,
653
+ current_delta_fetch: "Callable[[str, int, dt.datetime], list] | None" = None,
654
+ membership_of: "Callable[[object], bool] | None" = None,
410
655
  ) -> "list[object]":
411
656
  """Stateful Group A assembly: invalidation + watermark + ``cached_buckets``.
412
657
 
@@ -427,13 +672,23 @@ def build_cached_group_a(
427
672
  order, and updates this builder's last-seen state.
428
673
  """
429
674
  cur_max_id = _max_id(cache_conn, "session_entries")
675
+ # #270 §7c: split max_id's two duties — the incremental watermark + "any
676
+ # new work?" gate move to the O(1) `cache_meta` mutation counter (an
677
+ # id-stable in-place finalization advances the seq while `max_id` stays
678
+ # flat), while `max_id` stays the regression backstop below.
679
+ cur_max_seq = _entry_mutation_seq(cache_conn)
430
680
  state = _GROUP_A_LAST_SEEN.get(builder_key)
431
681
  full_invalidate = state is not None and (
432
682
  state.get("extra") != extra_signature
433
683
  or cur_max_id < int(state.get("max_id", 0))
684
+ or cur_max_seq < int(state.get("max_seq", 0)) # belt-and-suspenders
434
685
  )
435
686
  if full_invalidate:
436
687
  _GROUP_A_CACHE.drop_from(builder_key, lambda _lbl: True)
688
+ # The current bucket's boundaries may have shifted (a weekly signature
689
+ # move / cache.db rebuild) → discard the accumulator so the override
690
+ # cold-refolds this tick (#271 §8c).
691
+ _GROUP_A_CURRENT.pop(builder_key, None)
437
692
 
438
693
  if state is None or full_invalidate:
439
694
  # Cold start OR post-invalidation: recompute every label. (Cold already
@@ -442,7 +697,11 @@ def build_cached_group_a(
442
697
  def dirty(_label: str) -> bool:
443
698
  return True
444
699
  else:
445
- new_min_ts = new_min_timestamp(cache_conn, int(state.get("max_id", 0)))
700
+ # #270 §7b: the change-aware watermark over `mutation_min_ts` (the
701
+ # earliest event time any row CHANGED-since-last-tick has held), so an
702
+ # id-stable finalization that landed in / moved to a closed bucket marks
703
+ # it dirty. Reduces to `new_min_timestamp(max_id)` on a pure-insert tick.
704
+ new_min_ts = changed_min_timestamp(cache_conn, int(state.get("max_seq", 0)))
446
705
 
447
706
  def dirty(label: str) -> bool:
448
707
  if new_min_ts is None:
@@ -450,6 +709,34 @@ def build_cached_group_a(
450
709
  end = bucket_end_of(label)
451
710
  return end is not None and end > new_min_ts
452
711
 
712
+ # #271 §8a/§8b: when the incremental accumulator is enabled, the current
713
+ # bucket is produced by folding only the delta each tick instead of a full
714
+ # re-aggregate. #270 §8: the delta's lower bound is now the mutation_seq
715
+ # watermark `cur_max_seq` (the same O(1) `cache_meta` counter the signature
716
+ # + Group A dirty-predicate read), reused via prior.last_seen_seq inside the
717
+ # accumulator, so an id-stable in-place finalization of a current-bucket row
718
+ # is caught (and cold-refolded). Gated ON only by the sync-thread
719
+ # `_group_a_*_buckets` closures; every other caller leaves it off
720
+ # (byte-identical to today).
721
+ current_override = None
722
+ if use_current_accumulator and current_label is not None:
723
+ def current_override():
724
+ prior = _GROUP_A_CURRENT.get(builder_key)
725
+ bucket, new_state = accumulate_current_bucket(
726
+ prior,
727
+ current_label=current_label,
728
+ cur_now=now_utc,
729
+ cur_max_id=cur_max_id,
730
+ cur_max_seq=cur_max_seq,
731
+ fetch_all=lambda: current_all_fetch(current_label),
732
+ fetch_delta=lambda aseq, ats: current_delta_fetch(
733
+ current_label, aseq, ats),
734
+ membership=membership_of,
735
+ mode="auto",
736
+ )
737
+ _GROUP_A_CURRENT[builder_key] = new_state
738
+ return bucket
739
+
453
740
  buckets = cached_buckets(
454
741
  builder_key,
455
742
  cache=_GROUP_A_CACHE,
@@ -458,8 +745,11 @@ def build_cached_group_a(
458
745
  dirty_predicate=dirty,
459
746
  fetch_bucket_entries=fetch_bucket_entries,
460
747
  aggregate_one=aggregate_one,
748
+ current_override=current_override,
461
749
  )
462
- _GROUP_A_LAST_SEEN[builder_key] = {"max_id": cur_max_id, "extra": extra_signature}
750
+ _GROUP_A_LAST_SEEN[builder_key] = {
751
+ "max_id": cur_max_id, "max_seq": cur_max_seq, "extra": extra_signature,
752
+ }
463
753
  return buckets
464
754
 
465
755
 
@@ -468,9 +758,9 @@ def build_cached_group_a(
468
758
 
469
759
  def affected_session_keys(
470
760
  cache_conn: sqlite3.Connection,
471
- last_seen_max_id: int,
761
+ last_seen_seq: int,
472
762
  ) -> "set[str]":
473
- """Resolved session identities for entries with ``id > last_seen`` (spec §5.2).
763
+ """Resolved session identities for entries CHANGED since ``last_seen_seq`` (§5.2).
474
764
 
475
765
  Mirrors ``_aggregate_claude_sessions`` grouping EXACTLY: identity is
476
766
  ``session_files.session_id`` when the ``LEFT JOIN`` on ``source_path``
@@ -485,14 +775,21 @@ def affected_session_keys(
485
775
  the join to ``session_files`` — so the returned keys key IDENTICALLY to
486
776
  the aggregator's session grouping (Codex F5). Returns an empty set on a
487
777
  missing table (fresh / partially-migrated DB) so callers never raise.
778
+
779
+ #270 (§7d, Codex-2c): keyed on ``mutation_seq > last_seen_seq``, NOT
780
+ ``id > last_seen`` — so an id-stable in-place finalization of an EXISTING
781
+ session's row (same id, advanced seq) surfaces that session as affected and
782
+ it re-aggregates. On a pure-insert interval every changed row carries a
783
+ fresh seq (> last_seen_seq) exactly when its id > last_seen, so the affected
784
+ set is byte-identical to the old id-keyed one.
488
785
  """
489
786
  try:
490
787
  rows = cache_conn.execute(
491
788
  "SELECT se.source_path, sf.session_id "
492
789
  "FROM session_entries se "
493
790
  "LEFT JOIN session_files sf ON sf.path = se.source_path "
494
- "WHERE se.id > ? AND se.model != '<synthetic>'",
495
- (last_seen_max_id,),
791
+ "WHERE se.mutation_seq > ? AND se.model != '<synthetic>'",
792
+ (last_seen_seq,),
496
793
  ).fetchall()
497
794
  except sqlite3.Error:
498
795
  return set()
@@ -567,25 +864,35 @@ def build_cached_sessions(
567
864
  fallback sessions), so the cache keys match ``affected_session_keys``.
568
865
  """
569
866
  cur_max_id = _max_id(cache_conn, "session_entries")
867
+ # #270 §7c/§7d: the warm "any new work?" gate + the affected-set query key
868
+ # on the `mutation_seq` counter (an id-stable in-place finalization of an
869
+ # EXISTING session advances the seq while `max_id` stays flat); `max_id`
870
+ # remains the cache.db-rebuild regression backstop.
871
+ cur_max_seq = _entry_mutation_seq(cache_conn)
570
872
  state = _SESSION_LAST_SEEN
571
873
  cold = (
572
874
  not state
573
875
  or state.get("extra") != extra_signature
574
876
  or cur_max_id < int(state.get("max_id", 0))
877
+ or cur_max_seq < int(state.get("max_seq", 0)) # belt-and-suspenders
575
878
  )
576
879
  if cold:
577
880
  _SESSION_CACHE.clear()
578
881
  for sess in aggregate_all():
579
882
  _SESSION_CACHE.put(sess.session_id, sess)
580
883
  else:
581
- last_seen = int(state.get("max_id", 0))
582
- if cur_max_id > last_seen:
583
- affected = affected_session_keys(cache_conn, last_seen)
884
+ last_seen_seq = int(state.get("max_seq", 0))
885
+ if cur_max_seq > last_seen_seq:
886
+ affected = affected_session_keys(cache_conn, last_seen_seq)
584
887
  if affected:
585
- for sess in reaggregate(last_seen, affected):
888
+ # `reaggregate` re-fetches each affected session's FULL in-window
889
+ # entry set (seq-keyed too — Codex-2c), so the aggregate is
890
+ # byte-identical to from-scratch even for an id-stable update.
891
+ for sess in reaggregate(last_seen_seq, affected):
586
892
  _SESSION_CACHE.put(sess.session_id, sess)
587
893
  _SESSION_LAST_SEEN.clear()
588
894
  _SESSION_LAST_SEEN["max_id"] = cur_max_id
895
+ _SESSION_LAST_SEEN["max_seq"] = cur_max_seq
589
896
  _SESSION_LAST_SEEN["extra"] = extra_signature
590
897
  return list(_SESSION_CACHE.get_all().values())
591
898
 
@@ -775,54 +1082,66 @@ def cached_weekref_cost(*, week_start_at, week_end_at, now_utc, compute):
775
1082
  return val
776
1083
 
777
1084
 
778
- def reconcile_weekref_cache(cache_conn, *, max_entry_id, reset_sig):
1085
+ def reconcile_weekref_cache(cache_conn, *, max_entry_id, max_mutation_seq, reset_sig):
779
1086
  """Once-per-non-idle-rebuild invalidation for the weekref-cost cache (spec §4).
780
1087
 
781
1088
  Driven by ``_tui_build_snapshot`` after the idle-path check, before the
782
1089
  builders run, using the already-computed dispatch-signature legs
783
- (``max_entry_id`` + ``reset_sig`` are passed in — no extra query for those):
1090
+ (``max_entry_id`` + ``max_mutation_seq`` + ``reset_sig`` are passed in — no
1091
+ extra query for those):
784
1092
 
785
1093
  - Cold (no last-seen): record last-seen, return — no eviction.
786
1094
  - ``reset_sig`` changed OR ``max_entry_id < last_seen`` (cache.db rebuilt
787
- out-of-process): full ``clear()``. A credit/reset re-shapes a past week's
788
- cost; reset events are rare, so a conservative full clear is correct and
789
- cheap. A max-id regression means the ids no longer map to the same rows.
790
- - ``max_entry_id > last_seen``: evict cached weeks whose ``week_end_at`` is
791
- ``>= new_min_timestamp(cache_conn, last_seen)`` — a genuinely-new row
792
- could fall inside them (F1 late-ingest). The bound is ``>=``, NOT ``>``,
793
- because ``_sum_cost_for_range`` / ``iter_entries`` sum an inclusive
1095
+ out-of-process) OR ``max_mutation_seq < last_seen_seq``: full ``clear()``.
1096
+ A credit/reset re-shapes a past week's cost; reset events are rare, so a
1097
+ conservative full clear is correct and cheap. A max-id / seq regression
1098
+ means the ids no longer map to the same rows.
1099
+ - ``max_mutation_seq > last_seen_seq`` (#270 §7c the seq gate, so an
1100
+ id-stable in-place finalization with a flat ``max_entry_id`` still
1101
+ evicts): evict cached weeks whose ``week_end_at`` is
1102
+ ``>= changed_min_timestamp(cache_conn, last_seen_seq)`` — a
1103
+ genuinely-changed row (new OR finalized-in-place) could fall inside them
1104
+ (F1 late-ingest / #270 in-place). The bound is ``>=``, NOT ``>``, because
1105
+ ``_sum_cost_for_range`` / ``iter_entries`` sum an inclusive
794
1106
  ``[start, end]`` window, so a row whose timestamp lands exactly on
795
1107
  ``week_end_at`` belongs to that week (Codex-1). Over-eviction is byte-safe
796
1108
  (forces a recompute). Normally ``wm`` is recent and no closed week drops.
797
1109
 
798
1110
  Idempotent within a tick: after the first call updates last-seen, a later
799
- call in the same tick sees no delta (``max_entry_id == last_seen``,
1111
+ call in the same tick sees no delta (``max_mutation_seq == last_seen_seq``,
800
1112
  ``reset_sig`` unchanged) and no-ops — never re-running the watermark query.
801
1113
 
802
- Connection lifecycle (Codex-4): the ``new_min_timestamp`` watermark query is
803
- the only use of ``cache_conn`` and runs only on the
804
- ``max_entry_id > last_seen`` branch; the caller passes a short-lived cache
805
- connection, opened for that query.
1114
+ Connection lifecycle (Codex-4): the ``changed_min_timestamp`` watermark query
1115
+ is the only use of ``cache_conn`` and runs only on the
1116
+ ``max_mutation_seq > last_seen_seq`` branch; the caller passes a short-lived
1117
+ cache connection, opened for that query.
806
1118
  """
807
1119
  ls = _WEEKREF_COST_LAST_SEEN
808
1120
  if not ls: # cold
809
1121
  ls["max_id"] = max_entry_id
1122
+ ls["max_seq"] = max_mutation_seq
810
1123
  ls["reset_sig"] = reset_sig
811
1124
  return
812
- if reset_sig != ls["reset_sig"] or max_entry_id < ls["max_id"]:
1125
+ if (
1126
+ reset_sig != ls["reset_sig"]
1127
+ or max_entry_id < ls["max_id"]
1128
+ or max_mutation_seq < ls["max_seq"]
1129
+ ):
813
1130
  _WEEKREF_COST_CACHE.clear() # reset/credit, or cache.db rebuilt
814
1131
  ls["max_id"] = max_entry_id
1132
+ ls["max_seq"] = max_mutation_seq
815
1133
  ls["reset_sig"] = reset_sig
816
1134
  return
817
- if max_entry_id > ls["max_id"]:
818
- wm = new_min_timestamp(cache_conn, ls["max_id"])
1135
+ if max_mutation_seq > ls["max_seq"]:
1136
+ wm = changed_min_timestamp(cache_conn, ls["max_seq"])
819
1137
  if wm is not None:
820
1138
  for key in list(_WEEKREF_COST_CACHE):
821
1139
  # key = (start_iso, end_iso); inclusive [start,end] window, so
822
- # evict when the week's end >= the earliest new event time.
1140
+ # evict when the week's end >= the earliest changed event time.
823
1141
  if dt.datetime.fromisoformat(key[1]) >= wm:
824
1142
  del _WEEKREF_COST_CACHE[key]
825
1143
  ls["max_id"] = max_entry_id
1144
+ ls["max_seq"] = max_mutation_seq
826
1145
  ls["reset_sig"] = reset_sig
827
1146
 
828
1147
 
@@ -855,6 +1174,18 @@ _PROJECTS_ENV_WEEK_CACHE: dict = {} # {(bucket_path, week_iso): agg}
855
1174
  _PROJECTS_ENV_WEEK_TOTALS: dict = {} # {week_iso: total_cost} (also the registry)
856
1175
  _PROJECTS_ENV_LAST_SEEN: dict = {} # {"max_id", "max_wus_id", "sf_sig"}
857
1176
 
1177
+ # #271 M4 (spec §20): the CURRENT-week per-project aggregate is re-folded from
1178
+ # scratch every warm tick (the closed weeks are already cache-served). This
1179
+ # single-slot accumulator folds only the new-by-id delta each warm tick,
1180
+ # byte-identically — the exact Item-1 incremental single-left-fold, but per
1181
+ # project and simpler (the current-week window `[cw_start, cw_end]` is FIXED, not
1182
+ # `now`-clamped, so the delta predicate is purely `id > reconciled_max_id`; no
1183
+ # `last_now` machinery). Single-writer (sync thread only), module state, never
1184
+ # reachable from a published DataSnapshot (F7 — the snapshot holds the finalized
1185
+ # buckets, not the running `mut`). The `mut` is opaque here (the dashboard
1186
+ # packs/unpacks it — same "no dashboard import" discipline as `BucketCache`).
1187
+ _PROJECTS_ENV_CURRENT: dict = {"state": None} # single-slot current-week fold
1188
+
858
1189
 
859
1190
  def projects_env_week_key(week_start):
860
1191
  """Canonical UTC-ISO key for a Monday-anchored subscription week start.
@@ -867,15 +1198,31 @@ def projects_env_week_key(week_start):
867
1198
 
868
1199
 
869
1200
  def reset_projects_env_state():
870
- """Clear the projects-envelope week cache + totals + watermark.
1201
+ """Clear the projects-envelope week cache + totals + watermark + the
1202
+ CURRENT-week accumulator slot (#271 M4).
871
1203
 
872
1204
  Called from the orphan-prune site (a prune deletes ``session_entries``
873
1205
  possibly WITHOUT lowering ``MAX(id)``, so the reconcile's regression check
874
- cannot catch it — the explicit clear must) and as a test hook.
1206
+ cannot catch it — the explicit clear must) and as a test hook. The
1207
+ current-week slot rides this same clear (spec §20): a prune can re-key a
1208
+ current-week bucket_path, so it must cold-refold next tick.
875
1209
  """
876
1210
  _PROJECTS_ENV_WEEK_CACHE.clear()
877
1211
  _PROJECTS_ENV_WEEK_TOTALS.clear()
878
1212
  _PROJECTS_ENV_LAST_SEEN.clear()
1213
+ reset_projects_env_current_state()
1214
+
1215
+
1216
+ def reset_projects_env_current_state():
1217
+ """Drop the projects-envelope CURRENT-week accumulator slot (#271 §20).
1218
+
1219
+ A standalone hook (mirrors ``reset_group_a_current_state``); also invoked by
1220
+ ``reset_projects_env_state`` (prune site + test reset) and by
1221
+ ``reconcile_projects_env_cache``'s full-clear branch, so a project-key remap
1222
+ / prune / cache.db rebuild cold-refolds the current week the same tick the
1223
+ closed-week cache clears.
1224
+ """
1225
+ _PROJECTS_ENV_CURRENT["state"] = None
879
1226
 
880
1227
 
881
1228
  def session_files_sig(cache_conn) -> "tuple[int, int]":
@@ -888,6 +1235,15 @@ def session_files_sig(cache_conn) -> "tuple[int, int]":
888
1235
  So the envelope cache keys this cheap change-signal and full-clears when it
889
1236
  moves. Returns ``(0, 0)`` on a missing table (fresh DB) so callers never
890
1237
  raise.
1238
+
1239
+ #271 §9d rider (from the #269 final review): this ``(COUNT(*), MAX(rowid))``
1240
+ leg does NOT by itself catch the in-place ``ON CONFLICT(path) DO UPDATE SET
1241
+ project_path = COALESCE(...)`` attribution backfill — that UPDATE preserves
1242
+ the rowid and the row count, so both legs are unmoved. It is covered
1243
+ belt-and-suspenders, though: the backfill lands in the SAME ``sync_cache``
1244
+ ingest-loop iteration as the file's new ``session_entries`` rows, which bump
1245
+ ``max_entry_id`` — caught by the watermark eviction path. So a pure
1246
+ attribution move never both slips this signal and leaves the cache stale.
891
1247
  """
892
1248
  try:
893
1249
  row = cache_conn.execute(
@@ -932,47 +1288,189 @@ def projects_env_week_put(week_iso, buckets_by_bp, total) -> None:
932
1288
  _PROJECTS_ENV_WEEK_CACHE[(bp, week_iso)] = agg
933
1289
 
934
1290
 
935
- def reconcile_projects_env_cache(cache_conn, *, max_entry_id, max_wus_id, sf_sig):
1291
+ def accumulate_projects_current_week(*, week_key, cur_max_id, cur_max_seq,
1292
+ fetch_all_raw, fetch_delta_rows,
1293
+ finalize, fold):
1294
+ """Single-slot incremental fold of the projects-envelope CURRENT week (#271 §20).
1295
+
1296
+ Returns ``(finalized_buckets, week_total)`` — the exact
1297
+ ``_aggregate_projects_week`` public shape — folding only the changed-row delta
1298
+ each warm tick instead of re-folding the whole ~12K-entry current week
1299
+ (the closed weeks are already cache-served). The dashboard injects the
1300
+ fold/finalize/fetch callables so this module keeps no dashboard import
1301
+ (the ``BucketCache`` / ``build_cached_group_a`` discipline).
1302
+
1303
+ Injected closures (all capture the current-week window ``[cw_start, cw_end]``
1304
+ and the live conn on the dashboard side):
1305
+ ``fetch_all_raw() -> (mut_with_sessions_sets, week_total, tail)`` — the
1306
+ full-window raw fold (cold seed / fold-order fallback).
1307
+ ``fetch_delta_rows(after_seq) -> list[row]`` — rows with
1308
+ ``mutation_seq > after_seq`` in the current-week window (#270 §8 re-key),
1309
+ already membership/``<synthetic>``-filtered AND sorted
1310
+ ``(timestamp_utc, id)``; ``row[0]`` is the ``id``, ``row[1]`` the
1311
+ ``timestamp_utc``, so ``rows[0]`` is the minimum genuine current-week entry.
1312
+ ``fold(mut, row) -> entry_cost | None`` — the shared ``_fold_projects_entry``
1313
+ (byte-identical arithmetic to the cold path).
1314
+ ``finalize(mut) -> {bucket_path: agg}`` — the public finalize.
1315
+
1316
+ Simpler than the Group A accumulator (spec §20): the current-week aggregate
1317
+ is a pure function of ``(conn, cw_start)`` over the FIXED ``[cw_start, cw_end]``
1318
+ window (no moving-``now`` clamp), so there is no ``last_now`` field / empty-
1319
+ delta-by-time machinery.
1320
+
1321
+ #270 §8: the delta is re-keyed from ``id > reconciled_max_id`` to
1322
+ ``mutation_seq > reconciled_max_seq`` — a strict superset (every insert
1323
+ carries a fresh seq monotone with id, so on a pure-insert tick the delta row
1324
+ set is byte-identical to today), so an id-stable in-place finalization of a
1325
+ current-week row now appears in the delta. But the incremental fold cannot
1326
+ un-fold that row's already-folded stale partial: any delta row that is a
1327
+ PRE-EXISTING row (``id <= reconciled_max_id``) discards the delta and forces a
1328
+ cold refold (checked BEFORE the fold-order gate, so a timestamp-non-decreasing
1329
+ finalization that sorts past ``tail`` is still caught). Genuine new inserts
1330
+ (``id > reconciled_max_id``) proceed through the unchanged fold-order gate +
1331
+ append.
1332
+
1333
+ - **Cold** (``prior is None``, ``label`` changed = Monday rollover / window
1334
+ slide, or ``cur_max_id < reconciled_max_id`` = cache.db rebuilt): seed the
1335
+ slot from ``fetch_all_raw()``; finalize and return.
1336
+ - **Warm**: fetch the ``mutation_seq > reconciled_max_seq`` delta (empty when
1337
+ ``cur_max_seq`` did not advance — the fast path, no fetch). **Pre-existing-row
1338
+ cold-refold (#270 §8):** any delta ``row`` with ``row[0] <= reconciled_max_id``
1339
+ → discard + cold refold. **Fold-order gate:** the delta is ``(ts_iso, id)``-
1340
+ sorted, so ``rows[0]`` is the minimum; if it sorts ``(ts_iso, id) <= tail`` a
1341
+ late older-timestamp backfill landed mid-week → discard the delta and cold-
1342
+ refold (byte-safe; first-row-only is sufficient under the total order).
1343
+ ``tail is None`` ⇒ prior folded nothing ⇒ the delta is a pure suffix, so no
1344
+ gate. Otherwise ``fold`` each delta row onto ``prior.mut`` in order.
1345
+ - ``reconciled_max_id`` / ``reconciled_max_seq`` advance to
1346
+ ``cur_max_id`` / ``cur_max_seq`` on EVERY path (cold, empty delta, append,
1347
+ fallback). They are the tick's GLOBAL ``MAX(session_entries.id)`` /
1348
+ ``MAX(mutation_seq)``, DECOUPLED from ``tail`` (Codex-P2a): a quiet current
1349
+ week has a folded-max far below the global max because high ids/seqs land in
1350
+ OTHER weeks (recent backfills), so keying the delta on the folded-max would
1351
+ re-scan every tick and let the ~63ms floor creep back.
1352
+
1353
+ F7: the running ``mut`` is module state, never reachable from a published
1354
+ snapshot — ``finalize`` builds a FRESH bucket map each tick, so mutating
1355
+ ``mut`` next tick cannot tear a prior snapshot.
1356
+ """
1357
+ prior = _PROJECTS_ENV_CURRENT["state"]
1358
+ cold = (
1359
+ prior is None
1360
+ or prior["label"] != week_key
1361
+ or cur_max_id < prior["reconciled_max_id"]
1362
+ )
1363
+ rows: list = []
1364
+ if not cold:
1365
+ if cur_max_seq > prior["reconciled_max_seq"]:
1366
+ rows = fetch_delta_rows(prior["reconciled_max_seq"])
1367
+ # #270 §8: any PRE-EXISTING row (id <= reconciled_max_id) is an id-stable
1368
+ # in-place finalization the incremental fold cannot reconcile → cold
1369
+ # refold. Checked BEFORE the fold-order gate (a timestamp-non-decreasing
1370
+ # finalization sorts PAST tail and would evade that gate).
1371
+ if rows and any(r[0] <= prior["reconciled_max_id"] for r in rows):
1372
+ cold = True
1373
+ # Fold-order gate: rows are (ts_iso, id)-sorted so rows[0] is the
1374
+ # minimum; if it sorts <= tail, some genuinely-new row is out of the
1375
+ # from-scratch fold order → cold refold. (tail None ⇒ prior folded
1376
+ # nothing ⇒ the delta is a pure suffix; never gate against None.)
1377
+ elif (
1378
+ rows
1379
+ and prior["tail"] is not None
1380
+ and (rows[0][1], rows[0][0]) <= prior["tail"]
1381
+ ):
1382
+ cold = True
1383
+ if cold:
1384
+ mut, week_total, tail = fetch_all_raw()
1385
+ _PROJECTS_ENV_CURRENT["state"] = {
1386
+ "label": week_key,
1387
+ "mut": mut,
1388
+ "week_total": week_total,
1389
+ "tail": tail,
1390
+ "reconciled_max_id": cur_max_id,
1391
+ "reconciled_max_seq": cur_max_seq,
1392
+ }
1393
+ return finalize(mut), week_total
1394
+ # Warm: append the delta onto the running mut in (ts_iso, id) order. The
1395
+ # week_total is its OWN entry-order accumulator (never re-summed from the
1396
+ # per-bucket costs — spec §14(a) non-association rule).
1397
+ mut = prior["mut"]
1398
+ week_total = prior["week_total"]
1399
+ tail = prior["tail"]
1400
+ for row in rows:
1401
+ entry_cost = fold(mut, row)
1402
+ if entry_cost is None: # defensive: fetch_delta_rows pre-filters
1403
+ continue
1404
+ week_total += entry_cost
1405
+ tail = (row[1], row[0])
1406
+ prior["week_total"] = week_total
1407
+ prior["tail"] = tail
1408
+ prior["reconciled_max_id"] = cur_max_id
1409
+ prior["reconciled_max_seq"] = cur_max_seq
1410
+ return finalize(mut), week_total
1411
+
1412
+
1413
+ def reconcile_projects_env_cache(cache_conn, *, max_entry_id, max_mutation_seq,
1414
+ max_wus_id, sf_sig):
936
1415
  """Once-per-non-idle-rebuild invalidation for the projects-envelope cache.
937
1416
 
938
1417
  Driven by ``_tui_build_snapshot`` after the idle-path check, before the
939
1418
  envelope builder runs, using the dispatch-signature legs (``max_entry_id``,
940
- ``max_wus_id``) + a ``session_files`` signal (``sf_sig``) passed in:
1419
+ ``max_mutation_seq``, ``max_wus_id``) + a ``session_files`` signal
1420
+ (``sf_sig``) passed in:
941
1421
 
942
1422
  - Cold (no last-seen): record last-seen, return — no eviction.
943
- - ``max_wus_id`` changed (attribution denominator), ``sf_sig`` changed
944
- (attribution backfill, Codex-M4 P2), OR ``max_entry_id < last_seen``
945
- (cache.db rebuilt): full clear. Conservative but byte-safe (recompute).
946
- - ``max_entry_id > last_seen``: evict cached weeks (and their bucket rows +
947
- week total) whose ``week_end (= parse(week_iso) + 7d)`` is
948
- ``>= new_min_timestamp(cache_conn, last_seen)`` a genuinely-new row
949
- could fall inside them (F1 late-ingest). The bound is ``>=`` (Codex-1);
950
- over-eviction is byte-safe.
1423
+ - ``sf_sig`` changed (attribution backfill, Codex-M4 P2) OR
1424
+ ``max_entry_id < last_seen`` (cache.db rebuilt) OR
1425
+ ``max_mutation_seq < last_seen_seq``: full clear. Conservative but
1426
+ byte-safe (recompute). ``max_wus_id`` stays tracked in last-seen
1427
+ but is deliberately NOT a full-clear trigger (#271 §9) a `record-usage`
1428
+ write reuses this cost cache; the whole-envelope memo refreshes attribution.
1429
+ - ``max_mutation_seq > last_seen_seq`` (#270 §7c the seq gate, so an
1430
+ id-stable in-place finalization with a flat ``max_entry_id`` still evicts):
1431
+ evict cached weeks (and their bucket rows + week total) whose
1432
+ ``week_end (= parse(week_iso) + 7d)`` is
1433
+ ``>= changed_min_timestamp(cache_conn, last_seen_seq)`` — a
1434
+ genuinely-changed row could fall inside them (F1 late-ingest / #270
1435
+ in-place). The bound is ``>=`` (Codex-1); over-eviction is byte-safe.
951
1436
 
952
1437
  Idempotent within a tick: after the first call updates last-seen, a later
953
1438
  call with the same signature sees no delta and no-ops (never re-running the
954
1439
  watermark query). The short-lived ``cache_conn`` is used only for the
955
- watermark query on the ``max_entry_id > last_seen`` branch (Codex-4).
1440
+ watermark query on the ``max_mutation_seq > last_seen_seq`` branch (Codex-4).
956
1441
  """
957
1442
  ls = _PROJECTS_ENV_LAST_SEEN
958
1443
  if not ls: # cold
959
1444
  ls["max_id"] = max_entry_id
1445
+ ls["max_seq"] = max_mutation_seq
960
1446
  ls["max_wus_id"] = max_wus_id
961
1447
  ls["sf_sig"] = sf_sig
962
1448
  return
963
1449
  if (
964
- max_wus_id != ls["max_wus_id"]
965
- or sf_sig != ls["sf_sig"]
1450
+ sf_sig != ls["sf_sig"]
966
1451
  or max_entry_id < ls["max_id"]
1452
+ or max_mutation_seq < ls["max_seq"]
967
1453
  ):
1454
+ # NOTE (#271 §9): max_wus_id is deliberately NOT a full-clear trigger. The
1455
+ # cached per-(project, week) aggregates are session_entries-only; a WUS
1456
+ # bump (a `record-usage` write) changes only the attribution denominator,
1457
+ # which the whole-envelope memo (_PROJECTS_ENV_MEMO, still keyed on
1458
+ # max_wus_id) recomputes fresh on its own miss. Reusing this cost cache
1459
+ # across a WUS bump is byte-identical. Do NOT re-add
1460
+ # `max_wus_id != ls["max_wus_id"]` here.
968
1461
  _PROJECTS_ENV_WEEK_CACHE.clear()
969
1462
  _PROJECTS_ENV_WEEK_TOTALS.clear()
1463
+ # #271 M4 (spec §20): the current-week accumulator rides the SAME
1464
+ # full-clear signals (sf_sig attribution remap / max_entry_id regression)
1465
+ # — an sf_sig move can re-key a current-week bucket_path, so cold-refold.
1466
+ reset_projects_env_current_state()
970
1467
  ls["max_id"] = max_entry_id
1468
+ ls["max_seq"] = max_mutation_seq
971
1469
  ls["max_wus_id"] = max_wus_id
972
1470
  ls["sf_sig"] = sf_sig
973
1471
  return
974
- if max_entry_id > ls["max_id"]:
975
- wm = new_min_timestamp(cache_conn, ls["max_id"])
1472
+ if max_mutation_seq > ls["max_seq"]:
1473
+ wm = changed_min_timestamp(cache_conn, ls["max_seq"])
976
1474
  if wm is not None:
977
1475
  dirty = [
978
1476
  wk
@@ -984,5 +1482,316 @@ def reconcile_projects_env_cache(cache_conn, *, max_entry_id, max_wus_id, sf_sig
984
1482
  for key in [k for k in _PROJECTS_ENV_WEEK_CACHE if k[1] == wk]:
985
1483
  del _PROJECTS_ENV_WEEK_CACHE[key]
986
1484
  ls["max_id"] = max_entry_id
1485
+ ls["max_seq"] = max_mutation_seq
987
1486
  ls["max_wus_id"] = max_wus_id
988
1487
  ls["sf_sig"] = sf_sig
1488
+
1489
+
1490
+ # === #271 M3 — Bug-K pre-credit segment cache (spec §18) =====================
1491
+ #
1492
+ # The dashboard Weekly panel's Bug-K synthesis (`_dashboard_build_weekly_periods`)
1493
+ # rebuilds, per in-place credit event, the pre-credit aggregate over the CLOSED
1494
+ # past interval `[original_start, effective)` on EVERY warm tick — a wide
1495
+ # `get_entries` re-fetch + re-cost (~100ms wall / 5 windows / ~42.6K entries on
1496
+ # the 314K-entry prod copy). Because `effective` is a historical credit moment
1497
+ # (always <= now), that aggregate is IMMUTABLE — the SAME "re-aggregate immutable
1498
+ # history every tick" pattern the #269 weekref cost cache (above, §4) fixed. Cache
1499
+ # it and recompute only when a genuine data change reaches the window.
1500
+ #
1501
+ # Module-level state mirrors the weekref cost cache exactly: a plain dict of
1502
+ # immutable `BugKSegment` values + a per-cache last-seen dict, mutated only on the
1503
+ # dashboard sync thread (single-writer). Every cached value is immutable and each
1504
+ # rebuild builds a FRESH `WeeklyPeriodRow` from the cached segment, never mutating
1505
+ # it (F7). The `models` payload is a FROZEN TUPLE of (model, cost) in first-seen
1506
+ # order (NOT a live dict — Codex-BK-4), so the row's stable cost-desc sort
1507
+ # tie-order (which depends on first-seen order) can never be mutated after caching.
1508
+ #
1509
+ # Id-stable in-place mutation (Codex-BK-1) — RESOLVED by #270. `sync_cache`'s
1510
+ # `ON CONFLICT(msg_id, req_id) DO UPDATE` can finalize a streaming-intermediate
1511
+ # entry in place (same `id`, changed tokens/cost/timestamp) inside a closed
1512
+ # pre-credit window, advancing NEITHER `max_entry_id` NOR the old
1513
+ # `new_min_timestamp` watermark → a stale segment. This WAS the same exposure the
1514
+ # weekref cost cache and the Group A past-bucket caches carried (spec §6 /
1515
+ # first-review Codex-2). #270 closed it across every #268/#269/#271 cache at once
1516
+ # with the durable `session_entries.mutation_seq` change stamp: it is folded into
1517
+ # `compute_signature` (the `entry_mutation_seq` leg, so an id-stable finalization
1518
+ # leaves the idle path) AND drives `reconcile_bugk_cache`'s seq gate + the
1519
+ # `changed_min_timestamp(mutation_min_ts)` watermark below (so the affected CLOSED
1520
+ # segment — including one the finalization's timestamp MOVED the row across — is
1521
+ # recomputed). Regression: `test_reconcile_bugk_idstable_update_evicts`.
1522
+ #
1523
+ # One accepted trade-off remains, identical to the shipped #269 weekref cost cache
1524
+ # (deliberately NOT closed — closing it would be inconsistent with the already-
1525
+ # shipped caches):
1526
+ # - Pricing-at-cache-time (Codex-BK-3). Caching the folded `pre_cost` means an
1527
+ # in-process embedded-pricing edit is not reflected in the Bug-K pre-credit cost
1528
+ # until the segment invalidates — the SAME dashboard-only trade-off the weekref
1529
+ # cost cache already makes (see its module note above). Acceptable because
1530
+ # embedded-pricing edits require a code change + process restart regardless.
1531
+
1532
+
1533
+ class BugKSegment(NamedTuple):
1534
+ """Immutable folded pre-credit aggregate over a closed `[original_start,
1535
+ effective)` window (spec §18).
1536
+
1537
+ ``models`` is a frozen tuple of ``(display_model, cost)`` in FIRST-SEEN order
1538
+ (Codex-BK-4); the caller re-derives the cost-desc-sorted ``model_breakdowns``
1539
+ fresh each tick from it, and the stable sort preserves that first-seen tie
1540
+ order byte-for-byte.
1541
+ """
1542
+
1543
+ input: int
1544
+ output: int
1545
+ cache_create: int
1546
+ cache_read: int
1547
+ cost: float
1548
+ models: tuple # ((display_model, cost), ...) in first-seen order
1549
+ entry_count: int
1550
+
1551
+
1552
+ _BUGK_SEGMENT_CACHE: dict = {} # {(orig_start_utc_iso, eff_utc_iso): BugKSegment}
1553
+ _BUGK_SEGMENT_LAST_SEEN: dict = {} # {"max_id": int, "reset_sig": tuple}
1554
+
1555
+
1556
+ def _bugk_key(original_start_at, effective_at):
1557
+ """Canonical UTC-ISO key for a pre-credit segment window (spec §18).
1558
+
1559
+ Normalizes both boundaries to UTC before serializing (exactly like
1560
+ ``_weekref_key``), so two spellings of one window can't create duplicate
1561
+ entries. The RAW ISO strings are still used for the row output — this key is
1562
+ cache identity only.
1563
+ """
1564
+ return (
1565
+ original_start_at.astimezone(dt.timezone.utc).isoformat(),
1566
+ effective_at.astimezone(dt.timezone.utc).isoformat(),
1567
+ )
1568
+
1569
+
1570
+ def reset_bugk_segment_state():
1571
+ """Clear the Bug-K segment cache + its watermark (full invalidation).
1572
+
1573
+ Called from the orphan-prune site (a prune deletes ``session_entries``
1574
+ possibly WITHOUT lowering ``MAX(id)``, so the reconcile's max-id-regression
1575
+ check cannot catch it — the explicit clear must) and as a test hook.
1576
+ """
1577
+ _BUGK_SEGMENT_CACHE.clear()
1578
+ _BUGK_SEGMENT_LAST_SEEN.clear()
1579
+
1580
+
1581
+ def cached_bugk_segment(*, key, compute):
1582
+ """Get-or-compute one pre-credit segment aggregate (spec §18).
1583
+
1584
+ The ``[original_start, effective)`` window is ALWAYS a closed past interval
1585
+ (``effective`` is a historical credit moment), so it is always cacheable: a
1586
+ cache hit returns the stored ``BugKSegment``; a miss calls ``compute()`` — the
1587
+ caller's exact from-scratch fetch+fold closure, a ``(timestamp_utc, id)``-
1588
+ ordered fetch so the left-fold ``cost`` / first-seen ``models`` order is
1589
+ bit-identical to today's — and stores it. ``key`` is the canonical
1590
+ ``_bugk_key``.
1591
+ """
1592
+ hit = _BUGK_SEGMENT_CACHE.get(key)
1593
+ if hit is not None:
1594
+ return hit
1595
+ val = compute()
1596
+ _BUGK_SEGMENT_CACHE[key] = val
1597
+ return val
1598
+
1599
+
1600
+ def reconcile_bugk_cache(cache_conn, *, max_entry_id, max_mutation_seq, reset_sig):
1601
+ """Once-per-non-idle-rebuild invalidation for the Bug-K segment cache (§18).
1602
+
1603
+ Driven by ``_tui_build_snapshot`` after the idle-path check, before the
1604
+ builders run, ALONGSIDE ``reconcile_weekref_cache``, using the dispatch-
1605
+ signature legs already computed for the idle decision (``max_entry_id`` +
1606
+ ``max_mutation_seq`` + ``reset_sig`` passed in — no extra query for those):
1607
+
1608
+ - Cold (no last-seen): record last-seen, return — no eviction.
1609
+ - ``reset_sig`` changed (credit events / their ``effective`` moments moved)
1610
+ OR ``max_entry_id < last_seen`` (cache.db rebuilt out-of-process) OR
1611
+ ``max_mutation_seq < last_seen_seq``: full ``clear()``. Credit events are
1612
+ rare, so a conservative full clear is correct and cheap; a max-id / seq
1613
+ regression means the ids no longer map to the same rows.
1614
+ - ``max_mutation_seq > last_seen_seq`` (#270 §7c — the seq gate, so an
1615
+ id-stable in-place finalization with a flat ``max_entry_id`` still evicts):
1616
+ evict segments whose ``effective`` is
1617
+ ``> changed_min_timestamp(cache_conn, last_seen_seq)`` — a genuinely-changed
1618
+ row could fall inside them (F1 late-ingest / #270 in-place). The bound is
1619
+ ``>`` (STRICT), NOT ``>=``, because the segment window is HALF-OPEN
1620
+ ``[original_start, effective)`` (Codex-BK-5): a row EXACTLY at ``effective``
1621
+ is OUTSIDE the segment (never contributes), while a row at
1622
+ ``original_start`` .. just-below ``effective`` evicts. This is the ONE
1623
+ semantic difference from the weekref cache's inclusive ``>=`` (that window
1624
+ is ``[start, end]``). Over-eviction is byte-safe (forces a recompute);
1625
+ normally ``wm`` is recent and nothing drops.
1626
+
1627
+ Idempotent within a tick: after the first call updates last-seen, a later call
1628
+ with the same signature sees no delta and no-ops (never re-running the
1629
+ watermark query). The short-lived ``cache_conn`` is used only for the
1630
+ ``changed_min_timestamp`` query on the ``max_mutation_seq > last_seen_seq``
1631
+ branch (Codex-4 lifecycle from §4).
1632
+ """
1633
+ ls = _BUGK_SEGMENT_LAST_SEEN
1634
+ if not ls: # cold
1635
+ ls["max_id"] = max_entry_id
1636
+ ls["max_seq"] = max_mutation_seq
1637
+ ls["reset_sig"] = reset_sig
1638
+ return
1639
+ if (
1640
+ reset_sig != ls["reset_sig"]
1641
+ or max_entry_id < ls["max_id"]
1642
+ or max_mutation_seq < ls["max_seq"]
1643
+ ):
1644
+ _BUGK_SEGMENT_CACHE.clear() # a credit event moved, or cache.db rebuilt
1645
+ ls["max_id"] = max_entry_id
1646
+ ls["max_seq"] = max_mutation_seq
1647
+ ls["reset_sig"] = reset_sig
1648
+ return
1649
+ if max_mutation_seq > ls["max_seq"]:
1650
+ wm = changed_min_timestamp(cache_conn, ls["max_seq"])
1651
+ if wm is not None:
1652
+ for key in list(_BUGK_SEGMENT_CACHE):
1653
+ # key = (orig_start_iso, eff_iso); half-open [start, eff) window,
1654
+ # so evict when eff > the earliest changed event time (a row AT
1655
+ # eff is outside the segment; a row < eff could be inside it).
1656
+ if dt.datetime.fromisoformat(key[1]) > wm:
1657
+ del _BUGK_SEGMENT_CACHE[key]
1658
+ ls["max_id"] = max_entry_id
1659
+ ls["max_seq"] = max_mutation_seq
1660
+ ls["reset_sig"] = reset_sig
1661
+
1662
+
1663
+ # === #272 — cache-report per-day cache =====================================
1664
+ #
1665
+ # A per-day cache in front of the dashboard's `build_cache_report_snapshot`
1666
+ # warm-tick leg. Each closed day's raw aggregate + per-project partials is
1667
+ # an immutable `CachedCacheReportDay` (bin/_lib_cache_report.py §5); the
1668
+ # reconcile below invalidates it on the #270/#271 `mutation_seq` / `max_id`
1669
+ # / `reset_sig` / `sf_sig` / display-tz signals, mirroring
1670
+ # `reconcile_bugk_cache` + the projects-env `session_files_sig` leg.
1671
+
1672
+ _CACHE_REPORT_DAY_CACHE: dict = {} # {date_key: CachedCacheReportDay}
1673
+ _CACHE_REPORT_LAST_SEEN: dict = {} # {max_id, max_seq, reset_sig, sf_sig, tz_key}
1674
+
1675
+
1676
+ def reset_cache_report_state():
1677
+ """Clear the cache-report per-day cache + its watermark (full invalidation).
1678
+
1679
+ Called from the orphan-prune site (a prune can delete ``session_entries``
1680
+ WITHOUT lowering ``MAX(id)`` / advancing ``mutation_seq``, which the
1681
+ reconcile's regression check cannot catch — the explicit clear must) and
1682
+ as a test hook. Mirrors ``reset_bugk_segment_state`` /
1683
+ ``reset_projects_env_state``.
1684
+ """
1685
+ _CACHE_REPORT_DAY_CACHE.clear()
1686
+ _CACHE_REPORT_LAST_SEEN.clear()
1687
+
1688
+
1689
+ def cache_report_day_get(date_key):
1690
+ """Return the cached ``CachedCacheReportDay`` for ``date_key``, or ``None``."""
1691
+ return _CACHE_REPORT_DAY_CACHE.get(date_key)
1692
+
1693
+
1694
+ def cache_report_day_store(date_key, value) -> None:
1695
+ """Store one CLOSED day's immutable ``CachedCacheReportDay`` (never mutated)."""
1696
+ _CACHE_REPORT_DAY_CACHE[date_key] = value
1697
+
1698
+
1699
+ def cache_report_day_evict_before(oldest_key) -> None:
1700
+ """Drop cached days strictly older than ``oldest_key`` — the window-rolloff
1701
+ tail (#275).
1702
+
1703
+ ``reconcile_cache_report_cache`` only evicts CLOSED days that CHANGED (``>=``
1704
+ the seq-gated change watermark); a day that simply rolls off the trailing edge
1705
+ of the ``[since, now]`` window is never touched by it and would linger in the
1706
+ module dict until a reset / tz-change / ``sf_sig`` / regression full-clear or
1707
+ ``reset_cache_report_state()``. On a long-uptime dashboard that accretes ~1
1708
+ frozen ``CachedCacheReportDay`` per day. ``build_cache_report_snapshot`` calls
1709
+ this on its rare cold/rollover store tick with the window's oldest still-needed
1710
+ closed date, keeping the dict bounded to the live window. Over-eviction is
1711
+ byte-safe (a re-needed day just re-populates on the next cold tick), so the
1712
+ predicate is a plain lexical ``<`` on the ``YYYY-MM-DD`` keys.
1713
+ """
1714
+ for date_key in [d for d in _CACHE_REPORT_DAY_CACHE if d < oldest_key]:
1715
+ del _CACHE_REPORT_DAY_CACHE[date_key]
1716
+
1717
+
1718
+ def reconcile_cache_report_cache(
1719
+ cache_conn, *, max_entry_id, max_mutation_seq, reset_sig, sf_sig, bucket_tz,
1720
+ tz_key,
1721
+ ):
1722
+ """Once-per-non-idle-rebuild invalidation for the cache-report cache (#272 §5).
1723
+
1724
+ The canonical four-step shape (mirrors ``reconcile_bugk_cache`` +
1725
+ ``reconcile_weekref_cache``, plus the projects-env cache's
1726
+ ``session_files_sig`` leg and a display-tz leg):
1727
+
1728
+ 1. **Cold** (empty last-seen) → record
1729
+ ``{max_id, max_seq, reset_sig, sf_sig, tz_key}``, return, no eviction.
1730
+ 2. **Full-clear** on any of: ``reset_sig`` changed, ``sf_sig`` changed
1731
+ (Codex-1 — lazy ``session_files.project_path`` backfill can re-attribute
1732
+ a CLOSED day's by_project rows with no ``session_entries`` / seq change).
1733
+ INHERITED LIMITATION (accepted; matches the projects-env cache, same
1734
+ ``sf_sig`` leg): ``sf_sig`` is ``(COUNT(*), MAX(rowid))``, so an in-place
1735
+ ``ON CONFLICT(path) DO UPDATE SET project_path = COALESCE(...)`` backfill
1736
+ (rowid + count both stable — the actual lazy-backfill shape) does NOT
1737
+ move it. Such an UPDATE is only watermark-covered via the co-ingested new
1738
+ ``session_entries`` rows, which carry the NEW day — so the seq-gated
1739
+ eviction reaches only ``>= that day`` and a re-attributed CLOSED day can
1740
+ stale-serve ``(unknown)`` until it rolls out of the window (or a reset
1741
+ full-clears). A framework-wide fix belongs with the shared ``sf_sig``
1742
+ leg, not #272.
1743
+ ``tz_key`` changed (a display-tz change shifts every calendar-day
1744
+ boundary → all cached day keys invalid), ``max_entry_id`` regressed, or
1745
+ ``max_mutation_seq`` regressed (cache.db rebuilt out-of-process) →
1746
+ ``clear()``, update last-seen, return.
1747
+ 3. **Seq-gated eviction** (``max_mutation_seq > last_seen_seq`` — the #270
1748
+ §7c seq gate, so an id-stable in-place finalization with a flat
1749
+ ``max_entry_id`` still evicts): ``wm = changed_min_timestamp(cache_conn,
1750
+ last_seen_seq)`` (the one query on this branch; #270
1751
+ ``min(old_ts, new_ts)`` change-time). Evict every cached day whose
1752
+ ``date_key >= wm.astimezone(bucket_tz).strftime("%Y-%m-%d")``. The bound
1753
+ is ``>=`` (INCLUSIVE), NOT ``>`` — a day bucket is the CLOSED interval
1754
+ ``[start, end]``, so a changed row landing anywhere on ``wm_day`` (or a
1755
+ later day) is inside that day and must evict it; and a cross-day
1756
+ finalization pulls ``mutation_min_ts`` back to ``min(old, new)``, i.e.
1757
+ the OLD day, so the OLD day (exactly at the watermark) evicts too. This
1758
+ is the ONE semantic difference from ``reconcile_bugk_cache``'s strict
1759
+ ``>`` (its segment window is HALF-OPEN ``[start, effective)``).
1760
+ Over-eviction is byte-safe. Update last-seen.
1761
+ 4. **Idempotent within a tick**: after the first call updates last-seen, a
1762
+ same-signature second call sees ``max_mutation_seq == last_seen_seq`` and
1763
+ no sig delta → no watermark re-query, no eviction.
1764
+ """
1765
+ ls = _CACHE_REPORT_LAST_SEEN
1766
+ if not ls: # cold
1767
+ ls.update(
1768
+ max_id=max_entry_id, max_seq=max_mutation_seq,
1769
+ reset_sig=reset_sig, sf_sig=sf_sig, tz_key=tz_key,
1770
+ )
1771
+ return
1772
+ if (
1773
+ reset_sig != ls["reset_sig"]
1774
+ or sf_sig != ls["sf_sig"]
1775
+ or tz_key != ls["tz_key"]
1776
+ or max_entry_id < ls["max_id"]
1777
+ or max_mutation_seq < ls["max_seq"]
1778
+ ):
1779
+ _CACHE_REPORT_DAY_CACHE.clear()
1780
+ ls.update(
1781
+ max_id=max_entry_id, max_seq=max_mutation_seq,
1782
+ reset_sig=reset_sig, sf_sig=sf_sig, tz_key=tz_key,
1783
+ )
1784
+ return
1785
+ if max_mutation_seq > ls["max_seq"]:
1786
+ wm = changed_min_timestamp(cache_conn, ls["max_seq"])
1787
+ if wm is not None:
1788
+ wm_day = wm.astimezone(bucket_tz).strftime("%Y-%m-%d")
1789
+ for date_key in list(_CACHE_REPORT_DAY_CACHE):
1790
+ # Inclusive [start, end] day bucket: a changed row on wm_day (or
1791
+ # any later day) falls inside that day, so evict date_key >= wm_day.
1792
+ if date_key >= wm_day:
1793
+ del _CACHE_REPORT_DAY_CACHE[date_key]
1794
+ ls.update(
1795
+ max_id=max_entry_id, max_seq=max_mutation_seq,
1796
+ reset_sig=reset_sig, sf_sig=sf_sig, tz_key=tz_key,
1797
+ )