cctally 1.79.1 → 1.80.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2148 @@
1
+ """Journal I/O glue — append surface (spec §4.3).
2
+
3
+ The pure line codec / identity / segment rules live in ``_lib_journal``; this
4
+ module owns the *durable* side: appending a fully-encoded line to the current
5
+ month's segment under the leaf flock, with torn-tail repair and fsync
6
+ discipline, plus the high-water snapshot and segment listing the single-flight
7
+ ingester (Task 4+) consumes.
8
+
9
+ Lock discipline (spec §4.3 / §5.2 lock-order law):
10
+
11
+ - ``journal.lock`` is a **leaf** blocking exclusive flock, held for
12
+ microseconds. No other lock, flock, or SQLite transaction is ever acquired
13
+ while it is held — it may therefore be taken from inside any context
14
+ (including under a provider flock) without ordering hazards.
15
+ - The appender never reads, seeks, or rewrites the segment beyond the
16
+ bounded torn-tail repair: read the final byte; if it is not ``\\n`` the
17
+ previous appender crashed mid-write, so scan back within a 64 KiB window to
18
+ the last complete line and ``ftruncate`` to it before appending. A crash can
19
+ therefore only ever leave a torn *final* line, which the next append heals.
20
+
21
+ Path constants (``JOURNAL_DIR``, ``JOURNAL_LOCK_PATH``) are read from
22
+ ``_cctally_core`` at call time so dev/data-dir redirection and test isolation
23
+ apply. Permissions match the hardened DB sidecars: ``0o700`` dir, ``0o600``
24
+ segment files.
25
+ """
26
+ from __future__ import annotations
27
+
28
+ import datetime as dt
29
+ import fcntl
30
+ import os
31
+ import pathlib
32
+ import sqlite3
33
+ import sys
34
+ import time
35
+ from dataclasses import dataclass, field
36
+
37
+ import _cctally_core
38
+ import _lib_journal
39
+ import _lib_record
40
+
41
+
42
+ # Torn-tail scan window (spec §4.3). A single line must fit inside it — a
43
+ # longer line is a hard error by construction, which is what keeps torn-tail
44
+ # repair bounded (the previous newline is guaranteed to fall inside the window).
45
+ _TAIL_WINDOW = 64 * 1024
46
+ _MAX_LINE_BYTES = _TAIL_WINDOW
47
+
48
+
49
+ class JournalError(Exception):
50
+ """A structural journal-append failure (line too long, unrepairable tail)."""
51
+
52
+
53
+ # --------------------------------------------------------------------------
54
+ # leaf lock
55
+ # --------------------------------------------------------------------------
56
+
57
+ def _acquire_leaf_lock() -> int:
58
+ """Open + blocking-EX-flock ``journal.lock``; return the held fd.
59
+
60
+ LEAF: the caller must acquire no other lock while this is held."""
61
+ _cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
62
+ fd = os.open(str(_cctally_core.JOURNAL_LOCK_PATH), os.O_RDWR | os.O_CREAT, 0o600)
63
+ try:
64
+ fcntl.flock(fd, fcntl.LOCK_EX)
65
+ except BaseException:
66
+ os.close(fd)
67
+ raise
68
+ return fd
69
+
70
+
71
+ def _release_leaf_lock(fd: int) -> None:
72
+ try:
73
+ fcntl.flock(fd, fcntl.LOCK_UN)
74
+ finally:
75
+ os.close(fd)
76
+
77
+
78
+ # --------------------------------------------------------------------------
79
+ # low-level helpers
80
+ # --------------------------------------------------------------------------
81
+
82
+ def _write_all(fd: int, data: bytes) -> None:
83
+ """Write ``data`` fully, looping on partial writes and EINTR."""
84
+ view = memoryview(data)
85
+ total = 0
86
+ n = len(data)
87
+ while total < n:
88
+ try:
89
+ written = os.write(fd, view[total:])
90
+ except InterruptedError: # pragma: no cover — PEP 475 retries most EINTR
91
+ continue
92
+ total += written
93
+
94
+
95
+ def _fsync_dir(path) -> None:
96
+ """fsync a directory so a newly-created child entry is durable."""
97
+ dfd = os.open(str(path), os.O_RDONLY)
98
+ try:
99
+ os.fsync(dfd)
100
+ finally:
101
+ os.close(dfd)
102
+
103
+
104
+ def _repair_torn_tail(fd: int) -> None:
105
+ """Heal a torn final line under the lock (spec §4.3).
106
+
107
+ Fast path (spec §4.3): ``pread`` just the file's FINAL byte first. On the
108
+ hot append path the segment is already newline-terminated, so this avoids
109
+ the 64 KiB window read on every single append — the common case is one
110
+ 1-byte read, not a 64 KiB read.
111
+
112
+ Only when that final byte is not ``\\n`` did the previous appender crash
113
+ mid-write: read the final ≤64 KiB window and ``ftruncate`` back to the last
114
+ complete line. A window with no newline at all means either an empty file
115
+ with a single incomplete first line (truncate to 0) or — if the window is a
116
+ full 64 KiB — a line longer than the scan window, which is a hard error."""
117
+ size = os.fstat(fd).st_size
118
+ if size == 0:
119
+ return
120
+ if os.pread(fd, 1, size - 1) == b"\n":
121
+ return
122
+ window = min(size, _TAIL_WINDOW)
123
+ chunk_start = size - window
124
+ chunk = os.pread(fd, window, chunk_start)
125
+ valid = _lib_journal.valid_tail_offset(chunk, chunk_start)
126
+ if valid <= chunk_start and chunk_start > 0:
127
+ raise JournalError(
128
+ "torn journal tail exceeds the 64 KiB scan window "
129
+ "(a single line must fit the window, spec §4.3)")
130
+ os.ftruncate(fd, valid)
131
+
132
+
133
+ # --------------------------------------------------------------------------
134
+ # public append surface
135
+ # --------------------------------------------------------------------------
136
+
137
+ def append_record(record: dict, *, now_utc: dt.datetime | None = None) -> tuple[str, int]:
138
+ """Append one encoded line to the current UTC month's segment.
139
+
140
+ Implements spec §4.3 exactly: ``O_RDWR|O_APPEND|O_CREAT`` (0o600) → blocking
141
+ leaf flock → torn-tail repair → loop-write the full line → ``fsync(fd)`` →
142
+ parent-dir fsync on first segment/dir creation → unlock.
143
+
144
+ ``now_utc`` selects the segment (defaults to the current UTC time). Returns
145
+ ``(segment_basename, end_offset)`` where ``end_offset`` is the file size
146
+ just past the appended line — the byte position the ingest cursor advances
147
+ to when it consumes this line."""
148
+ if now_utc is None:
149
+ now_utc = dt.datetime.now(dt.timezone.utc)
150
+ data = _lib_journal.encode_line(record)
151
+ if len(data) > _MAX_LINE_BYTES:
152
+ raise JournalError(
153
+ f"journal line is {len(data)} bytes, exceeds the "
154
+ f"{_MAX_LINE_BYTES}-byte limit (spec §4.3)")
155
+
156
+ journal_dir = _cctally_core.JOURNAL_DIR
157
+ seg_name = _lib_journal.segment_name(now_utc)
158
+ seg_path = journal_dir / seg_name
159
+
160
+ dir_created = not journal_dir.exists()
161
+ journal_dir.mkdir(parents=True, exist_ok=True)
162
+ if dir_created:
163
+ try:
164
+ os.chmod(journal_dir, 0o700)
165
+ except OSError:
166
+ pass
167
+
168
+ lock_fd = _acquire_leaf_lock()
169
+ try:
170
+ seg_created = not seg_path.exists()
171
+ fd = os.open(str(seg_path), os.O_RDWR | os.O_APPEND | os.O_CREAT, 0o600)
172
+ try:
173
+ if seg_created:
174
+ try:
175
+ os.fchmod(fd, 0o600)
176
+ except OSError:
177
+ pass
178
+ _repair_torn_tail(fd)
179
+ _write_all(fd, data)
180
+ os.fsync(fd)
181
+ end_offset = os.fstat(fd).st_size
182
+ finally:
183
+ os.close(fd)
184
+ # Durably record new directory entries (spec §4.3: fsync the parent
185
+ # directory on first creation of a segment or the journal dir).
186
+ if seg_created:
187
+ _fsync_dir(journal_dir)
188
+ if dir_created:
189
+ _fsync_dir(journal_dir.parent)
190
+ return (seg_name, end_offset)
191
+ finally:
192
+ _release_leaf_lock(lock_fd)
193
+
194
+
195
+ def list_segments() -> list[str]:
196
+ """Journal segment basenames in canonical order (spec §4.1): bootstrap
197
+ segments first, then observation segments, each class lexicographic.
198
+ Excludes ``.partial`` cutover files and any non-segment entries."""
199
+ journal_dir = _cctally_core.JOURNAL_DIR
200
+ if not journal_dir.exists():
201
+ return []
202
+ names = []
203
+ for entry in journal_dir.iterdir():
204
+ name = entry.name
205
+ if not name.endswith(".jsonl"):
206
+ continue
207
+ if not (name.startswith(_lib_journal.BOOTSTRAP_PREFIX)
208
+ or name.startswith(_lib_journal.SEGMENT_PREFIX)):
209
+ continue
210
+ if entry.is_file():
211
+ names.append(name)
212
+ return sorted(names, key=_lib_journal.segment_sort_key)
213
+
214
+
215
+ def journal_high_water() -> tuple[str, int] | None:
216
+ """Snapshot ``(latest segment basename, size)`` under a µs leaf-lock hold.
217
+
218
+ "Latest" is the canonically-last segment (spec §4.1 order). The ingest
219
+ cycle takes this snapshot and consumes only ``cursor → HW`` so a line
220
+ appended after the snapshot belongs to the next cycle (spec §5.2.1).
221
+ Returns ``None`` when no segment exists yet."""
222
+ lock_fd = _acquire_leaf_lock()
223
+ try:
224
+ segments = list_segments()
225
+ if not segments:
226
+ return None
227
+ latest = segments[-1]
228
+ size = os.stat(_cctally_core.JOURNAL_DIR / latest).st_size
229
+ return (latest, size)
230
+ finally:
231
+ _release_leaf_lock(lock_fd)
232
+
233
+
234
+ # ==========================================================================
235
+ # Single-flight ingest cycle (spec §5.1 / §5.2, revision 3)
236
+ # ==========================================================================
237
+ #
238
+ # `run_stats_ingest` is the sole stats.db writer (spec §5.1). One cycle
239
+ # consumes `cursor -> HW` in canonical `(segment, offset)` order. The rev-3
240
+ # structure runs derivation INSIDE the index transaction (the reused Task-5
241
+ # chokepoints write rows directly on the connection), then journals the derived
242
+ # facts inside the same transaction (`journal.lock` is a leaf, legal here —
243
+ # every evt append is fsync'd BEFORE the commit that indexes it). Per cycle,
244
+ # under the ingest lock (spec §5.2):
245
+ #
246
+ # 1. HW snapshot (leaf lock, µs). journal_high_water()
247
+ # 2. read+decode cursor -> HW, counting malformed. _read_range()
248
+ # 3. cache leg (Codex quota) BEFORE the stats txn. QUOTA_APPLIER seam
249
+ # 4. ONE `BEGIN IMMEDIATE`:
250
+ # a. replay journal evt lines (apply-only, NO alerts). _apply_evt()
251
+ # b. per-record sequential PIPELINE over obs/op. PIPELINE hooks
252
+ # c. journal derived facts: Model-A emission + harvest. emit_model_a() /
253
+ # _harvest()
254
+ # d. advance the cursor. _write_cursor()
255
+ # 5. COMMIT (journal-first: every evt fsync'd before this).
256
+ # 6. post-commit alert dispatch from the step-4b sink. ALERT_DISPATCHER
257
+ #
258
+ # Seams later implementors (6b / Task 7) wire on top of this machinery:
259
+ # PIPELINE list of (ctx, record) -> None hooks; sequential, in-txn.
260
+ # 6b appends the obs-derivation hooks (snapshot_accept,
261
+ # milestones, resets/credits, cost snapshots, budgets); the
262
+ # built-in `_pipeline_op_weekly_credit_floor` op fold ships
263
+ # here (spec §5.3 "fold op").
264
+ # QUOTA_APPLIER the Codex quota cache leg (Task 7; wired to `_quota_applier`
265
+ # below). Contract: (decoded) -> stop_index | None. `decoded`
266
+ # is the ordered list of (record, segment, offset); a non-None
267
+ # int is a prefix-stop boundary (busy `cache.db.codex.lock` at
268
+ # a quota line): the cycle processes decoded[:stop] and
269
+ # advances the cursor to decoded[stop]'s offset (spec §5.2
270
+ # step 3). Always-on: a Claude-only batch scans + returns None.
271
+ # codex_apply per-cycle `(ctx) -> None` closure (Task 7, a `run_stats_
272
+ # ingest` arg, not a module global) run in step 4b'' on
273
+ # ctx.conn — the seam every Codex on-demand stats.db writer
274
+ # routes through: the quota projection re-materializer
275
+ # (`reconcile_codex_quota_projection`) and the on-demand codex
276
+ # budget/projected firings. Its harvest-family crossings are
277
+ # journaled by step 4c; its alerts ride ctx.pending_alerts.
278
+ # ALERT_DISPATCHER post-commit dispatch override (None -> _dispatch_pending
279
+ # _alerts). Consumes ctx.pending_alerts, populated by step-4b
280
+ # pipeline hooks AND the step-4b'' codex_apply leg — step-4a
281
+ # replay has no sink access.
282
+ #
283
+ # Lock-order law (spec §5.2): the ingest lock is acquired BEFORE any SQLite
284
+ # transaction and BEFORE the leaf `journal.lock`; it is never taken while
285
+ # holding a provider flock. The quota cache leg + its provider flock run BEFORE
286
+ # the stats `BEGIN IMMEDIATE`. Inside the txn the only leaf-lock holds are the
287
+ # discrete evt appends (emit_model_a / _harvest -> append_record); `journal.lock`
288
+ # is a leaf and may be taken inside a transaction — it never spans the commit.
289
+
290
+ PIPELINE: list = []
291
+ QUOTA_APPLIER = None
292
+ ALERT_DISPATCHER = None
293
+ FOLD_APPLIERS: dict = {}
294
+
295
+
296
+ @dataclass
297
+ class IngestResult:
298
+ """Outcome of one `run_stats_ingest` call."""
299
+
300
+ ran: bool # False when the lock was busy (opportunistic)
301
+ consumed: int # decoded records processed this cycle (obs/op/evt)
302
+ malformed: int # lines in range that failed to decode (spec §4.4)
303
+ events_emitted: int # evt lines emitted this cycle (Model-A + harvest)
304
+ alerts: list # alert payloads dispatched post-commit (step 6)
305
+ # Exception discipline (6b-gate P2): the exception that aborted the cycle on
306
+ # an OPPORTUNISTIC ingest — the txn rolled back, the cursor did NOT advance
307
+ # (invariant ii), and `run_stats_ingest` logged it loudly and returned
308
+ # `ran=True, error=<exc>` rather than break a statusline/hook tick. `None`
309
+ # on a clean cycle. Authoritative callers never see this — their cycle
310
+ # exception propagates.
311
+ error: object = None
312
+
313
+
314
+ @dataclass
315
+ class IngestContext:
316
+ """Per-cycle context handed to every PIPELINE hook (spec §5.2 step 4b).
317
+
318
+ `as_of_for(record)` is the capture-time-pure clock: a hook injects the
319
+ record's own `at` wherever the live code would consult wall time, so replay
320
+ is deterministic. `config` is read ONCE per cycle (only when the batch is
321
+ non-empty). `pending_alerts` is the post-commit dispatch SINK: a hook that
322
+ fires an alert appends its payload here, and step 6 dispatches them after the
323
+ commit. Replay (step 4a) folds evt lines with NO ctx, so it is structurally
324
+ unable to add to the sink. `events_emitted` counts the evt lines this cycle
325
+ journaled (Model-A `emit_model_a` + harvest).
326
+ """
327
+
328
+ conn: sqlite3.Connection
329
+ batch: list # decoded obs/op records this cycle
330
+ config: object = None
331
+ pending_alerts: list = field(default_factory=list)
332
+ events_emitted: int = 0
333
+ # Design B (DB journal redesign §5.3 event+effects): the per-cycle
334
+ # suppression map a reset/credit pipeline hook populates BEFORE it runs its
335
+ # stale-replica DELETE, keyed on the harvest natural-key parts of the reset
336
+ # it just inserted — `(old_week_end_at, new_week_end_at)` for a
337
+ # `week_reset_events` row, `(five_hour_window_key, effective_reset_at_utc)`
338
+ # for a `five_hour_reset_events` row. The value is the list of logical
339
+ # `journal_id`s the DELETE will hit. `_build_harvest_evt` reads it back and
340
+ # attaches the list to the reset's harvest evt payload, so the destructive
341
+ # effect replays deterministically (idempotent against absent ids). The
342
+ # hook captures BEFORE deleting and ONLY on the genuine-new-reset winner
343
+ # (reset INSERT OR IGNORE rowcount == 1), so a crash-replayed reset never
344
+ # re-suppresses with a divergent list.
345
+ suppression_map: dict = field(default_factory=dict)
346
+
347
+ def as_of_for(self, record: dict) -> str:
348
+ return record["at"]
349
+
350
+
351
+ @dataclass(frozen=True)
352
+ class _EvtSpec:
353
+ """How to fold one evt `kind` into its target table (step 4a replay + the
354
+ Model-A `emit_model_a` apply path — one applier, two callers, so live-emit
355
+ and crash-replay converge by construction).
356
+
357
+ `fk_refs` maps a payload key carrying a *logical* id to `(column,
358
+ ref_table)`: the fold resolves the logical id to the rebuilt DB's actual
359
+ rowid via `_resolve_ref` (spec §4.2 FK rule). Everything else in the payload
360
+ maps mechanically to a same-named column. `order` sequences folds so a
361
+ referenced family (snapshots, resets, blocks) folds before a referencing one
362
+ (milestones) — the FK-resolution dependency order (spec Appendix B I4 P2-8).
363
+ `applier`, when set, overrides the generic column-map fold (weekly credit
364
+ effects + block-close children need bespoke logic).
365
+ """
366
+
367
+ table: "str | None"
368
+ fk_refs: dict = field(default_factory=dict)
369
+ order: int = 60
370
+ applier: object = None
371
+ # A FK column that is NOT journaled as a logical id but RE-DERIVED at fold
372
+ # time from another (journaled) natural-key column, keyed `column ->
373
+ # (ref_table, lookup_column)`: `column = SELECT id FROM ref_table WHERE
374
+ # lookup_column = payload[lookup_column]`. Used for a FK into a MUTABLE
375
+ # PROJECTION that may have no `journal_id` (spec §5.3): `five_hour_milestones.
376
+ # block_id` points at the OPEN five_hour_blocks row (a projection,
377
+ # re-materialized on rebuild, never journaled), so it cannot carry a stable
378
+ # logical id — but it is recoverable from the milestone's own
379
+ # `five_hour_window_key`, which IS journaled.
380
+ derived_fk: dict = field(default_factory=dict)
381
+
382
+
383
+ @dataclass(frozen=True)
384
+ class _HarvestSpec:
385
+ """How to harvest one natural-keyed family (spec §5.3): after the pipeline,
386
+ every row `WHERE journal_id IS NULL` is a this-cycle insert — build its evt
387
+ with logical-FK refs (reverse lookup rowid -> journal_id), append+fsync, and
388
+ stamp `journal_id`.
389
+
390
+ `id_prefix` is the opaque evt-id prefix (`pm`, `wr`, …) — deliberately NOT
391
+ the fold `kind` (`percent_milestone`, `week_reset`, …); the id is an opaque
392
+ token, never parsed (spec §5.3 / Appendix B I4 P3-11). `id_parts` is the
393
+ ordered list of columns whose values follow the prefix; a column that is
394
+ also an FK ref contributes its *logical* id. `fk_refs` maps an FK column to
395
+ `(ref_table, payload_ref_key)` — the reverse of `_EvtSpec.fk_refs`.
396
+ `at_column` supplies the evt `at`. `order` harvests referenced families
397
+ before referencing ones. `closed_only` scopes the scan to `is_closed = 1`
398
+ (five_hour_blocks). `children` embeds rollup children into the payload
399
+ (five_hour_blocks' `_models`/`_projects`).
400
+ """
401
+
402
+ table: str
403
+ kind: str
404
+ id_prefix: str
405
+ id_parts: tuple
406
+ fk_refs: dict = field(default_factory=dict)
407
+ at_column: "str | None" = None
408
+ order: int = 60
409
+ closed_only: bool = False
410
+ children: tuple = ()
411
+ # A FK column re-derived at fold from a journaled natural-key column instead
412
+ # of a logical id, keyed `column -> (ref_table, lookup_column)` (see
413
+ # `_EvtSpec.derived_fk`). The harvest EXCLUDES these columns from the evt
414
+ # payload (the raw rowid is not stable) — the lookup column carries the info.
415
+ derived_fk: dict = field(default_factory=dict)
416
+ # Design B (event+effects): when True this family's harvest evt carries a
417
+ # `suppression` list (logical ids of `weekly_usage_snapshots` rows the reset
418
+ # deleted) that the fold applier replays. `_build_harvest_evt` sources the
419
+ # list from `ctx.suppression_map` keyed on this spec's `id_parts` values.
420
+ suppression: bool = False
421
+
422
+
423
+ # --------------------------------------------------------------------------
424
+ # ingest lock (spec §5.1: opportunistic NB / authoritative bounded-blocking)
425
+ # --------------------------------------------------------------------------
426
+
427
+ def _acquire_ingest_lock(mode: str, timeout_s: float) -> int | None:
428
+ """Acquire `journal.ingest.lock`; return the held fd or None (busy).
429
+
430
+ Opportunistic → single non-blocking attempt (busy = None). Authoritative →
431
+ poll LOCK_NB up to `timeout_s` (a bounded blocking wait; None on timeout).
432
+ """
433
+ _cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
434
+ fd = os.open(str(_cctally_core.JOURNAL_INGEST_LOCK_PATH),
435
+ os.O_RDWR | os.O_CREAT, 0o600)
436
+ if mode == "opportunistic":
437
+ try:
438
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
439
+ return fd
440
+ except (BlockingIOError, OSError):
441
+ os.close(fd)
442
+ return None
443
+ deadline = time.monotonic() + timeout_s
444
+ # Wrap the whole poll body so the lock fd cannot leak if the wait is
445
+ # interrupted mid-sleep (KeyboardInterrupt / any non-BlockingIOError raised
446
+ # by flock or time.sleep): close the fd on any escaping exception path. The
447
+ # busy-timeout branch closes + returns before this handler can fire, and a
448
+ # successful acquire returns fd, so the fd is closed exactly once.
449
+ try:
450
+ while True:
451
+ try:
452
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
453
+ return fd
454
+ except (BlockingIOError, OSError):
455
+ if time.monotonic() >= deadline:
456
+ os.close(fd)
457
+ return None
458
+ time.sleep(0.02)
459
+ except BaseException:
460
+ os.close(fd)
461
+ raise
462
+
463
+
464
+ def _release_ingest_lock(fd: int) -> None:
465
+ try:
466
+ fcntl.flock(fd, fcntl.LOCK_UN)
467
+ finally:
468
+ os.close(fd)
469
+
470
+
471
+ # --------------------------------------------------------------------------
472
+ # cursor (spec §5.2.2: segment-aware, prior-month tails covered)
473
+ # --------------------------------------------------------------------------
474
+
475
+ def _read_cursor(conn: sqlite3.Connection) -> tuple[str, int] | None:
476
+ """Return `(segment_basename, offset)` from `journal_cursor`, or None when
477
+ nothing has been consumed yet (start of the first segment)."""
478
+ row = conn.execute(
479
+ "SELECT segment, offset FROM journal_cursor WHERE id = 1"
480
+ ).fetchone()
481
+ if row is None:
482
+ return None
483
+ return (row[0], int(row[1]))
484
+
485
+
486
+ def _write_cursor(conn: sqlite3.Connection, segment: str, offset: int) -> None:
487
+ conn.execute(
488
+ "INSERT INTO journal_cursor (id, segment, offset) VALUES (1, ?, ?) "
489
+ "ON CONFLICT(id) DO UPDATE SET segment = excluded.segment, "
490
+ "offset = excluded.offset",
491
+ (segment, offset),
492
+ )
493
+
494
+
495
+ def _read_segment_lines(seg_path, lo: int, hi: int) -> list[tuple[str, int, bytes]]:
496
+ """`(basename, absolute-offset, raw-line-without-newline)` for every
497
+ complete line in `[lo, hi)`. `hi` is a line boundary (a HW snapshot size or
498
+ an immutable prior segment's full size), so no partial trailing line
499
+ appears."""
500
+ with open(seg_path, "rb") as fh:
501
+ fh.seek(lo)
502
+ data = fh.read(hi - lo)
503
+ out = []
504
+ start = 0
505
+ while True:
506
+ nl = data.find(b"\n", start)
507
+ if nl == -1:
508
+ break
509
+ out.append((seg_path.name, lo + start, data[start:nl]))
510
+ start = nl + 1
511
+ return out
512
+
513
+
514
+ def _read_range(cursor, hw) -> list[tuple[str, int, bytes]]:
515
+ """Read `cursor -> HW` across segments in canonical order (spec §5.2.2).
516
+
517
+ Prior segments (before HW's) are immutable and read to their full size;
518
+ HW's segment is read only up to the snapshot size, so appends past HW
519
+ belong to the next cycle.
520
+ """
521
+ hw_seg, hw_size = hw
522
+ segments = list_segments()
523
+ if hw_seg not in segments:
524
+ return []
525
+ hw_idx = segments.index(hw_seg)
526
+ if cursor is None:
527
+ start_idx, start_off = 0, 0
528
+ else:
529
+ cur_seg, cur_off = cursor
530
+ if cur_seg in segments:
531
+ start_idx, start_off = segments.index(cur_seg), cur_off
532
+ else:
533
+ start_idx, start_off = 0, 0
534
+ lines: list[tuple[str, int, bytes]] = []
535
+ for idx in range(start_idx, hw_idx + 1):
536
+ seg = segments[idx]
537
+ seg_path = _cctally_core.JOURNAL_DIR / seg
538
+ lo = start_off if idx == start_idx else 0
539
+ hi = hw_size if idx == hw_idx else os.path.getsize(seg_path)
540
+ if lo >= hi:
541
+ continue
542
+ lines.extend(_read_segment_lines(seg_path, lo, hi))
543
+ return lines
544
+
545
+
546
+ # --------------------------------------------------------------------------
547
+ # cache leg — Codex quota obs -> cache.db quota_window_snapshots (spec §5.2
548
+ # step 3, Task 7 Item 2). Runs BEFORE the stats BEGIN IMMEDIATE, under the
549
+ # `cache.db.codex.lock` provider flock (lock-order law: provider flocks precede
550
+ # SQLite write transactions). The journal Codex quota obs are the DURABLE truth
551
+ # (§1 latent data-loss hole — the source rollout JSONL evaporates); this leg
552
+ # re-materializes the disposable cache.db index from them, idempotently
553
+ # (INSERT OR IGNORE on the natural key). Distinct from the direct cache write in
554
+ # `sync_codex_cache._write_codex_file_batch` (kept byte-identical, Item 1) — the
555
+ # two converge on the same rows.
556
+ # --------------------------------------------------------------------------
557
+
558
+ _QUOTA_OBS_KIND = "quota_window_snapshot"
559
+
560
+ _QUOTA_SNAPSHOT_INSERT = (
561
+ "INSERT OR IGNORE INTO quota_window_snapshots "
562
+ "(source, source_root_key, source_path, line_offset, captured_at_utc, "
563
+ " observed_slot, logical_limit_key, limit_id, limit_name, window_minutes, "
564
+ " used_percent, resets_at_utc, plan_type, individual_limit_json, "
565
+ " reached_type, observed_model) "
566
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
567
+ )
568
+
569
+ _QUOTA_SNAPSHOT_COLS = (
570
+ "source", "source_root_key", "source_path", "line_offset", "captured_at_utc",
571
+ "observed_slot", "logical_limit_key", "limit_id", "limit_name",
572
+ "window_minutes", "used_percent", "resets_at_utc", "plan_type",
573
+ "individual_limit_json", "reached_type", "observed_model",
574
+ )
575
+
576
+
577
+ def _is_codex_quota_obs(rec: dict) -> bool:
578
+ return (
579
+ rec.get("t") == "obs"
580
+ and rec.get("provider") == "codex"
581
+ and (rec.get("payload") or {}).get("kind") == _QUOTA_OBS_KIND
582
+ )
583
+
584
+
585
+ def _quota_applier(decoded) -> int | None:
586
+ """Cache leg (spec §5.2 step 3): materialize this batch's Codex quota obs
587
+ into cache.db `quota_window_snapshots`, under a NON-BLOCKING
588
+ `cache.db.codex.lock`. Contract (journal seam): `(decoded) -> stop | None`,
589
+ `decoded = [(record, segment, offset), ...]` in canonical order.
590
+
591
+ - No Codex quota obs in the batch → return None (no flock taken).
592
+ - Busy codex flock, OR a cache write it cannot complete → PREFIX-STOP:
593
+ return the index of the FIRST codex quota obs, so the cycle processes only
594
+ `decoded[:stop]` and advances the cursor to `decoded[stop]`'s offset,
595
+ retrying the remainder next cycle (the scalar cursor never advances past an
596
+ unmaterialized obs — spec §5.2 step 3).
597
+ - Flock acquired + all obs upserted → return None (full consumption).
598
+ """
599
+ quota_idx = [i for i, (rec, _s, _o) in enumerate(decoded)
600
+ if _is_codex_quota_obs(rec)]
601
+ if not quota_idx:
602
+ return None
603
+ _cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
604
+ fd = os.open(str(_cctally_core.CACHE_LOCK_CODEX_PATH),
605
+ os.O_RDWR | os.O_CREAT, 0o600)
606
+ try:
607
+ try:
608
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
609
+ except (BlockingIOError, OSError):
610
+ return quota_idx[0] # busy codex flock -> prefix-stop
611
+ try:
612
+ cache = sqlite3.connect(str(_cctally_core.CACHE_DB_PATH), timeout=15.0)
613
+ except sqlite3.Error as exc: # pragma: no cover — cache.db unopenable
614
+ print(f"[ingest] quota cache leg connect failed: {exc}", file=sys.stderr)
615
+ return quota_idx[0]
616
+ try:
617
+ cache.execute("PRAGMA busy_timeout=15000")
618
+ cache.execute("BEGIN IMMEDIATE")
619
+ for i in quota_idx:
620
+ p = decoded[i][0]["payload"]
621
+ cache.execute(_QUOTA_SNAPSHOT_INSERT,
622
+ tuple(p.get(col) for col in _QUOTA_SNAPSHOT_COLS))
623
+ cache.commit()
624
+ except sqlite3.Error as exc:
625
+ try:
626
+ cache.rollback()
627
+ except sqlite3.Error:
628
+ pass
629
+ # Could not materialize -> prefix-stop so the cursor holds and the
630
+ # next cycle retries (the obs stay durable in the journal regardless).
631
+ print(f"[ingest] quota cache leg write failed: {exc}", file=sys.stderr)
632
+ return quota_idx[0]
633
+ finally:
634
+ cache.close()
635
+ return None
636
+ finally:
637
+ try:
638
+ fcntl.flock(fd, fcntl.LOCK_UN)
639
+ finally:
640
+ os.close(fd)
641
+
642
+
643
+ # Wire the seam (declared None near the top as the contract stub). Always-on:
644
+ # a Claude-only cycle's scan finds no Codex quota obs and returns None before
645
+ # any flock/DB touch, so the cost is one list comprehension over the batch.
646
+ QUOTA_APPLIER = _quota_applier
647
+
648
+
649
+ # --------------------------------------------------------------------------
650
+ # fold appliers (spec §5.3)
651
+ # --------------------------------------------------------------------------
652
+
653
+ def _resolve_ref(conn: sqlite3.Connection, table: str, logical_id) -> int | None:
654
+ """Resolve a logical journal id to its rebuilt rowid in `table` via the
655
+ `journal_id` column (spec §4.2 FK rule).
656
+
657
+ A falsy logical id (0 / "0" / None / "") is the "no FK" sentinel — e.g.
658
+ `reset_event_id` defaults to 0 — and resolves to 0 without a lookup. An
659
+ unresolved id returns None so the caller can decide (Tasks 6-7).
660
+ """
661
+ if logical_id in (0, "0", None, ""):
662
+ return 0
663
+ row = conn.execute(
664
+ f"SELECT id FROM {table} WHERE journal_id = ?", (logical_id,)
665
+ ).fetchone()
666
+ if row is None:
667
+ return None
668
+ return int(row[0])
669
+
670
+
671
+ def _insert_or_ignore(conn: sqlite3.Connection, table: str, cols: dict):
672
+ keys = list(cols.keys())
673
+ colnames = ", ".join(keys)
674
+ placeholders = ", ".join("?" for _ in keys)
675
+ return conn.execute(
676
+ f"INSERT OR IGNORE INTO {table} ({colnames}) VALUES ({placeholders})",
677
+ tuple(cols[k] for k in keys),
678
+ )
679
+
680
+
681
+ def _reverse_ref(conn: sqlite3.Connection, ref_table: str, rowid) -> "str | None":
682
+ """Reverse of `_resolve_ref` — a FK rowid to the referenced row's *logical*
683
+ id (its `journal_id`), for building a harvest evt's logical-FK ref (spec
684
+ §4.2). The falsy sentinel (0 / None) — e.g. `reset_event_id`'s no-event
685
+ default — maps to the literal ``"0"`` so the id/payload stay stable across
686
+ replay. A referenced row whose `journal_id` is still NULL returns None — a
687
+ harvest-order violation (the referenced family must harvest first); the
688
+ caller degrades loudly rather than journaling an unresolvable ref.
689
+ """
690
+ if rowid in (0, None):
691
+ return "0"
692
+ row = conn.execute(
693
+ f"SELECT journal_id FROM {ref_table} WHERE id = ?", (rowid,)
694
+ ).fetchone()
695
+ if row is None or row[0] is None:
696
+ return None
697
+ return row[0]
698
+
699
+
700
+ def _now_iso() -> str:
701
+ """Fallback capture time for a derived evt with no natural `at` column
702
+ (UTC, seconds, ``Z``). Live emission passes the triggering record's `at`."""
703
+ return (
704
+ dt.datetime.now(dt.timezone.utc)
705
+ .isoformat(timespec="seconds")
706
+ .replace("+00:00", "Z")
707
+ )
708
+
709
+
710
+ def _usage_snapshot_fold_decision(conn, payload) -> tuple[bool, object]:
711
+ """The apply-time dedup for a Claude rate-limit obs — the exact predicate
712
+ ported from `cmd_record_usage`'s insert guard (bin/_cctally_record.py), now
713
+ at fold time (spec §4.5 / §5.3).
714
+
715
+ Returns `(skip, adjusted_five_hour_percent)`:
716
+ - reset-aware 7d HWM clamp (`_reset_aware_floor` + reset-aware MAX +
717
+ `hwm_clamp_applies`) → skip when a lower 7d % would be clamped;
718
+ - the 5h clamp adjusts `five_hour_percent` UP to the in-window MAX but
719
+ never gates (mirrors the nested-else in the live code);
720
+ - dedup vs the latest snapshot in the week: both percents unchanged → skip.
721
+ """
722
+ week_start_date = payload["week_start_date"]
723
+ week_start_at = payload.get("week_start_at")
724
+ week_end_at = payload.get("week_end_at")
725
+ weekly_percent = float(payload["weekly_percent"])
726
+ five_hour_percent = payload.get("five_hour_percent")
727
+ five_hour_window_key = payload.get("five_hour_window_key")
728
+
729
+ clamp_floor_iso = _cctally_core._reset_aware_floor(
730
+ conn, week_start_date, week_start_at, week_end_at
731
+ ) or "1970-01-01T00:00:00Z"
732
+ max_row = conn.execute(
733
+ "SELECT MAX(weekly_percent) FROM weekly_usage_snapshots "
734
+ "WHERE week_start_date = ? AND unixepoch(captured_at_utc) >= unixepoch(?)",
735
+ (week_start_date, clamp_floor_iso),
736
+ ).fetchone()
737
+ max_v = max_row[0] if max_row else None
738
+ if _lib_record.hwm_clamp_applies(weekly_percent, max_v):
739
+ return True, five_hour_percent
740
+
741
+ adjusted_5h = five_hour_percent
742
+ if five_hour_percent is not None and five_hour_window_key is not None:
743
+ max_5h_row = conn.execute(
744
+ "SELECT MAX(five_hour_percent) FROM weekly_usage_snapshots "
745
+ "WHERE five_hour_window_key = ? "
746
+ " AND unixepoch(captured_at_utc) >= unixepoch(COALESCE("
747
+ " (SELECT effective_reset_at_utc FROM five_hour_reset_events "
748
+ " WHERE five_hour_window_key = ? ORDER BY id DESC LIMIT 1),"
749
+ " '1970-01-01T00:00:00Z'))",
750
+ (int(five_hour_window_key), int(five_hour_window_key)),
751
+ ).fetchone()
752
+ max_5h = max_5h_row[0] if max_5h_row else None
753
+ if _lib_record.hwm_clamp_applies(float(five_hour_percent), max_5h):
754
+ adjusted_5h = float(max_5h)
755
+
756
+ last = conn.execute(
757
+ "SELECT weekly_percent, five_hour_percent FROM weekly_usage_snapshots "
758
+ "WHERE week_start_date = ? ORDER BY captured_at_utc DESC, id DESC LIMIT 1",
759
+ (week_start_date,),
760
+ ).fetchone()
761
+ if last is not None and float(last[0]) == weekly_percent:
762
+ last_5h = last[1]
763
+ if adjusted_5h is None or (
764
+ last_5h is not None and float(last_5h) == float(adjusted_5h)
765
+ ):
766
+ return True, adjusted_5h
767
+ return False, adjusted_5h
768
+
769
+
770
+ # NOTE (rev 3): the direct obs -> weekly_usage_snapshots fold is GONE. That
771
+ # table is now written ONLY via `snapshot_accept` Model-A evts (spec §5.3); the
772
+ # accept/skip DECISION above (`_usage_snapshot_fold_decision`) is 6b's
773
+ # snapshot_accept deriver's to make, ONCE, at capture time — the decision itself
774
+ # is journaled, so replay never re-derives it.
775
+
776
+
777
+ def _apply_op_weekly_credit_floor(conn, record) -> None:
778
+ """Fold a `record-credit` `op` into `weekly_credit_floors` (spec §5.3
779
+ "fold op"). `INSERT OR IGNORE` dedups on both `journal_id` and the table's
780
+ own `UNIQUE(week_start_date, effective_at_utc)`."""
781
+ payload = record["payload"]
782
+ _insert_or_ignore(conn, "weekly_credit_floors", {
783
+ "journal_id": record["id"],
784
+ "week_start_date": payload["week_start_date"],
785
+ "effective_at_utc": payload["effective_at_utc"],
786
+ "observed_pre_credit_pct": float(payload["observed_pre_credit_pct"]),
787
+ "applied_at_utc": payload.get("applied_at_utc", record["at"]),
788
+ })
789
+
790
+
791
+ # Obs/op fold registry (spec §5.3 "fold op"). Keyed by `payload.kind`; the
792
+ # built-in `_pipeline_op_fold` pipeline hook dispatches through it. 6b may
793
+ # register more op folds; obs no longer fold directly (see the NOTE above).
794
+ FOLD_APPLIERS = {"weekly_credit_floor": _apply_op_weekly_credit_floor}
795
+
796
+
797
+ # --------------------------------------------------------------------------
798
+ # evt fold appliers (step 4a replay + the emit_model_a apply path)
799
+ # --------------------------------------------------------------------------
800
+
801
+ _BLOCK_CHILDREN = (
802
+ ("_models", "five_hour_block_models"),
803
+ ("_projects", "five_hour_block_projects"),
804
+ )
805
+ _BLOCK_CHILD_KEYS = frozenset(k for k, _t in _BLOCK_CHILDREN)
806
+
807
+
808
+ def _apply_generic_evt(conn, evt):
809
+ """Fold an evt line into its target table (spec §5.3), returning the sqlite
810
+ cursor of the `INSERT OR IGNORE`.
811
+
812
+ Table + logical-FK spec come from `_EVT_SPECS[payload['kind']]`. Non-FK
813
+ payload keys map to same-named columns; FK-ref keys resolve logical ids to
814
+ rowids via `_resolve_ref`. `INSERT OR IGNORE` keyed on `journal_id` (and the
815
+ table's natural-key UNIQUE) makes replay idempotent. An unknown kind, or a
816
+ spec with no `table`, is a no-op (additive-evolution tolerance, spec §4.2).
817
+ """
818
+ payload = evt.get("payload") or {}
819
+ spec = _EVT_SPECS.get(payload.get("kind"))
820
+ if spec is None or spec.table is None:
821
+ return None
822
+ cols = {"journal_id": evt["id"]}
823
+ for key, value in payload.items():
824
+ if key == "kind":
825
+ continue
826
+ if key in spec.fk_refs:
827
+ column, ref_table = spec.fk_refs[key]
828
+ cols[column] = _resolve_ref(conn, ref_table, value)
829
+ else:
830
+ cols[key] = value
831
+ # Re-derive any projection-FK columns from a journaled natural-key column
832
+ # (spec §5.3 — e.g. five_hour_milestones.block_id from five_hour_window_key,
833
+ # since the open block is a projection with no logical id). 0 when absent.
834
+ for column, (ref_table, lookup_col) in spec.derived_fk.items():
835
+ row = conn.execute(
836
+ f"SELECT id FROM {ref_table} WHERE {lookup_col} = ?",
837
+ (cols.get(lookup_col),),
838
+ ).fetchone()
839
+ cols[column] = int(row[0]) if row is not None else 0
840
+ return _insert_or_ignore(conn, spec.table, cols)
841
+
842
+
843
+ def _apply_weekly_credit_effects(conn, evt):
844
+ """Apply a `weekly_credit_effects` evt (spec §5.3 event+effects). The
845
+ same-window sub-25pp credit writes NO reset row, so its DESTRUCTIVE effects
846
+ ride this vehicle: delete the stale-replica snapshots by their logical
847
+ `journal_id` (idempotent — deleting an already-absent id is a clean no-op),
848
+ then force the HWM floor file down (mirrors `_apply_credit` step 4b; an
849
+ idempotent overwrite). The synthetic post-credit snapshots ride their own
850
+ `snapshot_accept` evts. Effects-only — no target-table row, so no journal_id
851
+ of its own; convergence is the natural idempotence of DELETE + overwrite.
852
+
853
+ A ``--force`` re-record's destructive clear (the ingest-path replacement for
854
+ ``_force_clear_credit``) rides the SAME evt: ``suppression`` also carries the
855
+ OLD command-owned synthetic snapshots' `journal_id`s (deleted from the same
856
+ ``weekly_usage_snapshots`` table), and ``floor_suppression`` carries the OLD
857
+ ``weekly_credit_floors`` rows' `journal_id`s (the prior credit's floor,
858
+ NEVER the new op's own floor — the op fold owns that). Both delete by logical
859
+ id, so replay reproduces the clear deterministically and idempotently; the
860
+ NEW floor + NEW synthetic are keyed by the current op's id and never appear
861
+ in either list, so this effect is order-independent w.r.t. them (spec §5.3)."""
862
+ payload = evt.get("payload") or {}
863
+ table = payload.get("suppression_table", "weekly_usage_snapshots")
864
+ for logical_id in (payload.get("suppression") or []):
865
+ conn.execute(f"DELETE FROM {table} WHERE journal_id = ?", (logical_id,))
866
+ for logical_id in (payload.get("floor_suppression") or []):
867
+ conn.execute(
868
+ "DELETE FROM weekly_credit_floors WHERE journal_id = ?", (logical_id,))
869
+ floor = payload.get("hwm_floor")
870
+ if floor:
871
+ try:
872
+ (_cctally_core.APP_DIR / "hwm-7d").write_text(
873
+ f"{floor['week_start_date']} {floor['weekly_percent']}\n"
874
+ )
875
+ except OSError:
876
+ pass
877
+ return None
878
+
879
+
880
+ def _apply_quota_alert_arming(conn, evt):
881
+ """Fold a `quota_alert_arming` evt (spec §5.3 "state", Task 7 Item 5). The
882
+ quota-alert arming boundary is journaled state — its `activated_at_utc` is a
883
+ forward-only alert boundary that MUST survive a stats.db rebuild so the
884
+ reconcile honors it (no historical re-fires). Applied as an UPSERT on the
885
+ arming natural key, in canonical order, so the latest state per key wins and
886
+ re-applying an already-present evt is a clean no-op. `quota_alert_arming` has
887
+ no `journal_id` column (it is not in the Task-4 additive list); idempotence
888
+ is the natural-key upsert, not a journal_id INSERT OR IGNORE."""
889
+ p = evt.get("payload") or {}
890
+ conn.execute(
891
+ "INSERT INTO quota_alert_arming "
892
+ "(source, source_root_key, logical_limit_key, observed_slot, "
893
+ " window_minutes, rule_fingerprint, activated_at_utc) "
894
+ "VALUES (?,?,?,?,?,?,?) "
895
+ "ON CONFLICT(source, source_root_key, logical_limit_key, observed_slot, "
896
+ " window_minutes) DO UPDATE SET "
897
+ " rule_fingerprint=excluded.rule_fingerprint, "
898
+ " activated_at_utc=excluded.activated_at_utc",
899
+ (p.get("source"), p.get("source_root_key"), p.get("logical_limit_key"),
900
+ p.get("observed_slot"), p.get("window_minutes"),
901
+ p.get("rule_fingerprint"), p.get("activated_at_utc")),
902
+ )
903
+ return None
904
+
905
+
906
+ def _apply_block_close(conn, evt):
907
+ """Fold a `five_hour_block_close` evt (spec §5.3): insert the frozen parent
908
+ block (`INSERT OR IGNORE` on window-key / journal_id), then its embedded
909
+ `_models`/`_projects` rollup children under the resolved parent block_id
910
+ (each idempotent on its own natural key). Live harvest only STAMPS the
911
+ parent's journal_id — 6b's close hook already wrote parent+children — so this
912
+ insert path runs for replay/rebuild."""
913
+ payload = evt.get("payload") or {}
914
+ parent = {"journal_id": evt["id"]}
915
+ children = {}
916
+ for key, value in payload.items():
917
+ if key == "kind":
918
+ continue
919
+ if key in _BLOCK_CHILD_KEYS:
920
+ children[key] = value or []
921
+ continue
922
+ parent[key] = value
923
+ _insert_or_ignore(conn, "five_hour_blocks", parent)
924
+ prow = conn.execute(
925
+ "SELECT id FROM five_hour_blocks WHERE five_hour_window_key = ?",
926
+ (parent.get("five_hour_window_key"),),
927
+ ).fetchone()
928
+ if prow is None:
929
+ return None
930
+ block_id = int(prow[0])
931
+ for payload_key, child_table in _BLOCK_CHILDREN:
932
+ for child in children.get(payload_key, []):
933
+ cols = dict(child)
934
+ cols["block_id"] = block_id
935
+ _insert_or_ignore(conn, child_table, cols)
936
+ return None
937
+
938
+
939
+ def _apply_reset_with_suppression(conn, evt):
940
+ """Fold a reset/credit evt (`week_reset`/`five_hour_credit`) that carries a
941
+ `suppression` list (spec §5.3 event+effects, Design B). Insert the reset row
942
+ (idempotent on natural key / journal_id via the generic column map — the
943
+ `suppression`/`suppression_table` keys are effects, NOT columns, so they are
944
+ stripped), THEN apply the destructive stale-replica DELETE by logical
945
+ `journal_id` (idempotent — deleting an already-absent id is a clean no-op,
946
+ mirroring `_apply_weekly_credit_effects`). The synthetic post-credit
947
+ snapshots ride their own `snapshot_accept` evts; this vehicle only inserts
948
+ the reset row and replays its suppression."""
949
+ payload = evt.get("payload") or {}
950
+ spec = _EVT_SPECS.get(payload.get("kind"))
951
+ if spec is None or spec.table is None:
952
+ return None
953
+ cols = {"journal_id": evt["id"]}
954
+ for key, value in payload.items():
955
+ if key in ("kind", "suppression", "suppression_table"):
956
+ continue
957
+ if key in spec.fk_refs:
958
+ column, ref_table = spec.fk_refs[key]
959
+ cols[column] = _resolve_ref(conn, ref_table, value)
960
+ else:
961
+ cols[key] = value
962
+ _insert_or_ignore(conn, spec.table, cols)
963
+ supp_table = payload.get("suppression_table", "weekly_usage_snapshots")
964
+ for logical_id in (payload.get("suppression") or []):
965
+ conn.execute(f"DELETE FROM {supp_table} WHERE journal_id = ?", (logical_id,))
966
+ return None
967
+
968
+
969
+ def _apply_evt(conn, evt):
970
+ """Dispatch one evt line to its fold applier by `payload.kind` (step 4a
971
+ replay + the emit_model_a apply path). A kind with a bespoke `applier`
972
+ (weekly_credit_effects, five_hour_block_close) uses it; everything else
973
+ goes through the generic column-map fold. Apply-only: NO alert dispatch,
974
+ NO ctx — replay is structurally unable to fire alerts (spec §5.2 step 4a)."""
975
+ spec = _EVT_SPECS.get((evt.get("payload") or {}).get("kind"))
976
+ if spec is not None and spec.applier is not None:
977
+ return spec.applier(conn, evt)
978
+ return _apply_generic_evt(conn, evt)
979
+
980
+
981
+ # --------------------------------------------------------------------------
982
+ # harvest registry (natural-keyed families, spec §5.3)
983
+ # --------------------------------------------------------------------------
984
+
985
+ _HARVEST_SPECS = [
986
+ _HarvestSpec(
987
+ "week_reset_events", "week_reset", "wr",
988
+ id_parts=("old_week_end_at", "new_week_end_at"),
989
+ at_column="detected_at_utc", order=30, suppression=True,
990
+ ),
991
+ _HarvestSpec(
992
+ "five_hour_reset_events", "five_hour_credit", "fhc",
993
+ id_parts=("five_hour_window_key", "effective_reset_at_utc"),
994
+ at_column="detected_at_utc", order=30, suppression=True,
995
+ ),
996
+ _HarvestSpec(
997
+ "five_hour_blocks", "five_hour_block_close", "fhbc",
998
+ id_parts=("five_hour_window_key",),
999
+ at_column="last_updated_at_utc", order=40, closed_only=True,
1000
+ children=_BLOCK_CHILDREN,
1001
+ ),
1002
+ _HarvestSpec(
1003
+ "percent_milestones", "percent_milestone", "pm",
1004
+ id_parts=("week_start_date", "reset_event_id", "percent_threshold"),
1005
+ fk_refs={
1006
+ "usage_snapshot_id": ("weekly_usage_snapshots", "usage_snapshot_ref"),
1007
+ "cost_snapshot_id": ("weekly_cost_snapshots", "cost_snapshot_ref"),
1008
+ "reset_event_id": ("week_reset_events", "reset_event_ref"),
1009
+ },
1010
+ at_column="captured_at_utc", order=60,
1011
+ ),
1012
+ _HarvestSpec(
1013
+ "five_hour_milestones", "five_hour_milestone", "fhm",
1014
+ id_parts=("five_hour_window_key", "reset_event_id", "percent_threshold"),
1015
+ fk_refs={
1016
+ "usage_snapshot_id": ("weekly_usage_snapshots", "usage_snapshot_ref"),
1017
+ "reset_event_id": ("five_hour_reset_events", "reset_event_ref"),
1018
+ },
1019
+ # block_id points at the OPEN five_hour_blocks row (a projection with no
1020
+ # journal_id) — re-derive it at fold from the journaled window key
1021
+ # instead of a broken logical FK (harvest-order violation otherwise).
1022
+ derived_fk={"block_id": ("five_hour_blocks", "five_hour_window_key")},
1023
+ at_column="captured_at_utc", order=60,
1024
+ ),
1025
+ _HarvestSpec(
1026
+ "budget_milestones", "budget", "bm",
1027
+ id_parts=("vendor", "period_start_at", "period", "threshold"),
1028
+ at_column="crossed_at_utc", order=60,
1029
+ ),
1030
+ _HarvestSpec(
1031
+ "projected_milestones", "projected", "pjm",
1032
+ id_parts=("week_start_at", "period", "metric", "threshold"),
1033
+ at_column="crossed_at_utc", order=60,
1034
+ ),
1035
+ _HarvestSpec(
1036
+ "project_budget_milestones", "project_budget", "pbm",
1037
+ id_parts=("week_start_at", "project_key", "threshold"),
1038
+ at_column="crossed_at_utc", order=60,
1039
+ ),
1040
+ ]
1041
+
1042
+
1043
+ # Evt fold specs, keyed by `payload.kind`. The Model-A families are declared
1044
+ # here (snapshot_accept + weekly_cost_snapshot are generic column-map folds;
1045
+ # weekly_credit_effects is effects-only). The natural-keyed harvest families
1046
+ # contribute the INVERSE of their `fk_refs` so one registry drives both harvest
1047
+ # (rowid -> logical) and fold (logical -> rowid) without drift. `order` is the
1048
+ # FK-dependency fold order (referenced families before referencing ones).
1049
+ _EVT_SPECS = {
1050
+ "snapshot_accept": _EvtSpec("weekly_usage_snapshots", order=10),
1051
+ "weekly_cost_snapshot": _EvtSpec("weekly_cost_snapshots", order=20),
1052
+ "weekly_credit_effects": _EvtSpec(
1053
+ None, order=50, applier=_apply_weekly_credit_effects),
1054
+ # Quota-alert arming state (Task 7 Item 5): an independent stats.db table
1055
+ # with no FK into the journal-covered families and its own natural-key
1056
+ # upsert applier. order is arbitrary among evts (no cross-family FK).
1057
+ "quota_alert_arming": _EvtSpec(
1058
+ None, order=45, applier=_apply_quota_alert_arming),
1059
+ }
1060
+ for _hs in _HARVEST_SPECS:
1061
+ if _hs.children:
1062
+ _applier = _apply_block_close
1063
+ elif _hs.suppression:
1064
+ _applier = _apply_reset_with_suppression
1065
+ else:
1066
+ _applier = None
1067
+ _EVT_SPECS[_hs.kind] = _EvtSpec(
1068
+ _hs.table,
1069
+ fk_refs={ref_key: (col, ref_table)
1070
+ for col, (ref_table, ref_key) in _hs.fk_refs.items()},
1071
+ order=_hs.order,
1072
+ applier=_applier,
1073
+ derived_fk=dict(_hs.derived_fk),
1074
+ )
1075
+
1076
+
1077
+ # --------------------------------------------------------------------------
1078
+ # Model-A emission + harvest (spec §5.3, step 4c)
1079
+ # --------------------------------------------------------------------------
1080
+
1081
+ def emit_model_a(ctx, *, kind, evt_id, table, columns, refs=None, at=None):
1082
+ """Emit one Model-A evt (spec §5.3): append the evt line FIRST (leaf lock,
1083
+ inside the txn — legal per the lock-order law; fsync'd before the commit
1084
+ that indexes it), then apply it through the SAME fold applier replay uses,
1085
+ so live emission and crash-replay converge by construction. Used by 6b's
1086
+ obs-derivation hooks for the no-natural-key families — `snapshot_accept`
1087
+ (the ONLY writer of weekly_usage_snapshots now), `weekly_cost_snapshot`,
1088
+ and `weekly_credit_effects` (effects-only, `table=None`).
1089
+
1090
+ `columns` are the target row's canonical column values; `refs` (optional)
1091
+ are logical-FK ref payload keys the fold resolves to rowids. `at` should be
1092
+ the triggering record's capture time (`ctx.as_of_for(record)`) for replay
1093
+ determinism. Returns the target row's rowid (freshly inserted OR a converged
1094
+ crash-replay — 6b callers need it for FK linkage regardless of which cycle
1095
+ inserted it), or None for an effects-only family (`table=None`).
1096
+ """
1097
+ payload = dict(columns)
1098
+ if refs:
1099
+ payload.update(refs)
1100
+ evt = _lib_journal.make_evt(kind=kind, id=evt_id, at=(at or _now_iso()),
1101
+ payload=payload)
1102
+ append_record(evt)
1103
+ ctx.events_emitted += 1
1104
+ _apply_evt(ctx.conn, evt)
1105
+ if table is None:
1106
+ return None
1107
+ row = ctx.conn.execute(
1108
+ f"SELECT id FROM {table} WHERE journal_id = ?", (evt_id,)
1109
+ ).fetchone()
1110
+ return int(row[0]) if row is not None else None
1111
+
1112
+
1113
+ def _build_harvest_evt(ctx, spec, row):
1114
+ """Build the evt for one harvested natural-keyed row (spec §5.3): map the
1115
+ plain columns, replace FK rowids with their referenced row's logical id
1116
+ (reverse lookup), embed rollup children, and assemble the opaque natural-key
1117
+ id from `spec.id_prefix` + `spec.id_parts` (FK parts contribute their
1118
+ logical id).
1119
+
1120
+ For a suppression family (`week_reset`/`five_hour_credit`, Design B) the
1121
+ reset's destructive effects also ride the evt: the list of logical
1122
+ `journal_id`s the live pipeline hook captured (in `ctx.suppression_map`,
1123
+ keyed on this row's `id_parts` values) is attached as `payload["suppression"]`
1124
+ so the effects replay deterministically. The id stays the pure natural key —
1125
+ `suppression` is an effect, never an id component."""
1126
+ conn = ctx.conn
1127
+ fk_cols = set(spec.fk_refs.keys())
1128
+ # derived-FK columns (e.g. block_id) are NOT journaled — the raw rowid is
1129
+ # not stable and the fold re-derives them from a journaled natural key.
1130
+ skip_cols = fk_cols | set(spec.derived_fk.keys())
1131
+ payload = {}
1132
+ for key in row.keys():
1133
+ if key in ("id", "journal_id") or key in skip_cols:
1134
+ continue
1135
+ payload[key] = row[key]
1136
+ refs = {}
1137
+ for col, (ref_table, ref_key) in spec.fk_refs.items():
1138
+ logical = _reverse_ref(conn, ref_table, row[col])
1139
+ if logical is None:
1140
+ raise JournalError(
1141
+ f"harvest {spec.kind}: unresolved FK {col} -> {ref_table} "
1142
+ "(referenced row has no journal_id — harvest-order violation)")
1143
+ payload[ref_key] = logical
1144
+ refs[col] = logical
1145
+ for payload_key, child_table in spec.children:
1146
+ child_rows = conn.execute(
1147
+ f"SELECT * FROM {child_table} WHERE block_id = ? ORDER BY id",
1148
+ (row["id"],),
1149
+ ).fetchall()
1150
+ payload[payload_key] = [
1151
+ {k: cr[k] for k in cr.keys() if k not in ("id", "block_id")}
1152
+ for cr in child_rows
1153
+ ]
1154
+ parts = [refs[name] if name in fk_cols else row[name]
1155
+ for name in spec.id_parts]
1156
+ if spec.suppression:
1157
+ supp = ctx.suppression_map.get(tuple(row[name] for name in spec.id_parts))
1158
+ if supp:
1159
+ payload["suppression"] = list(supp)
1160
+ eid = _lib_journal.evt_id(spec.id_prefix, *parts)
1161
+ at = row[spec.at_column] if spec.at_column else _now_iso()
1162
+ return _lib_journal.make_evt(kind=spec.kind, id=eid, at=at, payload=payload)
1163
+
1164
+
1165
+ def _harvest(ctx) -> None:
1166
+ """Step 4c: journal + stamp every natural-keyed row inserted this cycle
1167
+ (`journal_id IS NULL`). Families harvest in dependency order so a referenced
1168
+ family (resets, blocks) stamps its journal_id before a referencing family
1169
+ (milestones) reverse-looks-it-up (spec §5.3 / Appendix B I4 P2-8). Each evt
1170
+ is appended+fsync'd before its row is stamped, inside the cycle's txn."""
1171
+ conn = ctx.conn
1172
+ for spec in sorted(_HARVEST_SPECS, key=lambda s: s.order):
1173
+ where = "journal_id IS NULL"
1174
+ if spec.closed_only:
1175
+ where += " AND is_closed = 1"
1176
+ rows = conn.execute(
1177
+ f"SELECT * FROM {spec.table} WHERE {where} ORDER BY id"
1178
+ ).fetchall()
1179
+ for row in rows:
1180
+ evt = _build_harvest_evt(ctx, spec, row)
1181
+ append_record(evt)
1182
+ ctx.events_emitted += 1
1183
+ conn.execute(
1184
+ f"UPDATE {spec.table} SET journal_id = ? WHERE id = ?",
1185
+ (evt["id"], row["id"]),
1186
+ )
1187
+
1188
+
1189
+ # --------------------------------------------------------------------------
1190
+ # pipeline (step 4b) + post-commit alert dispatch (step 6)
1191
+ # --------------------------------------------------------------------------
1192
+
1193
+ def _pipeline_op_fold(ctx, record) -> None:
1194
+ """Built-in pipeline hook: fold an obs/op record whose `payload.kind` has a
1195
+ registered `FOLD_APPLIERS` entry (spec §5.3 "fold op" — the
1196
+ weekly_credit_floor op ships here). No-op for every other record."""
1197
+ applier = FOLD_APPLIERS.get((record.get("payload") or {}).get("kind"))
1198
+ if applier is not None:
1199
+ applier(ctx.conn, record)
1200
+
1201
+
1202
+ PIPELINE.append(_pipeline_op_fold)
1203
+
1204
+
1205
+ def _dispatch_pending_alerts(alerts: list) -> None:
1206
+ """Default post-commit dispatch (spec §5.2 step 6): fire each queued alert
1207
+ payload through the cctally dispatch glue (bin/_lib_alert_dispatch via
1208
+ `_dispatch_alert_notification`). Failures are logged, never raised — a bad
1209
+ payload can't suppress healthy ones, and a dispatch failure never rolls back
1210
+ a committed milestone (set-then-dispatch, docs/alerts-gotchas.md)."""
1211
+ cctally = sys.modules.get("cctally")
1212
+ dispatch = getattr(cctally, "_dispatch_alert_notification", None) if cctally else None
1213
+ if dispatch is None:
1214
+ return
1215
+ for payload in alerts:
1216
+ try:
1217
+ dispatch(payload, mode="real")
1218
+ except Exception as exc: # pragma: no cover — best-effort dispatch
1219
+ print(f"[alerts] dispatch failed: {exc}", file=sys.stderr)
1220
+
1221
+
1222
+ def _load_config_once() -> dict:
1223
+ """Read config once per cycle for the pipeline hooks (spec §5.2 step 4b).
1224
+ Config-at-ingest is acceptable because derived records are journaled — replay
1225
+ never re-derives, so a config change between capture and ingest only shifts
1226
+ which cycle derived the event. Only called for a non-empty batch, so an empty
1227
+ cycle never touches config."""
1228
+ cctally = sys.modules.get("cctally")
1229
+ if cctally is not None and hasattr(cctally, "load_config"):
1230
+ try:
1231
+ return cctally.load_config()
1232
+ except Exception:
1233
+ return {}
1234
+ return {}
1235
+
1236
+
1237
+ def _run_config_reconcile(ctx, reconcile_config) -> None:
1238
+ """Design C (DB journal redesign §5.3): run the three Task-5 budget-reconcile
1239
+ chokepoints INSIDE the cycle transaction, on `ctx.conn`, after the batch
1240
+ pipeline and BEFORE harvest — so any newly-latched crossing row (journal_id
1241
+ NULL, `commit=False`) is picked up by the natural-keyed budget harvest and
1242
+ journaled as a `budget`/`projected`/`project_budget` evt.
1243
+
1244
+ `reconcile_config` is `{"budget": <validated_budget>, "touched_projects":
1245
+ set | None, "axes": set | None}` (6c widening + 6f axes gate).
1246
+ `touched_projects` threads into the per-project reconcile: a SCOPED `budget
1247
+ set/unset --project` write (6e/6f) passes `{root}` so touching project A never
1248
+ latches a sibling project B's crossed-but-not-yet-dispatched threshold — which
1249
+ would permanently suppress B's real alert (memory: the per-project reconcile's
1250
+ `touched_projects` contract). `None` reconciles every configured project (the
1251
+ config-set / dashboard-toggle / wholesale `budget.projects` "suppress the
1252
+ retroactive storm for all" case).
1253
+
1254
+ `axes` ⊆ {"budget", "codex_budget", "project_budget"} names which reconcile
1255
+ axes to run — the per-call-site touched-leaf mapping (6f writer reroute), so
1256
+ a `budget set` write reconciles ONLY the global axis and never latches a
1257
+ Codex/project crossing that its config write didn't touch. `axes = None`
1258
+ runs ALL three axes (the pre-6f behavior; kept so a caller that doesn't scope
1259
+ still reconciles everything).
1260
+
1261
+ There is NO journaled op line for a config write; this is a LIVE-only entry
1262
+ (never seen at rebuild — replay of the harvested budget evts reproduces the
1263
+ latched rows). The reconcile family is stamp-no-dispatch by construction
1264
+ (retroactive-storm suppression), so it never pushes to `ctx.pending_alerts`;
1265
+ passing the sink is vacuous for this path — the latch is recorded and
1266
+ journaled, never popped. `as_of=None` lets each reconcile stamp at its own
1267
+ (live) moment, which the harvest then freezes into the evt.
1268
+ """
1269
+ cctally = sys.modules.get("cctally")
1270
+ if cctally is None or not reconcile_config:
1271
+ return
1272
+ validated_budget = reconcile_config.get("budget")
1273
+ touched_projects = reconcile_config.get("touched_projects")
1274
+ axes = reconcile_config.get("axes") # None => run all (pre-6f behavior)
1275
+ if not validated_budget:
1276
+ return
1277
+
1278
+ def _wants(axis: str) -> bool:
1279
+ return axes is None or axis in axes
1280
+
1281
+ # The per-call guard is NOT the redundant belt the 6c order flagged for
1282
+ # removal: each reconcile self-guards best-effort AND never re-raises (the
1283
+ # reconcile family is stamp-no-dispatch and MUST NOT break the cycle, unlike
1284
+ # the milestone chokepoints that re-raise on a passed conn). Keeping the
1285
+ # guard here is deliberate defense-in-depth for that "never break the cycle
1286
+ # over a reconcile" contract — it holds even if a future reconcile forgets
1287
+ # its own self-guard. (P3 disposition: justified-in-comment, not dropped.)
1288
+ for axis, name in (
1289
+ ("budget", "_reconcile_budget_on_config_write"),
1290
+ ("codex_budget", "_reconcile_codex_budget_on_config_write"),
1291
+ ):
1292
+ if not _wants(axis):
1293
+ continue
1294
+ fn = getattr(cctally, name, None)
1295
+ if fn is None:
1296
+ continue
1297
+ try:
1298
+ fn(validated_budget, conn=ctx.conn)
1299
+ except Exception as exc: # best-effort; never break the cycle over a reconcile
1300
+ print(f"[budget-reconcile] {name} failed: {exc}", file=sys.stderr)
1301
+ # Per-project reconcile takes `touched_projects` as its 2nd positional
1302
+ # (scoped-vs-wholesale, above); split out from the loop for that one extra arg.
1303
+ if _wants("project_budget"):
1304
+ proj_fn = getattr(cctally, "_reconcile_project_budget_milestones_on_write", None)
1305
+ if proj_fn is not None:
1306
+ try:
1307
+ proj_fn(validated_budget, touched_projects, conn=ctx.conn)
1308
+ except Exception as exc: # best-effort; never break the cycle over a reconcile
1309
+ print(
1310
+ "[budget-reconcile] _reconcile_project_budget_milestones_on_write "
1311
+ f"failed: {exc}",
1312
+ file=sys.stderr,
1313
+ )
1314
+
1315
+
1316
+ def reconcile_budget_config(validated_budget, *, axes, touched_projects=None):
1317
+ """Route a budget-config-write forward-only reconcile THROUGH the ingest
1318
+ cycle (spec §5.3 Design C / Appendix A "dashboard config-change / forward-
1319
+ only budget reconciliations → opportunistic ingest cycle").
1320
+
1321
+ The single chokepoint the 6f writer reroute points every budget-config write
1322
+ site at (`budget set/set-codex/set-project/unset-project`, `config set
1323
+ budget.*`, dashboard POST /api/settings): instead of opening its own stats.db
1324
+ connection and writing the latched crossings directly (the last remaining
1325
+ direct-writer class), the site names the axes its touched config leaves feed
1326
+ and this runs those reconciles INSIDE `run_stats_ingest` on the cycle
1327
+ connection — so the latched crossing rows are journaled by the budget harvest
1328
+ and become rebuild-replayable.
1329
+
1330
+ Mode is OPPORTUNISTIC and the whole thing is exception-wrapped: a config
1331
+ write must NEVER fail (or block on a busy ingest lock) because a forward-only
1332
+ reconcile could not run — it is a best-effort retroactive-storm suppression,
1333
+ identical to today's fire-and-forget semantics. `axes` ⊆ {"budget",
1334
+ "codex_budget", "project_budget"}; empty `axes` (or a falsy budget) is a
1335
+ no-op. `touched_projects` scopes the per-project reconcile (spec §5.3)."""
1336
+ if not validated_budget or not axes:
1337
+ return
1338
+ try:
1339
+ run_stats_ingest(
1340
+ mode="opportunistic",
1341
+ reconcile_config={
1342
+ "budget": validated_budget,
1343
+ "touched_projects": touched_projects,
1344
+ "axes": set(axes),
1345
+ },
1346
+ )
1347
+ except Exception as exc: # best-effort; a config write must never fail here
1348
+ print(f"[budget-reconcile] ingest reconcile failed: {exc}", file=sys.stderr)
1349
+
1350
+
1351
+ # Fold order for an evt whose kind is unknown to this binary — sorts LAST so a
1352
+ # future kind never wedges before a known referenced family (additive tolerance).
1353
+ _UNKNOWN_EVT_SPEC = _EvtSpec(None, order=999)
1354
+
1355
+
1356
+ def _fold_order(evt) -> int:
1357
+ kind = (evt.get("payload") or {}).get("kind")
1358
+ return (_EVT_SPECS.get(kind) or _UNKNOWN_EVT_SPEC).order
1359
+
1360
+
1361
+ # --------------------------------------------------------------------------
1362
+ # the cycle (spec §5.2, revision 3)
1363
+ # --------------------------------------------------------------------------
1364
+
1365
+ def _run_cycle(conn: sqlite3.Connection, *, reconcile_config=None,
1366
+ codex_apply=None, post_commit=None) -> IngestResult:
1367
+ # Step 1: HW snapshot (leaf lock, µs). Lines appended after this — by other
1368
+ # processes OR by this cycle's own evt emission — are past HW and belong to
1369
+ # the next cycle (§5.2.1, closes the skipped-append race).
1370
+ hw = journal_high_water()
1371
+ # An empty journal (no segments yet) has nothing to consume. Normally that is
1372
+ # a no-op cycle — BUT a LIVE-only entry that appends no journal line of its
1373
+ # own must still run even on a still-empty journal: the Design-C budget-config
1374
+ # reconcile (§5.3, 6f) AND the Codex `codex_apply` leg (Task 7 — the quota
1375
+ # projection re-materializer + on-demand codex budget/projected firings; a
1376
+ # user may run `cctally budget` or a Codex hook-tick before any Claude usage
1377
+ # is recorded). In that case fall through with an empty batch and no cursor to
1378
+ # advance — any harvested budget evt lands in the freshly-created first
1379
+ # segment past the (absent) HW and replays idempotently on the next cycle.
1380
+ decoded: list = [] # (record, segment, offset)
1381
+ malformed = 0
1382
+ cursor_target = None
1383
+ if hw is None:
1384
+ if reconcile_config is None and codex_apply is None:
1385
+ return IngestResult(ran=True, consumed=0, malformed=0,
1386
+ events_emitted=0, alerts=[])
1387
+ else:
1388
+ hw_seg, hw_size = hw
1389
+
1390
+ # Step 2: read cursor -> HW in canonical order; decode, counting
1391
+ # malformed. Keep each record's (segment, offset) so the cache leg can
1392
+ # truncate the cursor on a prefix-stop.
1393
+ cursor = _read_cursor(conn)
1394
+ for seg, off, raw in _read_range(cursor, hw):
1395
+ rec = _lib_journal.decode_line(raw)
1396
+ if rec is None:
1397
+ malformed += 1
1398
+ continue
1399
+ decoded.append((rec, seg, off))
1400
+
1401
+ # Step 3: cache leg (Codex quota) BEFORE the stats txn (lock-order law).
1402
+ # QUOTA_APPLIER attempts the cache.db.codex.lock NB upsert; on a busy
1403
+ # flock it returns a prefix-stop index — the cycle processes
1404
+ # decoded[:stop], sets the cursor to decoded[stop]'s offset, and retries
1405
+ # the remainder next cycle (§5.2 step 3; prefix consumption keeps the
1406
+ # scalar cursor sound).
1407
+ cursor_target = (hw_seg, hw_size)
1408
+ if QUOTA_APPLIER is not None:
1409
+ stop = QUOTA_APPLIER(decoded)
1410
+ if stop is not None:
1411
+ _rec, stop_seg, stop_off = decoded[stop]
1412
+ cursor_target = (stop_seg, stop_off)
1413
+ decoded = decoded[:stop]
1414
+
1415
+ records = [r for (r, _s, _o) in decoded]
1416
+ batch = [r for r in records if r.get("t") in ("obs", "op")]
1417
+ journal_evts = [r for r in records if r.get("t") == "evt"]
1418
+
1419
+ # Step 4: ONE BEGIN IMMEDIATE — replay + pipeline + derived-fact journaling +
1420
+ # cursor advance, atomic (§5.2 crash boundary). A crash before COMMIT rolls
1421
+ # back rows + cursor together; the fsync'd evt lines replay idempotently in
1422
+ # the next cycle's step 4a.
1423
+ ctx = IngestContext(conn=conn, batch=batch,
1424
+ config=(_load_config_once() if batch else None))
1425
+ conn.execute("BEGIN IMMEDIATE")
1426
+ try:
1427
+ # 4a. Replay journal evt lines (a prior cycle's emission that landed past
1428
+ # its own HW, or a crashed cycle's orphans). Apply-only, sorted by fold
1429
+ # order so a referenced family (snapshots, resets, blocks) resolves
1430
+ # before a referencing one (milestones); NO ctx, so replay is
1431
+ # structurally unable to fire an alert (§5.2 step 4a).
1432
+ for evt in sorted(journal_evts, key=_fold_order):
1433
+ _apply_evt(conn, evt)
1434
+ # 4b. Per-record sequential pipeline over obs/op in canonical order —
1435
+ # sequential is REQUIRED (reset/credit detection precedes the same
1436
+ # record's snapshot-accept; a reset-spanning batch needs prior records'
1437
+ # effects already applied). Hooks emit Model-A evts and push alert
1438
+ # payloads to ctx.pending_alerts.
1439
+ pipeline_changes_before = conn.total_changes
1440
+ for rec in batch:
1441
+ for hook in PIPELINE:
1442
+ hook(ctx, rec)
1443
+ # 4b'. Design C (§5.3): run the live-only budget-config reconcile INSIDE
1444
+ # the txn, after the pipeline and BEFORE harvest, so any newly-latched
1445
+ # crossing row is journaled by the budget harvest below. No op line is
1446
+ # journaled for it — it is never seen at rebuild.
1447
+ if reconcile_config is not None:
1448
+ _run_config_reconcile(ctx, reconcile_config)
1449
+ # 4b''. Codex leg (Task 7): the quota projection re-materializer +
1450
+ # on-demand codex budget/projected alert firings, run on ctx.conn inside
1451
+ # the txn, AFTER the pipeline and BEFORE harvest so any newly-latched
1452
+ # budget/projected crossing (a natural-keyed harvest family) is journaled
1453
+ # below. The quota projection tables + arming are written by the closure
1454
+ # itself (arming via its own `quota_alert_arming` evt). Alerts land in
1455
+ # ctx.pending_alerts for the post-commit dispatch. A `_before_stats_commit`
1456
+ # hook (used by the reconcile's crash-consistency callers) fires at the
1457
+ # end of the closure — inside this txn, before COMMIT — so a raise rolls
1458
+ # the whole cycle back (invariant ii).
1459
+ if codex_apply is not None:
1460
+ codex_apply(ctx)
1461
+ # 4c. Journal + stamp the natural-keyed rows the pipeline inserted.
1462
+ # Early-out (Task 6 gate P2): the ONLY source of `journal_id IS NULL`
1463
+ # rows is a Task-5 chokepoint called from a step-4b pipeline hook —
1464
+ # step-4a replay and step-4b Model-A emit both set journal_id. So when
1465
+ # the pipeline wrote nothing (empty batch, or an all-replay cycle, or a
1466
+ # hook that short-circuited before any write), the 8 harvest scans have
1467
+ # nothing to find; skip them. When it DID write, we harvest (the
1468
+ # per-table partial `WHERE journal_id IS NULL` index keeps each scan
1469
+ # O(this-cycle inserts) even on the accept-only common tick).
1470
+ if conn.total_changes != pipeline_changes_before:
1471
+ _harvest(ctx)
1472
+ # 4d. Advance the cursor (to HW, or to the cache-leg prefix boundary).
1473
+ # `cursor_target is None` ONLY on a reconcile-only cycle over a still-
1474
+ # empty journal (§5.2 above): there are no consumed lines to advance
1475
+ # past, and the harvest's budget evts land in the freshly-created first
1476
+ # segment past the (absent) HW — replayed idempotently next cycle. So do
1477
+ # not touch the cursor here.
1478
+ if cursor_target is not None:
1479
+ _write_cursor(conn, cursor_target[0], cursor_target[1])
1480
+ conn.commit()
1481
+ except BaseException:
1482
+ try:
1483
+ conn.rollback()
1484
+ except Exception:
1485
+ pass
1486
+ raise
1487
+
1488
+ # `post_commit` (the reconcile's `_after_stats_commit` seam) fires AFTER the
1489
+ # commit and BEFORE alert dispatch — the committed state stands, and a raise
1490
+ # propagates (authoritative re-raises), skipping dispatch exactly as the
1491
+ # legacy reconcile's after-commit-then-cert-then-dispatch order did.
1492
+ if post_commit is not None:
1493
+ post_commit()
1494
+
1495
+ # Step 6: dispatch alerts post-commit, from the step-4b sink ONLY (never from
1496
+ # step-4a replay). A crash between 5 and 6 loses at most one dispatch — the
1497
+ # set-then-dispatch trade (§5.2 step 6; docs/alerts-gotchas.md).
1498
+ alerts = list(ctx.pending_alerts)
1499
+ if alerts:
1500
+ (ALERT_DISPATCHER or _dispatch_pending_alerts)(alerts)
1501
+
1502
+ return IngestResult(ran=True, consumed=len(records), malformed=malformed,
1503
+ events_emitted=ctx.events_emitted, alerts=alerts)
1504
+
1505
+
1506
+ def run_stats_ingest(*, mode: str = "opportunistic", timeout_s: float = 10.0,
1507
+ conn: sqlite3.Connection | None = None,
1508
+ reconcile_config=None, codex_apply=None,
1509
+ post_commit=None) -> IngestResult:
1510
+ """Run one ingest cycle as the single-flight stats.db writer (spec §5.1/§5.2).
1511
+
1512
+ `mode="opportunistic"` takes the ingest lock non-blocking (busy → `ran=False`;
1513
+ the current holder consumes the lines). `mode="authoritative"` waits up to
1514
+ `timeout_s` for the lock so a caller observes its own appended line
1515
+ synchronously. Pass `conn` to run the cycle on an existing stats.db
1516
+ connection; otherwise a fresh `open_db()` connection is opened and closed.
1517
+
1518
+ `reconcile_config` (Design C, §5.3): `{"budget": <validated_budget>,
1519
+ "touched_projects": set | None, "axes": set | None}` to reconcile LIVE inside
1520
+ this cycle (never journaled as an op — the latched crossings ride the budget
1521
+ harvest); `touched_projects` scopes the per-project reconcile (6c widening),
1522
+ `axes` ⊆ {"budget","codex_budget","project_budget"} names which axes run
1523
+ (6f writer reroute; `None` runs all). Prefer `reconcile_budget_config(...)`
1524
+ as the opportunistic+wrapped entry point from config-write sites. `None`
1525
+ skips reconcile entirely.
1526
+
1527
+ `codex_apply` (Task 7): a `(ctx) -> None` closure run on `ctx.conn` inside the
1528
+ txn (step 4b'', after the pipeline, before harvest). It is the seam every
1529
+ Codex on-demand stats.db writer routes through — the quota projection
1530
+ re-materializer (`reconcile_codex_quota_projection`) and the on-demand codex
1531
+ budget/projected alert firings — so those writers become single-flight instead
1532
+ of opening their own stats connections. `post_commit` (`() -> None`) fires
1533
+ AFTER the commit, before dispatch (the reconcile's `_after_stats_commit` seam).
1534
+
1535
+ Exception discipline (6b-gate P2): a pipeline-hook/chokepoint exception aborts
1536
+ the cycle — `_run_cycle` rolls back the txn and re-raises, so no cursor
1537
+ advance and no partial commit survive (invariant ii). `run_stats_ingest`
1538
+ catches at this boundary: an OPPORTUNISTIC ingest logs the failure loudly and
1539
+ returns `IngestResult(ran=True, error=<exc>)` so a statusline/hook tick is
1540
+ never broken; an AUTHORITATIVE ingest re-raises so its caller (record-usage,
1541
+ record-credit, sync-week, statusline publication) sees the failure.
1542
+ """
1543
+ lock_fd = _acquire_ingest_lock(mode, timeout_s)
1544
+ if lock_fd is None:
1545
+ return IngestResult(ran=False, consumed=0, malformed=0,
1546
+ events_emitted=0, alerts=[])
1547
+ own_conn = conn is None
1548
+ try:
1549
+ if own_conn:
1550
+ conn = _cctally_core.open_db()
1551
+ try:
1552
+ return _run_cycle(conn, reconcile_config=reconcile_config,
1553
+ codex_apply=codex_apply, post_commit=post_commit)
1554
+ except Exception as exc:
1555
+ if mode == "authoritative":
1556
+ raise
1557
+ print(f"[ingest] opportunistic cycle aborted, cursor unmoved: {exc}",
1558
+ file=sys.stderr)
1559
+ return IngestResult(ran=True, consumed=0, malformed=0,
1560
+ events_emitted=0, alerts=[], error=exc)
1561
+ finally:
1562
+ # Guard on ``conn is not None`` so a failing ``open_db()`` (e.g.
1563
+ # StatsDbCorruptError) surfaces its real error instead of an
1564
+ # AttributeError from ``None.close()`` masking it.
1565
+ if own_conn and conn is not None:
1566
+ conn.close()
1567
+ finally:
1568
+ _release_ingest_lock(lock_fd)
1569
+
1570
+
1571
+ # ==========================================================================
1572
+ # Rebuild — a FRESH stats index from the journal alone (spec §5.4, Task 8 Item 1)
1573
+ # ==========================================================================
1574
+ #
1575
+ # `rebuild_stats_index` makes stats.db DISPOSABLE: it replays the whole journal
1576
+ # in canonical `(segment, offset)` order into a fresh schema'd index (bootstrap
1577
+ # segments before observation segments, per list_segments()), NEVER running the
1578
+ # live PIPELINE — no Model-A emission, no harvest, no alert dispatch, no
1579
+ # `reconcile_config`; every fold is apply-only. The accept/skip DECISIONS were
1580
+ # journaled at capture (`snapshot_accept` evts), so rebuild replays decisions and
1581
+ # NEVER re-derives reset-aware clamps — this is what closes the spanning-reset
1582
+ # non-determinism (spec §5.3, Appendix B I4 P1-3).
1583
+ #
1584
+ # Fold order = `_fold_order` (referenced families before referencing ones) within
1585
+ # the canonical stream, exactly as the live replay path (§5.2 step 4a) does —
1586
+ # generalized to the whole journal. Two projection passes sit between the
1587
+ # structural folds (< milestone order) and the milestone/budget folds (>=
1588
+ # milestone order):
1589
+ # * the OPEN 5h block re-materialization (block-only), so a five_hour_milestone's
1590
+ # `block_id` derived_fk resolves against a real block row (§5.3 / Appendix B
1591
+ # I4 P2-8); and
1592
+ # * the quota `quota_*` projection re-materialization over the (journal-sourced)
1593
+ # cache.db `quota_window_snapshots`, run AFTER the `quota_alert_arming` evts
1594
+ # fold (order 45) so `honor-no-refire` holds.
1595
+ #
1596
+ # Duplicate evt lines with byte-identical payloads are LEGAL (crash-replay appends
1597
+ # duplicates; the 6g/Task-7 purity fixes guarantee byte-identity) — every fold is
1598
+ # idempotent (`INSERT OR IGNORE` on journal_id / natural key, DELETE-by-id,
1599
+ # natural-key UPSERT), so rebuild is idempotent over them.
1600
+ #
1601
+ # The hwm-7d statusline file is NOT re-materialized by a dedicated pass — the SQL
1602
+ # 7d-HWM clamp re-establishes the floor on the next statusline tick, so a stale/
1603
+ # absent hwm-7d file self-heals (the only hwm-7d write during a rebuild is the
1604
+ # incidental one inside a `weekly_credit_effects` credit-effect replay, harmless
1605
+ # last-write-wins). Post-rebuild the cursor equals the journal high-water, so the
1606
+ # next ingest is a no-op over the already-folded lines.
1607
+
1608
+ # op-fold order: floors (`weekly_credit_floor`) fold BEFORE snapshot_accept (10)
1609
+ # and BEFORE any `weekly_credit_effects` (50) that deletes a PRIOR credit's floor.
1610
+ _OP_FOLD_ORDER = 5
1611
+ # fold-order threshold: milestone/budget folds (order 60) run in the second
1612
+ # phase, after the open-block + quota projection re-materialization passes.
1613
+ _REBUILD_MILESTONE_ORDER = 60
1614
+
1615
+ # Journal-covered families counted in the RebuildResult report (+ the two
1616
+ # re-materialized quota projection families, useful for the operator command).
1617
+ _REBUILD_COUNT_TABLES = (
1618
+ "weekly_usage_snapshots", "weekly_cost_snapshots", "week_reset_events",
1619
+ "five_hour_reset_events", "five_hour_blocks", "five_hour_block_models",
1620
+ "five_hour_block_projects", "weekly_credit_floors", "percent_milestones",
1621
+ "five_hour_milestones", "budget_milestones", "projected_milestones",
1622
+ "project_budget_milestones", "quota_alert_arming", "quota_window_blocks",
1623
+ "quota_percent_milestones", "quota_threshold_events",
1624
+ )
1625
+
1626
+
1627
+ @dataclass
1628
+ class RebuildResult:
1629
+ """Outcome of a `rebuild_stats_index` call (spec §5.4)."""
1630
+
1631
+ rows_by_table: dict # journal-covered table -> row count in the rebuild
1632
+ malformed: int # journal lines that failed to decode (spec §4.4)
1633
+ duration_s: float # wall time of the whole rebuild
1634
+ segments_read: int # journal segments folded
1635
+ lines_folded: int # op + evt lines applied (obs are rederive input)
1636
+
1637
+
1638
+ def _remove_db_sidecars(path) -> None:
1639
+ for suffix in ("-wal", "-shm"):
1640
+ try:
1641
+ pathlib.Path(str(path) + suffix).unlink()
1642
+ except OSError:
1643
+ pass
1644
+
1645
+
1646
+ def _remove_db_family(path) -> None:
1647
+ for suffix in ("", "-wal", "-shm"):
1648
+ try:
1649
+ pathlib.Path(str(path) + suffix).unlink()
1650
+ except OSError:
1651
+ pass
1652
+
1653
+
1654
+ def _rebuild_quota_cache_leg(records) -> None:
1655
+ """Re-materialize cache.db `quota_window_snapshots` from the journal's Codex
1656
+ quota obs (spec §5.4). The journal obs are the DURABLE source (§1 latent
1657
+ data-loss hole — the rollout JSONL evaporates); this INSERT OR IGNOREs them
1658
+ on the natural key, mirroring `_quota_applier`. Runs BEFORE any stats
1659
+ transaction, under the `cache.db.codex.lock` provider flock (lock-order law).
1660
+ Best-effort: a missing/busy cache.db is a clean skip (the obs stay durable in
1661
+ the journal; the stats quota projection pass then degrades cleanly)."""
1662
+ quota_obs = [r for r in records if _is_codex_quota_obs(r)]
1663
+ if not quota_obs:
1664
+ return
1665
+ cache_path = _cctally_core.CACHE_DB_PATH
1666
+ if not cache_path.exists():
1667
+ return
1668
+ _cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
1669
+ fd = os.open(str(_cctally_core.CACHE_LOCK_CODEX_PATH),
1670
+ os.O_RDWR | os.O_CREAT, 0o600)
1671
+ try:
1672
+ fcntl.flock(fd, fcntl.LOCK_EX)
1673
+ try:
1674
+ cache = sqlite3.connect(str(cache_path), timeout=15.0)
1675
+ except sqlite3.Error as exc: # pragma: no cover — cache.db unopenable
1676
+ print(f"[rebuild] quota cache leg connect failed: {exc}", file=sys.stderr)
1677
+ return
1678
+ try:
1679
+ cache.execute("PRAGMA busy_timeout=15000")
1680
+ cache.execute("BEGIN IMMEDIATE")
1681
+ for r in quota_obs:
1682
+ p = r["payload"]
1683
+ cache.execute(_QUOTA_SNAPSHOT_INSERT,
1684
+ tuple(p.get(col) for col in _QUOTA_SNAPSHOT_COLS))
1685
+ cache.commit()
1686
+ except sqlite3.Error as exc:
1687
+ try:
1688
+ cache.rollback()
1689
+ except sqlite3.Error:
1690
+ pass
1691
+ print(f"[rebuild] quota cache leg write failed: {exc}", file=sys.stderr)
1692
+ finally:
1693
+ cache.close()
1694
+ finally:
1695
+ try:
1696
+ fcntl.flock(fd, fcntl.LOCK_UN)
1697
+ finally:
1698
+ os.close(fd)
1699
+
1700
+
1701
+ def rebuild_stats_index(*, target_path=None) -> RebuildResult:
1702
+ """Build a FRESH stats index from the journal alone (spec §5.4).
1703
+
1704
+ Replays every segment in canonical `(segment, offset)` order into a fresh
1705
+ schema'd DB at a scratch sibling of the destination, then ATOMICALLY swaps it
1706
+ in — crash-safe (a mid-fold crash leaves only a discardable scratch). Folds
1707
+ are apply-only: the PIPELINE never runs, so no Model-A emission, no harvest,
1708
+ no alerts, no `reconcile_config` (see the module note above). Post-rebuild the
1709
+ cursor equals the journal high-water.
1710
+
1711
+ `target_path` selects the destination (default `DB_PATH`). The caller
1712
+ (auto-heal HEAL_HOOK / `db rebuild`) forensics-quarantines the damaged/old DB
1713
+ FIRST, so the destination is absent at swap time; a `target_path` build (used
1714
+ by determinism tests) writes an independent index without touching `DB_PATH`.
1715
+ """
1716
+ start = time.monotonic()
1717
+ dest = (pathlib.Path(target_path) if target_path is not None
1718
+ else pathlib.Path(_cctally_core.DB_PATH))
1719
+
1720
+ # HW snapshot at the START — lines appended during the rebuild are past HW
1721
+ # and belong to the next ingest cycle (they replay idempotently); mirrors the
1722
+ # live cycle's §5.2.1 HW-prefix rule.
1723
+ hw = journal_high_water()
1724
+ segments = list_segments()
1725
+
1726
+ stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%S_%f")
1727
+ scratch = dest.with_name(dest.name + f".rebuilding-{stamp}")
1728
+ _remove_db_family(scratch)
1729
+
1730
+ # Build a fresh schema'd empty index at the scratch path. `_target_path`
1731
+ # DISARMS open_db's auto-heal (no recursion) and yields the current schema
1732
+ # (migrations stamped, gated backfills no-op on empty, fixups marked).
1733
+ conn = _cctally_core.open_db(_target_path=str(scratch))
1734
+ malformed = 0
1735
+ lines_folded = 0
1736
+ try:
1737
+ decoded: list = []
1738
+ if hw is not None:
1739
+ for _seg, _off, raw in _read_range(None, hw):
1740
+ rec = _lib_journal.decode_line(raw)
1741
+ if rec is None:
1742
+ malformed += 1
1743
+ continue
1744
+ decoded.append(rec)
1745
+
1746
+ # Cache leg BEFORE any stats txn (provider-flock lock-order): journal
1747
+ # Codex quota obs -> cache.db quota_window_snapshots.
1748
+ _rebuild_quota_cache_leg(decoded)
1749
+
1750
+ # One ordered fold stream: op-folds (order 5) + evts, keyed by
1751
+ # (fold_order, canonical seq) so referenced families resolve before
1752
+ # referencing ones and crash-replay duplicates fold idempotently.
1753
+ stream: list = []
1754
+ for seq, rec in enumerate(decoded):
1755
+ t = rec.get("t")
1756
+ kind = (rec.get("payload") or {}).get("kind")
1757
+ if t == "op" and kind in FOLD_APPLIERS:
1758
+ stream.append((_OP_FOLD_ORDER, seq, "op", rec))
1759
+ elif t == "evt":
1760
+ stream.append((_fold_order(rec), seq, "evt", rec))
1761
+ stream.sort(key=lambda x: (x[0], x[1]))
1762
+ structural = [s for s in stream if s[0] < _REBUILD_MILESTONE_ORDER]
1763
+ tail = [s for s in stream if s[0] >= _REBUILD_MILESTONE_ORDER]
1764
+
1765
+ # Phase 1 (txn A) — structural folds: op floors, snapshot_accept, cost
1766
+ # snapshots, resets+suppression, block_close, arming, credit effects.
1767
+ conn.execute("BEGIN IMMEDIATE")
1768
+ try:
1769
+ for _order, _seq, kind, rec in structural:
1770
+ if kind == "op":
1771
+ FOLD_APPLIERS[(rec.get("payload") or {}).get("kind")](conn, rec)
1772
+ else:
1773
+ _apply_evt(conn, rec)
1774
+ lines_folded += 1
1775
+ conn.commit()
1776
+ except BaseException:
1777
+ try:
1778
+ conn.rollback()
1779
+ except Exception:
1780
+ pass
1781
+ raise
1782
+
1783
+ # Phase 2a — OPEN 5h block projection (own txn; block-only). Closed blocks
1784
+ # came from block_close evts; this materializes the never-closed window(s)
1785
+ # so the five_hour_milestone block_id derived_fk resolves. Best-effort
1786
+ # (the open block is a projection, §5.3).
1787
+ try:
1788
+ cctally = sys.modules.get("cctally")
1789
+ bf = getattr(cctally, "_backfill_five_hour_blocks", None)
1790
+ if bf is not None:
1791
+ bf(conn, only_missing=True)
1792
+ except Exception as exc: # pragma: no cover — projection is best-effort
1793
+ print(f"[rebuild] open 5h block re-materialization failed: {exc}",
1794
+ file=sys.stderr)
1795
+
1796
+ # Phase 2b + 3 (txn B) — quota projection re-materialization (after the
1797
+ # order-45 arming folds) + milestone/budget folds + cursor advance.
1798
+ conn.execute("BEGIN IMMEDIATE")
1799
+ try:
1800
+ try:
1801
+ import _cctally_quota as _q
1802
+ _q.rematerialize_quota_projection_for_rebuild(conn)
1803
+ except Exception as exc: # pragma: no cover — projection best-effort
1804
+ print(f"[rebuild] quota projection re-materialization failed: {exc}",
1805
+ file=sys.stderr)
1806
+ for _order, _seq, _kind, rec in tail:
1807
+ _apply_evt(conn, rec)
1808
+ lines_folded += 1
1809
+ if hw is not None:
1810
+ _write_cursor(conn, hw[0], hw[1])
1811
+ conn.commit()
1812
+ except BaseException:
1813
+ try:
1814
+ conn.rollback()
1815
+ except Exception:
1816
+ pass
1817
+ raise
1818
+
1819
+ rows_by_table = {}
1820
+ for tbl in _REBUILD_COUNT_TABLES:
1821
+ try:
1822
+ rows_by_table[tbl] = conn.execute(
1823
+ f"SELECT COUNT(*) FROM {tbl}").fetchone()[0]
1824
+ except sqlite3.Error:
1825
+ rows_by_table[tbl] = 0
1826
+ # Drain the WAL into the main file so the atomic rename carries all data.
1827
+ conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
1828
+ finally:
1829
+ conn.close()
1830
+
1831
+ # Atomic swap: the freshly-built scratch becomes the destination. Its WAL was
1832
+ # drained above; drop the empty sidecars, rename, and clear any stale
1833
+ # destination sidecars (a fresh open recreates its own).
1834
+ _remove_db_sidecars(scratch)
1835
+ os.replace(str(scratch), str(dest))
1836
+ _remove_db_sidecars(dest)
1837
+
1838
+ return RebuildResult(
1839
+ rows_by_table=rows_by_table, malformed=malformed,
1840
+ duration_s=time.monotonic() - start, segments_read=len(segments),
1841
+ lines_folded=lines_folded,
1842
+ )
1843
+
1844
+
1845
+ # ==========================================================================
1846
+ # Cutover — one-time in-place upgrade of a pre-journal install (spec §8, Task 9)
1847
+ # ==========================================================================
1848
+ #
1849
+ # `run_cutover(conn)` exports every journal-covered row of a legacy stats.db
1850
+ # (already at migration head 13, schema applied) into a NEW `bootstrap-<ts>.jsonl`
1851
+ # segment, stamps `journal_id = b:<table>:<rowid>` back onto every exported row,
1852
+ # advances the ingest cursor past the bootstrap, and stamps
1853
+ # `user_version = STATS_INDEX_EPOCH` — the last three ALL inside ONE
1854
+ # `BEGIN IMMEDIATE` transaction, so a crash before the commit rolls the DB back
1855
+ # to the fully-functional legacy shape (PRAGMA user_version is transactional in
1856
+ # WAL). `open_db` calls this once per legacy open (version-only trigger, §8).
1857
+ #
1858
+ # Re-classification (§5.3 / §8): weekly_usage_snapshots rows export as
1859
+ # `snapshot_accept` evts (verbatim decisions — replay never re-derives clamps);
1860
+ # weekly_credit_floors as `op` lines (the ONLY op family); every harvest-family
1861
+ # row as its evt kind with logical-FK refs (`b:<ref_table>:<fk_rowid>`), so
1862
+ # `rebuild_stats_index` over the bootstrap alone reproduces the exported DB.
1863
+ # quota_window_snapshots (cache.db) export as `obs` lines (the §1 latent
1864
+ # data-loss hole: the rollout JSONL evaporates, so the journal becomes their
1865
+ # durable home). OPEN five_hour_blocks are re-materialized projections — they are
1866
+ # NOT exported and keep NULL journal_id, so a later close is still harvested.
1867
+ #
1868
+ # Retry safety (§8 / P1.9): rename-then-stamp with STABLE bootstrap ids. The
1869
+ # segment is built at a `.partial` name, fsync'd, then renamed into place before
1870
+ # the stamping runs; a re-run after any crash re-exports byte-identical lines
1871
+ # (ids are `b:<table>:<rowid>`, independent of the retry's timestamp), so a
1872
+ # duplicate/leftover bootstrap folds idempotently (`INSERT OR IGNORE`). The
1873
+ # cutover does NOT take the ingest lock (open_db reaches it from INSIDE
1874
+ # run_stats_ingest's own ingest-lock hold — re-acquiring would self-deadlock):
1875
+ # single-flight of the STAMP is provided by `BEGIN IMMEDIATE`, and concurrent
1876
+ # cutovers converge by id.
1877
+
1878
+
1879
+ def _cutover_iso(dt_utc: dt.datetime) -> str:
1880
+ return (dt_utc.astimezone(dt.timezone.utc)
1881
+ .isoformat(timespec="seconds").replace("+00:00", "Z"))
1882
+
1883
+
1884
+ def _cutover_ref(ref_table: str, fk_value) -> str:
1885
+ """The logical FK ref for a legacy integer FK: `b:<ref_table>:<rowid>`, or
1886
+ the ``"0"`` no-FK sentinel (spec §4.2). Every exported row's journal_id is
1887
+ `b:<table>:<its rowid>`, so a FK pointing at rowid N is exactly
1888
+ `b:<ref_table>:N` — no lookup needed; an orphan FK folds to a dropped row on
1889
+ rebuild (INSERT OR IGNORE), same as today."""
1890
+ if fk_value in (0, None, "0", ""):
1891
+ return "0"
1892
+ return _lib_journal.bootstrap_id(ref_table, fk_value)
1893
+
1894
+
1895
+ @dataclass(frozen=True)
1896
+ class _CutoverSpec:
1897
+ """How to export one legacy stats table at cutover (spec §8 / §5.3)."""
1898
+
1899
+ table: str
1900
+ kind: str
1901
+ line: str # "evt" | "op" | "obs"
1902
+ at_col: str # column supplying the line `at`
1903
+ fk_refs: dict = field(default_factory=dict) # column -> (ref_table, ref_key)
1904
+ exclude: tuple = () # extra payload columns to drop (derived_fk)
1905
+ closed_only: bool = False # five_hour_blocks: closed rows only
1906
+ children: tuple = () # (payload_key, child_table)
1907
+ # `stamp=False` for a §5.3 "state" family with NO journal_id column
1908
+ # (quota_alert_arming): its fold applier converges by NATURAL-KEY upsert, so
1909
+ # there is nothing to stamp back and it is excluded from the no-NULL-survivors
1910
+ # invariant (§8). When `natural_key_id` is set, the exported evt id is the
1911
+ # natural-key form (`<natural_key_prefix>:<col>:<col>…`) matching the LIVE
1912
+ # emission (so a cutover-exported record and a later live re-emission share
1913
+ # one id) instead of the `b:<table>:<rowid>` bootstrap id.
1914
+ stamp: bool = True
1915
+ natural_key_prefix: str = "" # evt_id kind prefix (e.g. "qaa")
1916
+ natural_key_id: tuple = () # columns forming the natural-key evt id
1917
+
1918
+
1919
+ # Order is cosmetic — the fold sorts by dependency (`_fold_order`); the file
1920
+ # order does not affect correctness. Kept referenced-before-referencing for
1921
+ # readability.
1922
+ _CUTOVER_SPECS = (
1923
+ _CutoverSpec("weekly_credit_floors", "weekly_credit_floor", "op",
1924
+ "applied_at_utc"),
1925
+ _CutoverSpec("weekly_usage_snapshots", "snapshot_accept", "evt",
1926
+ "captured_at_utc"),
1927
+ _CutoverSpec("weekly_cost_snapshots", "weekly_cost_snapshot", "evt",
1928
+ "captured_at_utc"),
1929
+ _CutoverSpec("week_reset_events", "week_reset", "evt", "detected_at_utc"),
1930
+ _CutoverSpec("five_hour_reset_events", "five_hour_credit", "evt",
1931
+ "detected_at_utc"),
1932
+ _CutoverSpec("five_hour_blocks", "five_hour_block_close", "evt",
1933
+ "last_updated_at_utc", closed_only=True,
1934
+ children=_BLOCK_CHILDREN),
1935
+ _CutoverSpec(
1936
+ "percent_milestones", "percent_milestone", "evt", "captured_at_utc",
1937
+ fk_refs={
1938
+ "usage_snapshot_id": ("weekly_usage_snapshots", "usage_snapshot_ref"),
1939
+ "cost_snapshot_id": ("weekly_cost_snapshots", "cost_snapshot_ref"),
1940
+ "reset_event_id": ("week_reset_events", "reset_event_ref"),
1941
+ }),
1942
+ _CutoverSpec(
1943
+ "five_hour_milestones", "five_hour_milestone", "evt", "captured_at_utc",
1944
+ fk_refs={
1945
+ "usage_snapshot_id": ("weekly_usage_snapshots", "usage_snapshot_ref"),
1946
+ "reset_event_id": ("five_hour_reset_events", "reset_event_ref"),
1947
+ },
1948
+ exclude=("block_id",)), # derived_fk: re-derived from five_hour_window_key
1949
+ _CutoverSpec("budget_milestones", "budget", "evt", "crossed_at_utc"),
1950
+ _CutoverSpec("projected_milestones", "projected", "evt", "crossed_at_utc"),
1951
+ _CutoverSpec("project_budget_milestones", "project_budget", "evt",
1952
+ "crossed_at_utc"),
1953
+ # quota_alert_arming (§5.3 "state") — its activation boundary is a
1954
+ # forward-only alert clock (`activated_at_utc`) that must survive rebuild so
1955
+ # the reconcile honors it (no historical re-fires). No journal_id column →
1956
+ # NOT stamped; the fold applier upserts by natural key. The evt id is the
1957
+ # `qaa:` natural-key form (matching the live emission in
1958
+ # `_cctally_quota._codex_leg._emit_arming`), so a cutover-exported arming
1959
+ # record and a later live re-emission for the same identity are ONE record.
1960
+ _CutoverSpec("quota_alert_arming", "quota_alert_arming", "evt",
1961
+ "activated_at_utc", stamp=False, natural_key_prefix="qaa",
1962
+ natural_key_id=("source", "source_root_key", "logical_limit_key",
1963
+ "observed_slot", "window_minutes")),
1964
+ )
1965
+
1966
+ # Journal-covered stats tables whose rows get a `journal_id` stamp at cutover.
1967
+ # five_hour_blocks stamps only its CLOSED rows (open blocks stay NULL — they are
1968
+ # re-materialized projections). `stamp=False` families (quota_alert_arming: no
1969
+ # journal_id column) are excluded — they converge by natural-key upsert.
1970
+ _CUTOVER_STAMP_TABLES = tuple(s.table for s in _CUTOVER_SPECS if s.stamp)
1971
+
1972
+
1973
+ def _export_stats_table(conn, spec) -> list:
1974
+ """Return `[(line_record, rowid), ...]` for every row of `spec.table`
1975
+ (closed rows only when `spec.closed_only`). Bootstrap id = b:<table>:<rowid>;
1976
+ FK columns become logical refs; block children embed under `_models`/
1977
+ `_projects` (spec §8)."""
1978
+ where = " WHERE is_closed = 1" if spec.closed_only else ""
1979
+ rows = conn.execute(f"SELECT * FROM {spec.table}{where}").fetchall()
1980
+ out = []
1981
+ for row in rows:
1982
+ rowid = row["id"]
1983
+ payload = {}
1984
+ for key in row.keys():
1985
+ if key in ("id", "journal_id") or key in spec.fk_refs \
1986
+ or key in spec.exclude:
1987
+ continue
1988
+ payload[key] = row[key]
1989
+ for col, (ref_table, ref_key) in spec.fk_refs.items():
1990
+ payload[ref_key] = _cutover_ref(ref_table, row[col])
1991
+ for payload_key, child_table in spec.children:
1992
+ child_rows = conn.execute(
1993
+ f"SELECT * FROM {child_table} WHERE block_id = ? ORDER BY id",
1994
+ (rowid,)).fetchall()
1995
+ payload[payload_key] = [
1996
+ {k: cr[k] for k in cr.keys() if k not in ("id", "block_id")}
1997
+ for cr in child_rows]
1998
+ if spec.natural_key_id:
1999
+ # §5.3 "state" family: the evt id is the natural-key form (matching
2000
+ # the live emission), NOT the b:<table>:<rowid> bootstrap id.
2001
+ bid = _lib_journal.evt_id(
2002
+ spec.natural_key_prefix, *(row[c] for c in spec.natural_key_id))
2003
+ else:
2004
+ bid = _lib_journal.bootstrap_id(spec.table, rowid)
2005
+ at = row[spec.at_col]
2006
+ if spec.line == "op":
2007
+ rec = _lib_journal.make_op(
2008
+ at=at, src="bootstrap", payload={**payload, "kind": spec.kind})
2009
+ rec["id"] = bid
2010
+ else: # evt
2011
+ rec = _lib_journal.make_evt(kind=spec.kind, id=bid, at=at,
2012
+ payload=payload)
2013
+ out.append((rec, rowid))
2014
+ return out
2015
+
2016
+
2017
+ def _export_quota_obs() -> list:
2018
+ """Export cache.db `quota_window_snapshots` as `obs` lines (spec §8/§5.3).
2019
+ Read-only, best-effort: a missing table / cache.db is a clean empty result
2020
+ (the durable obs simply have nothing to carry). id = b:quota_window_
2021
+ snapshots:<rowid>; NOT stamped in cache.db (that table has no journal_id —
2022
+ it re-materializes from the journal)."""
2023
+ cache_path = _cctally_core.CACHE_DB_PATH
2024
+ if not cache_path.exists():
2025
+ return []
2026
+ try:
2027
+ cache = sqlite3.connect(f"file:{cache_path}?mode=ro", uri=True)
2028
+ except sqlite3.Error:
2029
+ return []
2030
+ try:
2031
+ cols = ", ".join(_QUOTA_SNAPSHOT_COLS)
2032
+ rows = cache.execute(
2033
+ f"SELECT id, {cols} FROM quota_window_snapshots").fetchall()
2034
+ except sqlite3.Error:
2035
+ return []
2036
+ finally:
2037
+ cache.close()
2038
+ out = []
2039
+ for row in rows:
2040
+ rowid = row[0]
2041
+ payload = {"kind": _QUOTA_OBS_KIND}
2042
+ for i, col in enumerate(_QUOTA_SNAPSHOT_COLS):
2043
+ payload[col] = row[1 + i]
2044
+ at = payload.get("captured_at_utc") or _now_iso()
2045
+ rec = _lib_journal.make_obs(at=at, src="bootstrap", provider="codex",
2046
+ payload=payload)
2047
+ rec["id"] = _lib_journal.bootstrap_id("quota_window_snapshots", rowid)
2048
+ out.append(rec)
2049
+ return out
2050
+
2051
+
2052
+ def _cutover_segment_name(now_utc: dt.datetime) -> str:
2053
+ ts = now_utc.astimezone(dt.timezone.utc).strftime("%Y%m%dT%H%M%S_%f")
2054
+ return f"{_lib_journal.BOOTSTRAP_PREFIX}{ts}.jsonl"
2055
+
2056
+
2057
+ def _write_bootstrap_segment(seg_name: str, lines: list) -> int:
2058
+ """Materialize the bootstrap segment atomically (spec §8 rename-then-stamp):
2059
+ encode all lines, write to a `.partial` sibling, fsync file + dir, verify the
2060
+ line count, then `os.replace` into `seg_name`. Returns the final byte size.
2061
+ Every line must fit the torn-tail window (append discipline)."""
2062
+ journal_dir = _cctally_core.JOURNAL_DIR
2063
+ dir_created = not journal_dir.exists()
2064
+ journal_dir.mkdir(parents=True, exist_ok=True)
2065
+ if dir_created:
2066
+ try:
2067
+ os.chmod(journal_dir, 0o700)
2068
+ except OSError:
2069
+ pass
2070
+ encoded = []
2071
+ for rec in lines:
2072
+ data = _lib_journal.encode_line(rec)
2073
+ if len(data) > _MAX_LINE_BYTES:
2074
+ raise JournalError(
2075
+ f"cutover line is {len(data)} bytes, exceeds the "
2076
+ f"{_MAX_LINE_BYTES}-byte limit (spec §4.3)")
2077
+ encoded.append(data)
2078
+ blob = b"".join(encoded)
2079
+ if blob.count(b"\n") != len(lines):
2080
+ raise JournalError(
2081
+ "cutover export line count mismatch (spec §8 verify step)")
2082
+ partial = journal_dir / (seg_name + ".partial")
2083
+ fd = os.open(str(partial), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
2084
+ try:
2085
+ _write_all(fd, blob)
2086
+ os.fsync(fd)
2087
+ finally:
2088
+ os.close(fd)
2089
+ _fsync_dir(journal_dir)
2090
+ seg_path = journal_dir / seg_name
2091
+ os.replace(str(partial), str(seg_path))
2092
+ _fsync_dir(journal_dir)
2093
+ if dir_created:
2094
+ _fsync_dir(journal_dir.parent)
2095
+ return os.path.getsize(seg_path)
2096
+
2097
+
2098
+ def run_cutover(conn, *, now_utc: dt.datetime | None = None) -> "str | None":
2099
+ """Export a legacy stats.db to a bootstrap journal segment and stamp the
2100
+ epoch (spec §8). `conn` is an open stats.db at head 13 with the full schema
2101
+ (journal_id columns + journal_cursor) already applied by `open_db`.
2102
+
2103
+ ONE `BEGIN IMMEDIATE`: read+export every journal-covered row (§5.3
2104
+ re-classification), write the bootstrap segment (rename-then-stamp), stamp
2105
+ `journal_id` on every exported row, advance the cursor past the bootstrap,
2106
+ and stamp `user_version = STATS_INDEX_EPOCH`, then commit. A crash before the
2107
+ commit rolls the whole thing back (the legacy DB stays fully usable); the
2108
+ next open retries idempotently (stable bootstrap ids). A truly empty install
2109
+ (nothing to export) just stamps the epoch — no bootstrap file. Returns the
2110
+ bootstrap segment basename, or None when nothing was exported."""
2111
+ if now_utc is None:
2112
+ now_utc = dt.datetime.now(dt.timezone.utc)
2113
+ epoch = _cctally_core.STATS_INDEX_EPOCH
2114
+
2115
+ conn.execute("BEGIN IMMEDIATE")
2116
+ try:
2117
+ lines = []
2118
+ stamp = [] # (table, rowid)
2119
+ for spec in _CUTOVER_SPECS:
2120
+ for rec, rowid in _export_stats_table(conn, spec):
2121
+ lines.append(rec)
2122
+ if spec.stamp:
2123
+ stamp.append((spec.table, rowid))
2124
+ lines.extend(_export_quota_obs())
2125
+
2126
+ if not lines:
2127
+ # Fresh/empty install — no history to journal; just stamp the epoch.
2128
+ conn.execute(f"PRAGMA user_version = {epoch}")
2129
+ conn.commit()
2130
+ return None
2131
+
2132
+ seg_name = _cutover_segment_name(now_utc)
2133
+ seg_size = _write_bootstrap_segment(seg_name, lines)
2134
+
2135
+ for table, rowid in stamp:
2136
+ conn.execute(
2137
+ f"UPDATE {table} SET journal_id = ? WHERE id = ?",
2138
+ (_lib_journal.bootstrap_id(table, rowid), rowid))
2139
+ _write_cursor(conn, seg_name, seg_size)
2140
+ conn.execute(f"PRAGMA user_version = {epoch}")
2141
+ conn.commit()
2142
+ return seg_name
2143
+ except BaseException:
2144
+ try:
2145
+ conn.rollback()
2146
+ except Exception:
2147
+ pass
2148
+ raise