cctally 1.82.0 → 1.83.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CHANGELOG.md +70 -0
  2. package/README.md +52 -74
  3. package/bin/_cctally_alerts.py +8 -1
  4. package/bin/_cctally_cache.py +963 -149
  5. package/bin/_cctally_config.py +43 -4
  6. package/bin/_cctally_core.py +933 -759
  7. package/bin/_cctally_dashboard.py +157 -47
  8. package/bin/_cctally_dashboard_cache_report.py +13 -6
  9. package/bin/_cctally_dashboard_conversation.py +1 -0
  10. package/bin/_cctally_dashboard_envelope.py +186 -8
  11. package/bin/_cctally_dashboard_share.py +60 -20
  12. package/bin/_cctally_dashboard_sources.py +427 -128
  13. package/bin/_cctally_db.py +605 -128
  14. package/bin/_cctally_doctor.py +413 -28
  15. package/bin/_cctally_five_hour.py +12 -5
  16. package/bin/_cctally_journal.py +2050 -156
  17. package/bin/_cctally_journal_repair.py +519 -0
  18. package/bin/_cctally_milestone_history.py +142 -56
  19. package/bin/_cctally_milestones.py +179 -111
  20. package/bin/_cctally_parser.py +42 -0
  21. package/bin/_cctally_project.py +24 -18
  22. package/bin/_cctally_quota.py +139 -25
  23. package/bin/_cctally_record.py +279 -108
  24. package/bin/_cctally_rederive.py +1052 -0
  25. package/bin/_cctally_reporting.py +58 -53
  26. package/bin/_cctally_setup.py +1 -0
  27. package/bin/_cctally_source_analytics.py +4 -1
  28. package/bin/_cctally_statusline.py +11 -11
  29. package/bin/_cctally_store.py +1039 -31
  30. package/bin/_cctally_sync_week.py +17 -8
  31. package/bin/_cctally_tui.py +421 -54
  32. package/bin/_cctally_update.py +133 -8
  33. package/bin/_cctally_weekrefs.py +14 -0
  34. package/bin/_lib_aggregators.py +10 -6
  35. package/bin/_lib_cache_report.py +101 -9
  36. package/bin/_lib_codex_pools.py +82 -0
  37. package/bin/_lib_conversation_query.py +126 -33
  38. package/bin/_lib_dashboard_sources.py +126 -1
  39. package/bin/_lib_diff_kernel.py +28 -15
  40. package/bin/_lib_doctor.py +342 -4
  41. package/bin/_lib_journal.py +924 -2
  42. package/bin/_lib_jsonl.py +43 -14
  43. package/bin/_lib_pricing.py +140 -21
  44. package/bin/_lib_readme_refresh.py +401 -0
  45. package/bin/_lib_rederive.py +395 -0
  46. package/bin/_lib_share.py +58 -2
  47. package/bin/cctally +56 -8
  48. package/dashboard/static/assets/{index-DJP4gEB7.js → index-3bgCMVHb.js} +52 -52
  49. package/dashboard/static/assets/index-D27EIHEI.css +1 -0
  50. package/dashboard/static/dashboard.html +2 -2
  51. package/package.json +6 -1
  52. package/dashboard/static/assets/index-Dk1nplOz.css +0 -1
@@ -11,6 +11,7 @@ and the only legal monkeypatch target for the 23 promoted globals listed
11
11
  below. See docs/superpowers/specs/2026-05-22-cctally-core-data-globals.md.
12
12
  """
13
13
  from __future__ import annotations
14
+ import contextvars
14
15
  import datetime as dt
15
16
  import math
16
17
  import os
@@ -338,7 +339,17 @@ STATS_WAL_SIZE_LIMIT_BYTES = 16 * 1024 * 1024 # 16777216
338
339
  # epoch-transition coordinator (``_cctally_journal.run_epoch_transition``):
339
340
  # resolve the cutover identity, append the canonical cutover op, then rebuild
340
341
  # account-scoped. See docs/superpowers/specs/2026-07-23-multi-account-design.md §2.
341
- STATS_INDEX_EPOCH = 1001
342
+ # 1001 -> 1002 (#372 Task A): the disposable index gains the effective-event
343
+ # revision summary consumed by correction planning/live replay. Rebuild selects
344
+ # the highest completed revision before folding; the legacy migration registry
345
+ # remains frozen.
346
+ # 1002 -> 1003 (#402 Task A): persist the selector's bounded structural
347
+ # correction-batch violations in the disposable index so shallow Dashboard/TUI
348
+ # Doctor gathers cannot report false health without rescanning the whole journal.
349
+ # 1003 -> 1004 (#410 Task B): pair the public journal cursor with the exact
350
+ # prefix atomically applied to the materialized index. A cursor-only hand edit
351
+ # can no longer skip an already-durable event and make its natural key look new.
352
+ STATS_INDEX_EPOCH = 1004
342
353
  LEGACY_STATS_HEAD = 13
343
354
 
344
355
 
@@ -656,6 +667,93 @@ def ensure_dirs() -> None:
656
667
  eprint(f"[core] could not chmod data dir 0700 ({exc}); continuing")
657
668
 
658
669
 
670
+ # === stats.db maintenance-hold tracking (#386) ======================
671
+ #
672
+ # `flock` conflicts are per open-file-DESCRIPTION and apply WITHIN a process:
673
+ # holding LOCK_EX on one fd and then requesting LOCK_SH on a second fd of the
674
+ # same file blocks forever. That matters because `_cctally_store`'s #386 opener
675
+ # protocol takes `stats.db.maintenance.lock` SHARED around every live stats
676
+ # open, while `run_stats_ingest`'s legacy/fresh branch already holds it
677
+ # EXCLUSIVE when it calls `open_db()` (bin/_cctally_journal.py:2788). Without a
678
+ # re-entrancy signal that is an unconditional self-deadlock on first open of a
679
+ # pre-cutover install.
680
+ #
681
+ # A ContextVar, not a module global, and deliberately so: the suppressor must
682
+ # fire only for the execution context that actually owns the lock. A dashboard
683
+ # thread that does NOT own it and requests SHARED while another thread holds
684
+ # EXCLUSIVE is correctly made to WAIT — that is the protocol working, not a
685
+ # deadlock — and a process-global flag would wrongly wave it straight through
686
+ # into a family being replaced underneath it.
687
+ #
688
+ # Every acquisition of STATS_LOCK_MAINTENANCE_PATH in bin/ that is HELD ACROSS
689
+ # other work pairs with these (a site that takes the flock and releases it
690
+ # before returning does not, and must not — see `stats_open_guarded`):
691
+ # bin/_cctally_journal.py _acquire_maintenance_{shared,exclusive} / _release
692
+ # bin/_cctally_store.py _heal_flock_blocking, reached through
693
+ # _acquire_stats_maintenance_reentrant by the heal
694
+ # hook and the epoch resolver
695
+ # bin/_cctally_db.py cmd_db_rebuild, _acquire_db_admin_writer_flocks
696
+ # (db skip / db unskip), _cmd_db_repair_exclusive,
697
+ # _vacuum_one_db
698
+ # bin/_cctally_rederive.py _rederive_locks
699
+ # Adding another acquisition site without noting it here reintroduces the hang.
700
+ #
701
+ # The opener (`_cctally_store.stats_open_guarded`) takes the lock SHARED and
702
+ # releases it before handing the connection back, so it deliberately does NOT
703
+ # note a hold — but it DOES consult `holds_stats_maintenance()` to skip the
704
+ # acquire entirely when this context already owns the exclusive side.
705
+
706
+ _STATS_MAINTENANCE_HELD = contextvars.ContextVar(
707
+ "cctally_stats_maintenance_held", default=0
708
+ )
709
+
710
+
711
+ def holds_stats_maintenance() -> bool:
712
+ """True when THIS execution context already holds stats.db.maintenance.lock."""
713
+ return _STATS_MAINTENANCE_HELD.get() > 0
714
+
715
+
716
+ # === stats.db sanctioned-write scope (#386) =========================
717
+ #
718
+ # The state behind `_cctally_store.stats_write_scope` / `in_stats_write_scope`
719
+ # / `holds_ingest_lock`. It lives HERE, beside the maintenance tracker, rather
720
+ # than in `_cctally_store` for one concrete reason: `tests/conftest.py`'s
721
+ # `load_script()` drops every cached `_cctally_*` sibling from `sys.modules`
722
+ # (deliberately — see its docstring) but KEEPS `_cctally_core`. A ContextVar
723
+ # owned by `_cctally_store` would therefore be silently replaced by a fresh,
724
+ # empty one halfway through a test, so a scope entered before the reload would
725
+ # stop counting and an authorized write would be denied. `_cctally_core` is the
726
+ # kernel and is never reloaded, so the sanction survives.
727
+ #
728
+ # ContextVars, NOT module globals: the dashboard is threaded and a global would
729
+ # let one sanctioned thread authorize another (spec section 6.1).
730
+
731
+ _STATS_WRITE_SCOPE = contextvars.ContextVar(
732
+ "cctally_stats_write_scope", default=0
733
+ )
734
+ _STATS_INGEST_LOCK_HELD = contextvars.ContextVar(
735
+ "cctally_stats_ingest_held", default=0
736
+ )
737
+ _STATS_INTERRUPTED_RECOVERY_SUPPRESSED = contextvars.ContextVar(
738
+ "cctally_stats_interrupted_recovery_suppressed", default=0
739
+ )
740
+
741
+
742
+ def note_stats_maintenance_acquired() -> None:
743
+ """Record that this context now holds stats.db.maintenance.lock."""
744
+ _STATS_MAINTENANCE_HELD.set(_STATS_MAINTENANCE_HELD.get() + 1)
745
+
746
+
747
+ def note_stats_maintenance_released() -> None:
748
+ """Record that this context released stats.db.maintenance.lock.
749
+
750
+ Clamped at zero rather than asserting: an unbalanced release is a bug, but
751
+ turning it into an exception inside a ``finally`` would mask the original
752
+ failure that got us there.
753
+ """
754
+ _STATS_MAINTENANCE_HELD.set(max(0, _STATS_MAINTENANCE_HELD.get() - 1))
755
+
756
+
659
757
  # === Alerts validation cluster ======================================
660
758
 
661
759
 
@@ -1289,6 +1387,9 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
1289
1387
  _STATS_MIGRATIONS = c._STATS_MIGRATIONS
1290
1388
  _log_migration_error = c._log_migration_error
1291
1389
  _clear_migration_error_log_entries = c._clear_migration_error_log_entries
1390
+ _reconcile_durable_applied_migration_errors = (
1391
+ c._reconcile_durable_applied_migration_errors
1392
+ )
1292
1393
  # Unified opener policy (spec §6.1). Call-time import so the shared PRAGMA
1293
1394
  # policy applies without a module-load cycle (_cctally_store imports this
1294
1395
  # module). Routed through importlib.import_module rather than a bare
@@ -1303,16 +1404,16 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
1303
1404
  import importlib
1304
1405
  _cctally_store = importlib.import_module("_cctally_store")
1305
1406
 
1306
- repair_marker = db_path.with_name("stats.db.repairing")
1307
- if repair_marker.exists():
1308
- raise c.StatsDbMaintenanceError()
1309
1407
  ensure_dirs()
1310
- if repair_marker.exists():
1311
- raise c.StatsDbMaintenanceError()
1312
- conn = sqlite3.connect(db_path)
1313
- if repair_marker.exists():
1314
- conn.close()
1315
- raise c.StatsDbMaintenanceError()
1408
+ # #386: the opener half of the physical-replacement protocol. This replaces
1409
+ # three bare `repair_marker.exists()` checks around an unguarded connect —
1410
+ # which observed no quarantine-pending record and held no maintenance lock,
1411
+ # so a destructive maintenance path could publish its record, scan for
1412
+ # handles, and still be raced by an opener arriving before the first rename.
1413
+ # The guarded opener holds maintenance-SHARED across the marker/pending
1414
+ # checks AND the connect. Scratch (`_target_path`) opens keep the old
1415
+ # marker-only behaviour; see `_cctally_store.stats_open_guarded`.
1416
+ conn = _cctally_store.stats_open_guarded(db_path)
1316
1417
  conn.row_factory = sqlite3.Row
1317
1418
  # #279 S1 F4: probe connect + initial PRAGMAs so a corrupt stats.db (the
1318
1419
  # non-re-derivable DB) surfaces as a one-line diagnosis + staged exit 3 instead of
@@ -1342,17 +1443,21 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
1342
1443
  # POSITIVELY classified corruption of the REAL stats index (never a
1343
1444
  # ``_target_path`` rebuild build — that path is disarmed to avoid
1344
1445
  # recursion), hand off to the store's HEAL_HOOK: it re-checks under the
1345
- # maintenance lock, writes the forensics bundle FIRST, quarantines the
1346
- # damaged family into an incident dir, rebuilds a fresh index from the
1347
- # journal, and returns True. A single retry of the connect+probe then
1348
- # runs against the fresh index; a second failure surfaces loudly. HEAL
1349
- # returns False (declined) for a non-corruption ``DatabaseError`` (BUSY,
1350
- # disk-full, permissions), the dev-checkout-on-prod guard, or when a
1351
- # concurrent healer already fixed it all of which fall through to the
1352
- # original guided StatsDbCorruptError so the manual path still applies.
1446
+ # maintenance lock, writes the forensics bundle FIRST, builds and
1447
+ # validates a fresh index while the damaged family remains in place,
1448
+ # then preserves that family and atomically publishes the replacement.
1449
+ # A single retry of the connect+probe then runs against the fresh index;
1450
+ # a second failure surfaces loudly. HEAL returns False (declined) for a
1451
+ # non-corruption ``DatabaseError`` (BUSY, disk-full, permissions), the
1452
+ # dev-checkout-on-prod guard, or when a concurrent healer already fixed
1453
+ # it all of which fall through to the original guided
1454
+ # StatsDbCorruptError so the manual path still applies.
1353
1455
  heal = getattr(_cctally_store, "HEAL_HOOK", None)
1354
1456
  if _target_path is None and heal is not None and heal("stats", exc):
1355
- conn = sqlite3.connect(db_path)
1457
+ # #386: the post-heal retry is an opener too. The heal released the
1458
+ # maintenance lock before returning, so another maintenance path can
1459
+ # legitimately own the family by now.
1460
+ conn = _cctally_store.stats_open_guarded(db_path)
1356
1461
  conn.row_factory = sqlite3.Row
1357
1462
  try:
1358
1463
  _cctally_store.apply_policy(conn, "stats")
@@ -1390,6 +1495,12 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
1390
1495
  # STEADY STATE — zero schema work (spec §6.2/§7.1). Applies to a
1391
1496
  # ``_target_path`` open too: a rebuilt/scratch index is stamped at the
1392
1497
  # epoch, so opening it must be a pure connect + PRAGMA + version read.
1498
+ # Only the live path owns the live migration-error sentinel: a
1499
+ # scratch rebuild must not clear it before validated publication.
1500
+ if _target_path is None:
1501
+ _reconcile_durable_applied_migration_errors(
1502
+ conn, _STATS_MIGRATIONS, "stats.db"
1503
+ )
1393
1504
  return conn
1394
1505
  if _target_path is None:
1395
1506
  if _uv > LEGACY_STATS_HEAD:
@@ -1421,778 +1532,841 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
1421
1532
  # surrounding schema apply (CREATE TABLE IF NOT EXISTS / add_column_if_missing
1422
1533
  # / dispatcher) still runs every open — Task 9 folds THAT under the
1423
1534
  # STATS_INDEX_EPOCH gate; this task only removes the recurring backfill cost.
1424
- _fixups_current = _cctally_store.stats_open_fixups_current(conn)
1425
- conn.execute(
1426
- """
1427
- CREATE TABLE IF NOT EXISTS weekly_usage_snapshots (
1428
- id INTEGER PRIMARY KEY AUTOINCREMENT,
1429
- captured_at_utc TEXT NOT NULL,
1430
- week_start_date TEXT NOT NULL,
1431
- week_end_date TEXT NOT NULL,
1432
- week_start_at TEXT,
1433
- week_end_at TEXT,
1434
- weekly_percent REAL NOT NULL,
1435
- page_url TEXT,
1436
- source TEXT NOT NULL DEFAULT 'userscript',
1437
- payload_json TEXT NOT NULL,
1438
- account_key TEXT NOT NULL DEFAULT 'unattributed'
1535
+ # ── #386: the OPEN-TIME MUTATION REGIME (spec §3.1 second clause) ──
1536
+ # Everything below mutates stats.db: the full schema DDL, the quota
1537
+ # projection schema, the migration dispatcher, two backfills, the fixups
1538
+ # marker and the in-place cutover. Before #386 it ran under NO lock,
1539
+ # reachable from any of the 57 production `open_db` call sites — so two
1540
+ # commands racing a first open, an upgrade or a cutover both ran DDL on
1541
+ # the same file. The guard takes `stats.db.maintenance.lock` EXCLUSIVE
1542
+ # (re-entrant: the common case arrives from a caller that already holds
1543
+ # it) and enters the sanctioned write scope the authorizer checks.
1544
+ #
1545
+ # It is BELOW the epoch gate deliberately: the steady-state open returns
1546
+ # at `_uv == STATS_INDEX_EPOCH` above and never reaches here, so the hot
1547
+ # path takes no exclusive lock at all.
1548
+ with _cctally_store.stats_open_time_guard(live=_target_path is None):
1549
+ _fixups_current = _cctally_store.stats_open_fixups_current(conn)
1550
+ conn.execute(
1551
+ """
1552
+ CREATE TABLE IF NOT EXISTS weekly_usage_snapshots (
1553
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
1554
+ captured_at_utc TEXT NOT NULL,
1555
+ week_start_date TEXT NOT NULL,
1556
+ week_end_date TEXT NOT NULL,
1557
+ week_start_at TEXT,
1558
+ week_end_at TEXT,
1559
+ weekly_percent REAL NOT NULL,
1560
+ page_url TEXT,
1561
+ source TEXT NOT NULL DEFAULT 'userscript',
1562
+ payload_json TEXT NOT NULL,
1563
+ account_key TEXT NOT NULL DEFAULT 'unattributed'
1564
+ )
1565
+ """
1439
1566
  )
1440
- """
1441
- )
1442
- conn.execute(
1443
- """
1444
- CREATE INDEX IF NOT EXISTS idx_usage_week_time
1445
- ON weekly_usage_snapshots(week_start_date, captured_at_utc DESC, id DESC)
1446
- """
1447
- )
1448
- conn.execute(
1449
- """
1450
- CREATE TABLE IF NOT EXISTS weekly_cost_snapshots (
1451
- id INTEGER PRIMARY KEY AUTOINCREMENT,
1452
- captured_at_utc TEXT NOT NULL,
1453
- week_start_date TEXT NOT NULL,
1454
- week_end_date TEXT NOT NULL,
1455
- week_start_at TEXT,
1456
- week_end_at TEXT,
1457
- range_start_iso TEXT,
1458
- range_end_iso TEXT,
1459
- cost_usd REAL NOT NULL,
1460
- source TEXT NOT NULL DEFAULT 'cctally-range-cost',
1461
- mode TEXT NOT NULL DEFAULT 'auto',
1462
- project TEXT,
1463
- account_key TEXT NOT NULL DEFAULT 'unattributed'
1567
+ conn.execute(
1568
+ """
1569
+ CREATE INDEX IF NOT EXISTS idx_usage_week_time
1570
+ ON weekly_usage_snapshots(week_start_date, captured_at_utc DESC, id DESC)
1571
+ """
1464
1572
  )
1465
- """
1466
- )
1467
- conn.execute(
1468
- """
1469
- CREATE INDEX IF NOT EXISTS idx_cost_week_time
1470
- ON weekly_cost_snapshots(week_start_date, captured_at_utc DESC, id DESC)
1471
- """
1472
- )
1473
-
1474
- add_column_if_missing(conn, "weekly_usage_snapshots", "week_start_at", "TEXT")
1475
- add_column_if_missing(conn, "weekly_usage_snapshots", "week_end_at", "TEXT")
1476
- # account_key (#341): the account dimension rides the STATS_INDEX_EPOCH bump
1477
- # (a fresh rebuild carries it via the CREATE TABLE above); this backstop keeps
1478
- # an already-1001 DB that predates the column consistent. DEFAULT
1479
- # 'unattributed' — every production writer passes the key explicitly (rev 4.1
1480
- # defensive-backstop rule), enforced by the structural writer-audit test.
1481
- add_column_if_missing(
1482
- conn, "weekly_usage_snapshots", "account_key",
1483
- "TEXT NOT NULL DEFAULT 'unattributed'")
1484
- add_column_if_missing(conn, "weekly_usage_snapshots", "five_hour_percent", "REAL")
1485
- add_column_if_missing(conn, "weekly_usage_snapshots", "five_hour_resets_at", "TEXT")
1486
- # five_hour_window_key — canonical (10-min-floored epoch) key for
1487
- # jitter-tolerant equality. Anthropic's status-line API jitters
1488
- # rate_limits.5h.resets_at by ~seconds within the same physical 5h
1489
- # window; joining on the raw ISO string treats each jittered fetch as
1490
- # a new window, escaping the monotonic clamp at cmd_record_usage.
1491
- # Backfill is RESUMABLE: Python's sqlite3 auto-commits DDL,
1492
- # so a process killed mid-loop would leave the column added with NULL
1493
- # keys for unprocessed rows. The gating below detects that partial
1494
- # state on the next open_db() call (`five_hour_resets_at IS NOT NULL
1495
- # AND five_hour_window_key IS NULL`) and completes the backfill, so
1496
- # the original Bug B can't silently re-emerge for half-migrated rows.
1497
- needs_5h_key_backfill = add_column_if_missing(
1498
- conn, "weekly_usage_snapshots", "five_hour_window_key", "INTEGER"
1499
- )
1500
- # §6.2 backfill gate (Task 8): the resumable-partial probe + the backfill
1501
- # loop are open-time backfill work — skipped once the fixups marker is
1502
- # stamped. (The `add_column_if_missing` above is schema apply, Task 9's.)
1503
- if not _fixups_current:
1504
- if not needs_5h_key_backfill and conn.execute(
1505
- "SELECT 1 FROM weekly_usage_snapshots "
1506
- "WHERE five_hour_resets_at IS NOT NULL "
1507
- " AND five_hour_window_key IS NULL "
1508
- "LIMIT 1"
1509
- ).fetchone() is not None:
1510
- needs_5h_key_backfill = True
1511
- else:
1512
- needs_5h_key_backfill = False
1513
-
1514
- if needs_5h_key_backfill:
1515
- backfill_rows = conn.execute(
1516
- "SELECT id, five_hour_resets_at FROM weekly_usage_snapshots "
1517
- "WHERE five_hour_resets_at IS NOT NULL "
1518
- " AND five_hour_window_key IS NULL"
1519
- ).fetchall()
1520
- for row in backfill_rows:
1521
- try:
1522
- iso = row[1]
1523
- d = parse_iso_datetime(iso, "five_hour_resets_at backfill")
1524
- epoch = int(d.timestamp())
1525
- key = _canonical_5h_window_key(epoch)
1526
- conn.execute(
1527
- "UPDATE weekly_usage_snapshots "
1528
- "SET five_hour_window_key = ? WHERE id = ?",
1529
- (key, row[0]),
1530
- )
1531
- except (ValueError, TypeError) as exc:
1532
- eprint(f"[migration] skipped row {row[0]}: {exc}")
1533
1573
  conn.execute(
1534
- "CREATE INDEX IF NOT EXISTS idx_weekly_usage_snapshots_5h_window_key "
1535
- "ON weekly_usage_snapshots(five_hour_window_key)"
1574
+ """
1575
+ CREATE TABLE IF NOT EXISTS weekly_cost_snapshots (
1576
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
1577
+ captured_at_utc TEXT NOT NULL,
1578
+ week_start_date TEXT NOT NULL,
1579
+ week_end_date TEXT NOT NULL,
1580
+ week_start_at TEXT,
1581
+ week_end_at TEXT,
1582
+ range_start_iso TEXT,
1583
+ range_end_iso TEXT,
1584
+ cost_usd REAL NOT NULL,
1585
+ source TEXT NOT NULL DEFAULT 'cctally-range-cost',
1586
+ mode TEXT NOT NULL DEFAULT 'auto',
1587
+ project TEXT,
1588
+ account_key TEXT NOT NULL DEFAULT 'unattributed'
1589
+ )
1590
+ """
1591
+ )
1592
+ conn.execute(
1593
+ """
1594
+ CREATE INDEX IF NOT EXISTS idx_cost_week_time
1595
+ ON weekly_cost_snapshots(week_start_date, captured_at_utc DESC, id DESC)
1596
+ """
1536
1597
  )
1537
- conn.commit()
1538
1598
 
1539
- add_column_if_missing(conn, "weekly_cost_snapshots", "week_start_at", "TEXT")
1540
- add_column_if_missing(conn, "weekly_cost_snapshots", "week_end_at", "TEXT")
1541
- add_column_if_missing(conn, "weekly_cost_snapshots", "range_start_iso", "TEXT")
1542
- add_column_if_missing(conn, "weekly_cost_snapshots", "range_end_iso", "TEXT")
1543
- add_column_if_missing(
1544
- conn, "weekly_cost_snapshots", "account_key",
1545
- "TEXT NOT NULL DEFAULT 'unattributed'")
1599
+ add_column_if_missing(conn, "weekly_usage_snapshots", "week_start_at", "TEXT")
1600
+ add_column_if_missing(conn, "weekly_usage_snapshots", "week_end_at", "TEXT")
1601
+ # account_key (#341): the account dimension rides the STATS_INDEX_EPOCH bump
1602
+ # (a fresh rebuild carries it via the CREATE TABLE above); this backstop keeps
1603
+ # an already-1001 DB that predates the column consistent. DEFAULT
1604
+ # 'unattributed' — every production writer passes the key explicitly (rev 4.1
1605
+ # defensive-backstop rule), enforced by the structural writer-audit test.
1606
+ add_column_if_missing(
1607
+ conn, "weekly_usage_snapshots", "account_key",
1608
+ "TEXT NOT NULL DEFAULT 'unattributed'")
1609
+ add_column_if_missing(conn, "weekly_usage_snapshots", "five_hour_percent", "REAL")
1610
+ add_column_if_missing(conn, "weekly_usage_snapshots", "five_hour_resets_at", "TEXT")
1611
+ # five_hour_window_key — canonical (10-min-floored epoch) key for
1612
+ # jitter-tolerant equality. Anthropic's status-line API jitters
1613
+ # rate_limits.5h.resets_at by ~seconds within the same physical 5h
1614
+ # window; joining on the raw ISO string treats each jittered fetch as
1615
+ # a new window, escaping the monotonic clamp at cmd_record_usage.
1616
+ # Backfill is RESUMABLE: Python's sqlite3 auto-commits DDL,
1617
+ # so a process killed mid-loop would leave the column added with NULL
1618
+ # keys for unprocessed rows. The gating below detects that partial
1619
+ # state on the next open_db() call (`five_hour_resets_at IS NOT NULL
1620
+ # AND five_hour_window_key IS NULL`) and completes the backfill, so
1621
+ # the original Bug B can't silently re-emerge for half-migrated rows.
1622
+ needs_5h_key_backfill = add_column_if_missing(
1623
+ conn, "weekly_usage_snapshots", "five_hour_window_key", "INTEGER"
1624
+ )
1625
+ # §6.2 backfill gate (Task 8): the resumable-partial probe + the backfill
1626
+ # loop are open-time backfill work — skipped once the fixups marker is
1627
+ # stamped. (The `add_column_if_missing` above is schema apply, Task 9's.)
1628
+ if not _fixups_current:
1629
+ if not needs_5h_key_backfill and conn.execute(
1630
+ "SELECT 1 FROM weekly_usage_snapshots "
1631
+ "WHERE five_hour_resets_at IS NOT NULL "
1632
+ " AND five_hour_window_key IS NULL "
1633
+ "LIMIT 1"
1634
+ ).fetchone() is not None:
1635
+ needs_5h_key_backfill = True
1636
+ else:
1637
+ needs_5h_key_backfill = False
1638
+
1639
+ if needs_5h_key_backfill:
1640
+ backfill_rows = conn.execute(
1641
+ "SELECT id, five_hour_resets_at FROM weekly_usage_snapshots "
1642
+ "WHERE five_hour_resets_at IS NOT NULL "
1643
+ " AND five_hour_window_key IS NULL"
1644
+ ).fetchall()
1645
+ for row in backfill_rows:
1646
+ try:
1647
+ iso = row[1]
1648
+ d = parse_iso_datetime(iso, "five_hour_resets_at backfill")
1649
+ epoch = int(d.timestamp())
1650
+ key = _canonical_5h_window_key(epoch)
1651
+ conn.execute(
1652
+ "UPDATE weekly_usage_snapshots "
1653
+ "SET five_hour_window_key = ? WHERE id = ?",
1654
+ (key, row[0]),
1655
+ )
1656
+ except (ValueError, TypeError) as exc:
1657
+ eprint(f"[migration] skipped row {row[0]}: {exc}")
1658
+ conn.execute(
1659
+ "CREATE INDEX IF NOT EXISTS idx_weekly_usage_snapshots_5h_window_key "
1660
+ "ON weekly_usage_snapshots(five_hour_window_key)"
1661
+ )
1662
+ conn.commit()
1546
1663
 
1547
- conn.execute(
1548
- """
1549
- CREATE INDEX IF NOT EXISTS idx_usage_week_start_at_time
1550
- ON weekly_usage_snapshots(week_start_at, captured_at_utc DESC, id DESC)
1551
- """
1552
- )
1553
- conn.execute(
1554
- """
1555
- CREATE INDEX IF NOT EXISTS idx_cost_week_start_at_time
1556
- ON weekly_cost_snapshots(week_start_at, captured_at_utc DESC, id DESC)
1557
- """
1558
- )
1664
+ add_column_if_missing(conn, "weekly_cost_snapshots", "week_start_at", "TEXT")
1665
+ add_column_if_missing(conn, "weekly_cost_snapshots", "week_end_at", "TEXT")
1666
+ add_column_if_missing(conn, "weekly_cost_snapshots", "range_start_iso", "TEXT")
1667
+ add_column_if_missing(conn, "weekly_cost_snapshots", "range_end_iso", "TEXT")
1668
+ add_column_if_missing(
1669
+ conn, "weekly_cost_snapshots", "account_key",
1670
+ "TEXT NOT NULL DEFAULT 'unattributed'")
1559
1671
 
1560
- conn.execute(
1561
- """
1562
- CREATE TABLE IF NOT EXISTS percent_milestones (
1563
- id INTEGER PRIMARY KEY AUTOINCREMENT,
1564
- captured_at_utc TEXT NOT NULL,
1565
- week_start_date TEXT NOT NULL,
1566
- week_end_date TEXT NOT NULL,
1567
- week_start_at TEXT,
1568
- week_end_at TEXT,
1569
- percent_threshold INTEGER NOT NULL,
1570
- cumulative_cost_usd REAL NOT NULL,
1571
- marginal_cost_usd REAL,
1572
- usage_snapshot_id INTEGER NOT NULL,
1573
- cost_snapshot_id INTEGER NOT NULL,
1574
- reset_event_id INTEGER NOT NULL DEFAULT 0,
1575
- account_key TEXT NOT NULL DEFAULT 'unattributed',
1576
- UNIQUE(account_key, week_start_date, percent_threshold, reset_event_id)
1672
+ conn.execute(
1673
+ """
1674
+ CREATE INDEX IF NOT EXISTS idx_usage_week_start_at_time
1675
+ ON weekly_usage_snapshots(week_start_at, captured_at_utc DESC, id DESC)
1676
+ """
1677
+ )
1678
+ conn.execute(
1679
+ """
1680
+ CREATE INDEX IF NOT EXISTS idx_cost_week_start_at_time
1681
+ ON weekly_cost_snapshots(week_start_at, captured_at_utc DESC, id DESC)
1682
+ """
1577
1683
  )
1578
- """
1579
- )
1580
1684
 
1581
- add_column_if_missing(conn, "percent_milestones", "five_hour_percent_at_crossing", "REAL")
1582
- add_column_if_missing(
1583
- conn, "percent_milestones", "account_key",
1584
- "TEXT NOT NULL DEFAULT 'unattributed'")
1585
- # reset_event_id: segment column added by migration 005. Fresh-install
1586
- # DBs get it via the live CREATE TABLE above + the dispatcher
1587
- # fast-stamps the migration. Existing pre-005 DBs trip the migration's
1588
- # rename-recreate-copy idiom (handler in _cctally_db.py); the handler's
1589
- # fast-path probe stamps the marker when the column is already present
1590
- # (covers the corner case where a partially-upgraded DB has the column
1591
- # but not the new UNIQUE — re-run is safe).
1592
-
1593
- # alerted_at: populated by the alert-dispatch path when a milestone-INSERT
1594
- # row's threshold matches the user's configured alerts.weekly_thresholds /
1595
- # alerts.five_hour_thresholds (and alerts.enabled is true). NULL means
1596
- # "alerts were disabled at the moment of crossing OR the threshold wasn't
1597
- # in the configured list" — never "alert delivery failed" (dispatch is
1598
- # best-effort and write-once forward-only). The matching ALTER for
1599
- # `five_hour_milestones` lives right after that table's CREATE block
1600
- # below, since the table doesn't exist yet at this point in `open_db()`.
1601
- add_column_if_missing(conn, "percent_milestones", "alerted_at", "TEXT")
1602
-
1603
- # Mid-week reset events: when Anthropic advances `rate_limits.seven_day.
1604
- # resets_at` before the previously-declared reset actually fires (i.e.,
1605
- # gives the user a fresh weekly window before the old one naturally
1606
- # expired), we record one row here so display + cost layers can treat
1607
- # the effective reset moment as the old week's end AND the new week's
1608
- # start — preventing the API's -7d-derived new week from overlapping
1609
- # the old week. Inserted by cmd_record_usage on detection; read by
1610
- # _apply_reset_events_to_weekrefs and the cost live-recompute path.
1611
- conn.execute(
1612
- """
1613
- CREATE TABLE IF NOT EXISTS week_reset_events (
1614
- id INTEGER PRIMARY KEY AUTOINCREMENT,
1615
- detected_at_utc TEXT NOT NULL,
1616
- old_week_end_at TEXT NOT NULL,
1617
- new_week_end_at TEXT NOT NULL,
1618
- effective_reset_at_utc TEXT NOT NULL,
1619
- observed_pre_credit_pct REAL,
1620
- account_key TEXT NOT NULL DEFAULT 'unattributed',
1621
- UNIQUE(account_key, old_week_end_at, new_week_end_at)
1685
+ conn.execute(
1686
+ """
1687
+ CREATE TABLE IF NOT EXISTS percent_milestones (
1688
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
1689
+ captured_at_utc TEXT NOT NULL,
1690
+ week_start_date TEXT NOT NULL,
1691
+ week_end_date TEXT NOT NULL,
1692
+ week_start_at TEXT,
1693
+ week_end_at TEXT,
1694
+ percent_threshold INTEGER NOT NULL,
1695
+ cumulative_cost_usd REAL NOT NULL,
1696
+ marginal_cost_usd REAL,
1697
+ usage_snapshot_id INTEGER NOT NULL,
1698
+ cost_snapshot_id INTEGER NOT NULL,
1699
+ reset_event_id INTEGER NOT NULL DEFAULT 0,
1700
+ account_key TEXT NOT NULL DEFAULT 'unattributed',
1701
+ UNIQUE(account_key, week_start_date, percent_threshold, reset_event_id)
1702
+ )
1703
+ """
1622
1704
  )
1623
- """
1624
- )
1625
- add_column_if_missing(
1626
- conn, "week_reset_events", "account_key",
1627
- "TEXT NOT NULL DEFAULT 'unattributed'")
1628
- _backfill_week_reset_events(conn)
1629
-
1630
- # ── five_hour_reset_events (Anthropic-issued in-place 5h credits) ──
1631
- # Parallel concept to ``week_reset_events`` for the 5h dimension; lives
1632
- # adjacent in ``_apply_schema`` because the two carry the same kind of
1633
- # signal at different cadences. Diverges from weekly in that the payload
1634
- # is the *percent values* (prior + post) rather than boundary keys,
1635
- # because the 5h variant has a stable ``five_hour_window_key`` and only
1636
- # the percent moves. See spec
1637
- # docs/superpowers/specs/2026-05-16-5h-in-place-credit-detection.md §3.1
1638
- # for rationale.
1639
- #
1640
- # UNIQUE(five_hour_window_key, effective_reset_at_utc)supports stacked
1641
- # credits across DISTINCT 10-min slots inside one block (see spec §2.3
1642
- # "Bounded stacked-credit resolution" for the cap statement: ~30 distinct
1643
- # slots per 5h block when floor matches ``_canonical_5h_window_key``'s
1644
- # 600-second floor; same-slot collisions silently absorbed by
1645
- # INSERT OR IGNORE — an intentional cap, not a bug).
1646
- #
1647
- # No FK per CLAUDE.md gotcha: FKs in this codebase are documentation-only
1648
- # (``PRAGMA foreign_keys`` not enabled). ``five_hour_window_key`` provides
1649
- # the join key without a formal FK.
1650
- #
1651
- # No ``_backfill_five_hour_reset_events`` call follows (forward-only ship
1652
- # per spec Q5; historical backfill deferred to a future issue).
1653
- conn.execute(
1654
- """
1655
- CREATE TABLE IF NOT EXISTS five_hour_reset_events (
1656
- id INTEGER PRIMARY KEY AUTOINCREMENT,
1657
- detected_at_utc TEXT NOT NULL,
1658
- five_hour_window_key INTEGER NOT NULL,
1659
- prior_percent REAL NOT NULL,
1660
- post_percent REAL NOT NULL,
1661
- effective_reset_at_utc TEXT NOT NULL,
1662
- account_key TEXT NOT NULL DEFAULT 'unattributed',
1663
- UNIQUE(account_key, five_hour_window_key, effective_reset_at_utc)
1705
+
1706
+ add_column_if_missing(conn, "percent_milestones", "five_hour_percent_at_crossing", "REAL")
1707
+ add_column_if_missing(
1708
+ conn, "percent_milestones", "account_key",
1709
+ "TEXT NOT NULL DEFAULT 'unattributed'")
1710
+ # reset_event_id: segment column added by migration 005. Fresh-install
1711
+ # DBs get it via the live CREATE TABLE above + the dispatcher
1712
+ # fast-stamps the migration. Existing pre-005 DBs trip the migration's
1713
+ # rename-recreate-copy idiom (handler in _cctally_db.py); the handler's
1714
+ # fast-path probe stamps the marker when the column is already present
1715
+ # (covers the corner case where a partially-upgraded DB has the column
1716
+ # but not the new UNIQUE re-run is safe).
1717
+
1718
+ # alerted_at: populated by the alert-dispatch path when a milestone-INSERT
1719
+ # row's threshold matches the user's configured alerts.weekly_thresholds /
1720
+ # alerts.five_hour_thresholds (and alerts.enabled is true). NULL means
1721
+ # "alerts were disabled at the moment of crossing OR the threshold wasn't
1722
+ # in the configured list" never "alert delivery failed" (dispatch is
1723
+ # best-effort and write-once forward-only). The matching ALTER for
1724
+ # `five_hour_milestones` lives right after that table's CREATE block
1725
+ # below, since the table doesn't exist yet at this point in `open_db()`.
1726
+ add_column_if_missing(conn, "percent_milestones", "alerted_at", "TEXT")
1727
+
1728
+ # Mid-week reset events: when Anthropic advances `rate_limits.seven_day.
1729
+ # resets_at` before the previously-declared reset actually fires (i.e.,
1730
+ # gives the user a fresh weekly window before the old one naturally
1731
+ # expired), we record one row here so display + cost layers can treat
1732
+ # the effective reset moment as the old week's end AND the new week's
1733
+ # start preventing the API's -7d-derived new week from overlapping
1734
+ # the old week. Inserted by cmd_record_usage on detection; read by
1735
+ # _apply_reset_events_to_weekrefs and the cost live-recompute path.
1736
+ conn.execute(
1737
+ """
1738
+ CREATE TABLE IF NOT EXISTS week_reset_events (
1739
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
1740
+ detected_at_utc TEXT NOT NULL,
1741
+ old_week_end_at TEXT NOT NULL,
1742
+ new_week_end_at TEXT NOT NULL,
1743
+ effective_reset_at_utc TEXT NOT NULL,
1744
+ observed_pre_credit_pct REAL,
1745
+ account_key TEXT NOT NULL DEFAULT 'unattributed',
1746
+ UNIQUE(account_key, old_week_end_at, new_week_end_at)
1747
+ )
1748
+ """
1664
1749
  )
1665
- """
1666
- )
1667
- add_column_if_missing(
1668
- conn, "five_hour_reset_events", "account_key",
1669
- "TEXT NOT NULL DEFAULT 'unattributed'")
1750
+ add_column_if_missing(
1751
+ conn, "week_reset_events", "account_key",
1752
+ "TEXT NOT NULL DEFAULT 'unattributed'")
1753
+ _backfill_week_reset_events(conn)
1754
+
1755
+ # ── five_hour_reset_events (Anthropic-issued in-place 5h credits) ──
1756
+ # Parallel concept to ``week_reset_events`` for the 5h dimension; lives
1757
+ # adjacent in ``_apply_schema`` because the two carry the same kind of
1758
+ # signal at different cadences. Diverges from weekly in that the payload
1759
+ # is the *percent values* (prior + post) rather than boundary keys,
1760
+ # because the 5h variant has a stable ``five_hour_window_key`` and only
1761
+ # the percent moves. See spec
1762
+ # docs/superpowers/specs/2026-05-16-5h-in-place-credit-detection.md §3.1
1763
+ # for rationale.
1764
+ #
1765
+ # UNIQUE(five_hour_window_key, effective_reset_at_utc) — supports stacked
1766
+ # credits across DISTINCT 10-min slots inside one block (see spec §2.3
1767
+ # "Bounded stacked-credit resolution" for the cap statement: ~30 distinct
1768
+ # slots per 5h block when floor matches ``_canonical_5h_window_key``'s
1769
+ # 600-second floor; same-slot collisions silently absorbed by
1770
+ # INSERT OR IGNORE — an intentional cap, not a bug).
1771
+ #
1772
+ # No FK per CLAUDE.md gotcha: FKs in this codebase are documentation-only
1773
+ # (``PRAGMA foreign_keys`` not enabled). ``five_hour_window_key`` provides
1774
+ # the join key without a formal FK.
1775
+ #
1776
+ # No ``_backfill_five_hour_reset_events`` call follows (forward-only ship
1777
+ # per spec Q5; historical backfill deferred to a future issue).
1778
+ conn.execute(
1779
+ """
1780
+ CREATE TABLE IF NOT EXISTS five_hour_reset_events (
1781
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
1782
+ detected_at_utc TEXT NOT NULL,
1783
+ five_hour_window_key INTEGER NOT NULL,
1784
+ prior_percent REAL NOT NULL,
1785
+ post_percent REAL NOT NULL,
1786
+ effective_reset_at_utc TEXT NOT NULL,
1787
+ account_key TEXT NOT NULL DEFAULT 'unattributed',
1788
+ UNIQUE(account_key, five_hour_window_key, effective_reset_at_utc)
1789
+ )
1790
+ """
1791
+ )
1792
+ add_column_if_missing(
1793
+ conn, "five_hour_reset_events", "account_key",
1794
+ "TEXT NOT NULL DEFAULT 'unattributed'")
1670
1795
 
1671
- # ── five_hour_blocks (rollup, one row per API-anchored 5h block) ──
1672
- conn.execute(
1673
- """
1674
- CREATE TABLE IF NOT EXISTS five_hour_blocks (
1675
- id INTEGER PRIMARY KEY AUTOINCREMENT,
1676
- five_hour_window_key INTEGER NOT NULL,
1677
- five_hour_resets_at TEXT NOT NULL,
1678
- block_start_at TEXT NOT NULL,
1679
- first_observed_at_utc TEXT NOT NULL,
1680
- last_observed_at_utc TEXT NOT NULL,
1681
- final_five_hour_percent REAL NOT NULL,
1682
- seven_day_pct_at_block_start REAL,
1683
- seven_day_pct_at_block_end REAL,
1684
- crossed_seven_day_reset INTEGER NOT NULL DEFAULT 0,
1685
- total_input_tokens INTEGER NOT NULL DEFAULT 0,
1686
- total_output_tokens INTEGER NOT NULL DEFAULT 0,
1687
- total_cache_create_tokens INTEGER NOT NULL DEFAULT 0,
1688
- total_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
1689
- total_cost_usd REAL NOT NULL DEFAULT 0,
1690
- is_closed INTEGER NOT NULL DEFAULT 0,
1691
- created_at_utc TEXT NOT NULL,
1692
- last_updated_at_utc TEXT NOT NULL,
1693
- account_key TEXT NOT NULL DEFAULT 'unattributed',
1694
- UNIQUE(account_key, five_hour_window_key)
1796
+ # ── five_hour_blocks (rollup, one row per API-anchored 5h block) ──
1797
+ conn.execute(
1798
+ """
1799
+ CREATE TABLE IF NOT EXISTS five_hour_blocks (
1800
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
1801
+ five_hour_window_key INTEGER NOT NULL,
1802
+ five_hour_resets_at TEXT NOT NULL,
1803
+ block_start_at TEXT NOT NULL,
1804
+ first_observed_at_utc TEXT NOT NULL,
1805
+ last_observed_at_utc TEXT NOT NULL,
1806
+ final_five_hour_percent REAL NOT NULL,
1807
+ seven_day_pct_at_block_start REAL,
1808
+ seven_day_pct_at_block_end REAL,
1809
+ crossed_seven_day_reset INTEGER NOT NULL DEFAULT 0,
1810
+ total_input_tokens INTEGER NOT NULL DEFAULT 0,
1811
+ total_output_tokens INTEGER NOT NULL DEFAULT 0,
1812
+ total_cache_create_tokens INTEGER NOT NULL DEFAULT 0,
1813
+ total_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
1814
+ total_cost_usd REAL NOT NULL DEFAULT 0,
1815
+ is_closed INTEGER NOT NULL DEFAULT 0,
1816
+ created_at_utc TEXT NOT NULL,
1817
+ last_updated_at_utc TEXT NOT NULL,
1818
+ account_key TEXT NOT NULL DEFAULT 'unattributed',
1819
+ UNIQUE(account_key, five_hour_window_key)
1820
+ )
1821
+ """
1822
+ )
1823
+ add_column_if_missing(
1824
+ conn, "five_hour_blocks", "account_key",
1825
+ "TEXT NOT NULL DEFAULT 'unattributed'")
1826
+ conn.execute(
1827
+ """
1828
+ CREATE INDEX IF NOT EXISTS idx_five_hour_blocks_block_start
1829
+ ON five_hour_blocks(block_start_at DESC)
1830
+ """
1695
1831
  )
1696
- """
1697
- )
1698
- add_column_if_missing(
1699
- conn, "five_hour_blocks", "account_key",
1700
- "TEXT NOT NULL DEFAULT 'unattributed'")
1701
- conn.execute(
1702
- """
1703
- CREATE INDEX IF NOT EXISTS idx_five_hour_blocks_block_start
1704
- ON five_hour_blocks(block_start_at DESC)
1705
- """
1706
- )
1707
1832
 
1708
- # ── five_hour_milestones (per-percent crossings inside a 5h block) ──
1709
- conn.execute(
1710
- """
1711
- CREATE TABLE IF NOT EXISTS five_hour_milestones (
1712
- id INTEGER PRIMARY KEY AUTOINCREMENT,
1713
- block_id INTEGER NOT NULL,
1714
- five_hour_window_key INTEGER NOT NULL,
1715
- percent_threshold INTEGER NOT NULL,
1716
- captured_at_utc TEXT NOT NULL,
1717
- usage_snapshot_id INTEGER NOT NULL,
1718
- block_input_tokens INTEGER NOT NULL DEFAULT 0,
1719
- block_output_tokens INTEGER NOT NULL DEFAULT 0,
1720
- block_cache_create_tokens INTEGER NOT NULL DEFAULT 0,
1721
- block_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
1722
- block_cost_usd REAL NOT NULL DEFAULT 0,
1723
- marginal_cost_usd REAL,
1724
- seven_day_pct_at_crossing REAL,
1725
- reset_event_id INTEGER NOT NULL DEFAULT 0,
1726
- account_key TEXT NOT NULL DEFAULT 'unattributed',
1727
- UNIQUE(account_key, five_hour_window_key, percent_threshold, reset_event_id),
1728
- FOREIGN KEY (block_id) REFERENCES five_hour_blocks(id)
1833
+ # ── five_hour_milestones (per-percent crossings inside a 5h block) ──
1834
+ conn.execute(
1835
+ """
1836
+ CREATE TABLE IF NOT EXISTS five_hour_milestones (
1837
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
1838
+ block_id INTEGER NOT NULL,
1839
+ five_hour_window_key INTEGER NOT NULL,
1840
+ percent_threshold INTEGER NOT NULL,
1841
+ captured_at_utc TEXT NOT NULL,
1842
+ usage_snapshot_id INTEGER NOT NULL,
1843
+ block_input_tokens INTEGER NOT NULL DEFAULT 0,
1844
+ block_output_tokens INTEGER NOT NULL DEFAULT 0,
1845
+ block_cache_create_tokens INTEGER NOT NULL DEFAULT 0,
1846
+ block_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
1847
+ block_cost_usd REAL NOT NULL DEFAULT 0,
1848
+ marginal_cost_usd REAL,
1849
+ seven_day_pct_at_crossing REAL,
1850
+ reset_event_id INTEGER NOT NULL DEFAULT 0,
1851
+ account_key TEXT NOT NULL DEFAULT 'unattributed',
1852
+ UNIQUE(account_key, five_hour_window_key, percent_threshold, reset_event_id),
1853
+ FOREIGN KEY (block_id) REFERENCES five_hour_blocks(id)
1854
+ )
1855
+ """
1856
+ )
1857
+ conn.execute(
1858
+ """
1859
+ CREATE INDEX IF NOT EXISTS idx_five_hour_milestones_block
1860
+ ON five_hour_milestones(block_id)
1861
+ """
1729
1862
  )
1730
- """
1731
- )
1732
- conn.execute(
1733
- """
1734
- CREATE INDEX IF NOT EXISTS idx_five_hour_milestones_block
1735
- ON five_hour_milestones(block_id)
1736
- """
1737
- )
1738
1863
 
1739
- # alerted_at: see the matching ALTER on `percent_milestones` above for
1740
- # rationale. Same write-once forward-only semantics: the alert-dispatch
1741
- # path stamps this column on milestone-INSERT rows whose threshold
1742
- # matches the user's configured `alerts.five_hour_thresholds`. NULL =
1743
- # "alerts disabled at moment of crossing OR threshold not configured"
1744
- # — never "delivery failed".
1745
- add_column_if_missing(conn, "five_hour_milestones", "alerted_at", "TEXT")
1746
- add_column_if_missing(
1747
- conn, "five_hour_milestones", "account_key",
1748
- "TEXT NOT NULL DEFAULT 'unattributed'")
1749
-
1750
- # reset_event_id: segment column added by migration 006. Fresh-install
1751
- # DBs get it via the live CREATE TABLE above + the dispatcher fast-stamps
1752
- # the migration marker (the live DDL must carry the column AND the 3-col
1753
- # UNIQUE for fast-stamp to be safe — see spec §3.2). Existing pre-006
1754
- # DBs trip the migration's rename-recreate-copy idiom (handler in
1755
- # bin/_cctally_db.py); the handler's fast-path probe stamps the marker
1756
- # when the column is already present (covers the corner case where a
1757
- # partially-upgraded DB has the column but not the new UNIQUE — re-run
1758
- # is safe). Mirrors weekly migration 005 / `percent_milestones`.
1759
-
1760
- # ── five_hour_block_models (per-(block, model) rollup-child) ──
1761
- # MUST be created BEFORE the parent-backfill gate below, because
1762
- # _backfill_five_hour_blocks writes into this table on the fresh-install
1763
- # path. UNIQUE keyed on (five_hour_window_key, model) — durable across
1764
- # parent rebuilds. Live writes use DELETE WHERE five_hour_window_key = ?.
1765
- conn.execute(
1766
- """
1767
- CREATE TABLE IF NOT EXISTS five_hour_block_models (
1768
- id INTEGER PRIMARY KEY AUTOINCREMENT,
1769
- block_id INTEGER NOT NULL,
1770
- five_hour_window_key INTEGER NOT NULL,
1771
- model TEXT NOT NULL,
1772
- input_tokens INTEGER NOT NULL DEFAULT 0,
1773
- output_tokens INTEGER NOT NULL DEFAULT 0,
1774
- cache_create_tokens INTEGER NOT NULL DEFAULT 0,
1775
- cache_read_tokens INTEGER NOT NULL DEFAULT 0,
1776
- cost_usd REAL NOT NULL DEFAULT 0,
1777
- entry_count INTEGER NOT NULL DEFAULT 0,
1778
- account_key TEXT NOT NULL DEFAULT 'unattributed',
1779
- UNIQUE(account_key, five_hour_window_key, model),
1780
- FOREIGN KEY (block_id) REFERENCES five_hour_blocks(id)
1864
+ # alerted_at: see the matching ALTER on `percent_milestones` above for
1865
+ # rationale. Same write-once forward-only semantics: the alert-dispatch
1866
+ # path stamps this column on milestone-INSERT rows whose threshold
1867
+ # matches the user's configured `alerts.five_hour_thresholds`. NULL =
1868
+ # "alerts disabled at moment of crossing OR threshold not configured"
1869
+ # — never "delivery failed".
1870
+ add_column_if_missing(conn, "five_hour_milestones", "alerted_at", "TEXT")
1871
+ add_column_if_missing(
1872
+ conn, "five_hour_milestones", "account_key",
1873
+ "TEXT NOT NULL DEFAULT 'unattributed'")
1874
+
1875
+ # reset_event_id: segment column added by migration 006. Fresh-install
1876
+ # DBs get it via the live CREATE TABLE above + the dispatcher fast-stamps
1877
+ # the migration marker (the live DDL must carry the column AND the 3-col
1878
+ # UNIQUE for fast-stamp to be safe — see spec §3.2). Existing pre-006
1879
+ # DBs trip the migration's rename-recreate-copy idiom (handler in
1880
+ # bin/_cctally_db.py); the handler's fast-path probe stamps the marker
1881
+ # when the column is already present (covers the corner case where a
1882
+ # partially-upgraded DB has the column but not the new UNIQUE — re-run
1883
+ # is safe). Mirrors weekly migration 005 / `percent_milestones`.
1884
+
1885
+ # ── five_hour_block_models (per-(block, model) rollup-child) ──
1886
+ # MUST be created BEFORE the parent-backfill gate below, because
1887
+ # _backfill_five_hour_blocks writes into this table on the fresh-install
1888
+ # path. UNIQUE keyed on (five_hour_window_key, model) — durable across
1889
+ # parent rebuilds. Live writes use DELETE WHERE five_hour_window_key = ?.
1890
+ conn.execute(
1891
+ """
1892
+ CREATE TABLE IF NOT EXISTS five_hour_block_models (
1893
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
1894
+ block_id INTEGER NOT NULL,
1895
+ five_hour_window_key INTEGER NOT NULL,
1896
+ model TEXT NOT NULL,
1897
+ input_tokens INTEGER NOT NULL DEFAULT 0,
1898
+ output_tokens INTEGER NOT NULL DEFAULT 0,
1899
+ cache_create_tokens INTEGER NOT NULL DEFAULT 0,
1900
+ cache_read_tokens INTEGER NOT NULL DEFAULT 0,
1901
+ cost_usd REAL NOT NULL DEFAULT 0,
1902
+ entry_count INTEGER NOT NULL DEFAULT 0,
1903
+ account_key TEXT NOT NULL DEFAULT 'unattributed',
1904
+ UNIQUE(account_key, five_hour_window_key, model),
1905
+ FOREIGN KEY (block_id) REFERENCES five_hour_blocks(id)
1906
+ )
1907
+ """
1908
+ )
1909
+ add_column_if_missing(
1910
+ conn, "five_hour_block_models", "account_key",
1911
+ "TEXT NOT NULL DEFAULT 'unattributed'")
1912
+ conn.execute(
1913
+ """
1914
+ CREATE INDEX IF NOT EXISTS idx_five_hour_block_models_block
1915
+ ON five_hour_block_models(block_id)
1916
+ """
1917
+ )
1918
+ conn.execute(
1919
+ """
1920
+ CREATE INDEX IF NOT EXISTS idx_five_hour_block_models_window
1921
+ ON five_hour_block_models(five_hour_window_key)
1922
+ """
1781
1923
  )
1782
- """
1783
- )
1784
- add_column_if_missing(
1785
- conn, "five_hour_block_models", "account_key",
1786
- "TEXT NOT NULL DEFAULT 'unattributed'")
1787
- conn.execute(
1788
- """
1789
- CREATE INDEX IF NOT EXISTS idx_five_hour_block_models_block
1790
- ON five_hour_block_models(block_id)
1791
- """
1792
- )
1793
- conn.execute(
1794
- """
1795
- CREATE INDEX IF NOT EXISTS idx_five_hour_block_models_window
1796
- ON five_hour_block_models(five_hour_window_key)
1797
- """
1798
- )
1799
1924
 
1800
- # ── five_hour_block_projects (per-(block, project_path) rollup-child) ──
1801
- # NULL session_files.project_path → '(unknown)' sentinel at write time,
1802
- # keeping reconcile invariant SUM(child.cost) == parent.total intact.
1803
- conn.execute(
1804
- """
1805
- CREATE TABLE IF NOT EXISTS five_hour_block_projects (
1806
- id INTEGER PRIMARY KEY AUTOINCREMENT,
1807
- block_id INTEGER NOT NULL,
1808
- five_hour_window_key INTEGER NOT NULL,
1809
- project_path TEXT NOT NULL,
1810
- input_tokens INTEGER NOT NULL DEFAULT 0,
1811
- output_tokens INTEGER NOT NULL DEFAULT 0,
1812
- cache_create_tokens INTEGER NOT NULL DEFAULT 0,
1813
- cache_read_tokens INTEGER NOT NULL DEFAULT 0,
1814
- cost_usd REAL NOT NULL DEFAULT 0,
1815
- entry_count INTEGER NOT NULL DEFAULT 0,
1816
- account_key TEXT NOT NULL DEFAULT 'unattributed',
1817
- UNIQUE(account_key, five_hour_window_key, project_path),
1818
- FOREIGN KEY (block_id) REFERENCES five_hour_blocks(id)
1925
+ # ── five_hour_block_projects (per-(block, project_path) rollup-child) ──
1926
+ # NULL session_files.project_path → '(unknown)' sentinel at write time,
1927
+ # keeping reconcile invariant SUM(child.cost) == parent.total intact.
1928
+ conn.execute(
1929
+ """
1930
+ CREATE TABLE IF NOT EXISTS five_hour_block_projects (
1931
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
1932
+ block_id INTEGER NOT NULL,
1933
+ five_hour_window_key INTEGER NOT NULL,
1934
+ project_path TEXT NOT NULL,
1935
+ input_tokens INTEGER NOT NULL DEFAULT 0,
1936
+ output_tokens INTEGER NOT NULL DEFAULT 0,
1937
+ cache_create_tokens INTEGER NOT NULL DEFAULT 0,
1938
+ cache_read_tokens INTEGER NOT NULL DEFAULT 0,
1939
+ cost_usd REAL NOT NULL DEFAULT 0,
1940
+ entry_count INTEGER NOT NULL DEFAULT 0,
1941
+ account_key TEXT NOT NULL DEFAULT 'unattributed',
1942
+ UNIQUE(account_key, five_hour_window_key, project_path),
1943
+ FOREIGN KEY (block_id) REFERENCES five_hour_blocks(id)
1944
+ )
1945
+ """
1946
+ )
1947
+ add_column_if_missing(
1948
+ conn, "five_hour_block_projects", "account_key",
1949
+ "TEXT NOT NULL DEFAULT 'unattributed'")
1950
+ conn.execute(
1951
+ """
1952
+ CREATE INDEX IF NOT EXISTS idx_five_hour_block_projects_block
1953
+ ON five_hour_block_projects(block_id)
1954
+ """
1955
+ )
1956
+ conn.execute(
1957
+ """
1958
+ CREATE INDEX IF NOT EXISTS idx_five_hour_block_projects_window
1959
+ ON five_hour_block_projects(five_hour_window_key)
1960
+ """
1819
1961
  )
1820
- """
1821
- )
1822
- add_column_if_missing(
1823
- conn, "five_hour_block_projects", "account_key",
1824
- "TEXT NOT NULL DEFAULT 'unattributed'")
1825
- conn.execute(
1826
- """
1827
- CREATE INDEX IF NOT EXISTS idx_five_hour_block_projects_block
1828
- ON five_hour_block_projects(block_id)
1829
- """
1830
- )
1831
- conn.execute(
1832
- """
1833
- CREATE INDEX IF NOT EXISTS idx_five_hour_block_projects_window
1834
- ON five_hour_block_projects(five_hour_window_key)
1835
- """
1836
- )
1837
1962
 
1838
- # ── budget_milestones (equiv-$ budget threshold crossings — issue #19) ──
1839
- # Write-once, forward-only (the exact posture of `five_hour_milestones`). A
1840
- # mid-week quota reset re-anchors `week_start_at` (see
1841
- # `_resolve_current_budget_window`), so the new window naturally gets
1842
- # fresh rows under UNIQUE(week_start_at, period, threshold) — no
1843
- # `reset_event_id` segment column needed (unlike the percent/5h tables).
1844
- # `week_start_at` stores the effective/re-anchored ISO string from the
1845
- # resolver (`isoformat(timespec="seconds")`); the resolver's
1846
- # `parse_iso_datetime` returns a HOST-LOCAL tz-aware datetime, so this
1847
- # dedup key carries the host's UTC offset (e.g. `…T07:00:00-07:00`) —
1848
- # host-consistent, NOT portable across hosts, same posture as
1849
- # `five_hour_blocks.block_start_at`. Firing + reconcile + the dashboard
1850
- # envelope all read/write the identical string on a given host, so the
1851
- # UNIQUE dedup is exact. `alerted_at` is stamped BEFORE the osascript Popen
1852
- # (set-then-dispatch invariant); NULL = "recorded without dispatch" (the
1853
- # forward-only-from-set reconcile path) OR "not yet dispatched", never
1854
- # "delivery failed".
1855
- # Unified vendor-tagged table (#143): one row per (vendor, period_start_at,
1856
- # period, threshold). `vendor` ∈ 'claude'|'codex'. `period_start_at` is the
1857
- # resolved period-window start instant (subscription-week OR calendar
1858
- # period-start). `period` is the configured period at crossing; NULL = pre-012
1859
- # unknown. Owned by migration 012_unify_budget_milestones_vendor (merge of the
1860
- # former budget_milestones + codex_budget_milestones). The Codex table is NO
1861
- # LONGER live-created here — migration 012 drops it and this CREATE must not
1862
- # resurrect it; migration 011 is hardened to skip it when absent (#143).
1863
- conn.execute(
1864
- """
1865
- CREATE TABLE IF NOT EXISTS budget_milestones (
1866
- id INTEGER PRIMARY KEY AUTOINCREMENT,
1867
- vendor TEXT NOT NULL,
1868
- period_start_at TEXT NOT NULL,
1869
- period TEXT,
1870
- threshold INTEGER NOT NULL,
1871
- budget_usd REAL NOT NULL,
1872
- spent_usd REAL NOT NULL,
1873
- consumption_pct REAL NOT NULL,
1874
- crossed_at_utc TEXT NOT NULL,
1875
- alerted_at TEXT,
1876
- account_key TEXT NOT NULL DEFAULT '*',
1877
- UNIQUE(vendor, account_key, period_start_at, period, threshold)
1963
+ # ── budget_milestones (equiv-$ budget threshold crossings — issue #19) ──
1964
+ # Write-once, forward-only (the exact posture of `five_hour_milestones`). A
1965
+ # mid-week quota reset re-anchors `week_start_at` (see
1966
+ # `_resolve_current_budget_window`), so the new window naturally gets
1967
+ # fresh rows under UNIQUE(week_start_at, period, threshold) — no
1968
+ # `reset_event_id` segment column needed (unlike the percent/5h tables).
1969
+ # `week_start_at` stores the effective/re-anchored ISO string from the
1970
+ # resolver (`isoformat(timespec="seconds")`); the resolver's
1971
+ # `parse_iso_datetime` returns a HOST-LOCAL tz-aware datetime, so this
1972
+ # dedup key carries the host's UTC offset (e.g. `…T07:00:00-07:00`) —
1973
+ # host-consistent, NOT portable across hosts, same posture as
1974
+ # `five_hour_blocks.block_start_at`. Firing + reconcile + the dashboard
1975
+ # envelope all read/write the identical string on a given host, so the
1976
+ # UNIQUE dedup is exact. `alerted_at` is stamped BEFORE the osascript Popen
1977
+ # (set-then-dispatch invariant); NULL = "recorded without dispatch" (the
1978
+ # forward-only-from-set reconcile path) OR "not yet dispatched", never
1979
+ # "delivery failed".
1980
+ # Unified vendor-tagged table (#143): one row per (vendor, period_start_at,
1981
+ # period, threshold). `vendor` ∈ 'claude'|'codex'. `period_start_at` is the
1982
+ # resolved period-window start instant (subscription-week OR calendar
1983
+ # period-start). `period` is the configured period at crossing; NULL = pre-012
1984
+ # unknown. Owned by migration 012_unify_budget_milestones_vendor (merge of the
1985
+ # former budget_milestones + codex_budget_milestones). The Codex table is NO
1986
+ # LONGER live-created here — migration 012 drops it and this CREATE must not
1987
+ # resurrect it; migration 011 is hardened to skip it when absent (#143).
1988
+ conn.execute(
1989
+ """
1990
+ CREATE TABLE IF NOT EXISTS budget_milestones (
1991
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
1992
+ vendor TEXT NOT NULL,
1993
+ period_start_at TEXT NOT NULL,
1994
+ period TEXT,
1995
+ threshold INTEGER NOT NULL,
1996
+ budget_usd REAL NOT NULL,
1997
+ spent_usd REAL NOT NULL,
1998
+ consumption_pct REAL NOT NULL,
1999
+ crossed_at_utc TEXT NOT NULL,
2000
+ alerted_at TEXT,
2001
+ account_key TEXT NOT NULL DEFAULT '*',
2002
+ UNIQUE(vendor, account_key, period_start_at, period, threshold)
2003
+ )
2004
+ """
1878
2005
  )
1879
- """
1880
- )
1881
- add_column_if_missing(
1882
- conn, "budget_milestones", "account_key", "TEXT NOT NULL DEFAULT '*'")
1883
-
1884
- # ── projected_milestones (week-average-pace projection crossings #121) ──
1885
- # Write-once, forward-only same posture as `budget_milestones` (no
1886
- # `reset_event_id` segment column). Two metrics share the table, keyed by
1887
- # `metric` ('weekly_pct' | 'budget_usd'); a level fires once the
1888
- # WEEK-AVERAGE projection (not the displayed high-end verdict) crosses
1889
- # `threshold`. `denominator` snapshots the target AT crossing (target_usd
1890
- # for budget_usd, 100.0 for weekly_pct) so the dashboard envelope renders
1891
- # context "$312 of $300" / "102% of cap" from the ROW, not from live config
1892
- # that may have changed since (Codex P0-4). A mid-week reset re-anchors
1893
- # `week_start_at` (new window fresh rows under the UNIQUE key), the
1894
- # budget-pattern reset handling hence NO `reset_event_id` column.
1895
- # `alerted_at` is stamped BEFORE the osascript Popen (set-then-dispatch).
1896
- # Schema owned by migration 011_budget_milestone_period_keys (the `period`
1897
- # column + the period-inclusive UNIQUE; see _cctally_db.py). `period` is
1898
- # NULL for pre-011 rows.
1899
- conn.execute(
1900
- """
1901
- CREATE TABLE IF NOT EXISTS projected_milestones (
1902
- id INTEGER PRIMARY KEY AUTOINCREMENT,
1903
- week_start_at TEXT NOT NULL, -- period-start instant (subscription-week OR calendar period-start; back-compat name)
1904
- period TEXT, -- configured period at crossing; NULL = pre-011 unknown (migration 011)
1905
- metric TEXT NOT NULL, -- 'weekly_pct' | 'budget_usd' | 'codex_budget_usd'
1906
- threshold INTEGER NOT NULL, -- 90 | 100
1907
- projected_value REAL NOT NULL,
1908
- denominator REAL NOT NULL, -- target_usd (budget / codex_budget) | 100.0 (weekly)
1909
- crossed_at_utc TEXT NOT NULL,
1910
- alerted_at TEXT,
1911
- account_key TEXT NOT NULL DEFAULT '*', -- '*' for vendor-budget metrics; real account for weekly_pct (Task 3)
1912
- UNIQUE(account_key, week_start_at, period, metric, threshold)
2006
+ add_column_if_missing(
2007
+ conn, "budget_milestones", "account_key", "TEXT NOT NULL DEFAULT '*'")
2008
+
2009
+ # ── projected_milestones (week-average-pace projection crossings #121) ──
2010
+ # Write-once, forward-only — same posture as `budget_milestones` (no
2011
+ # `reset_event_id` segment column). Two metrics share the table, keyed by
2012
+ # `metric` ('weekly_pct' | 'budget_usd'); a level fires once the
2013
+ # WEEK-AVERAGE projection (not the displayed high-end verdict) crosses
2014
+ # `threshold`. `denominator` snapshots the target AT crossing (target_usd
2015
+ # for budget_usd, 100.0 for weekly_pct) so the dashboard envelope renders
2016
+ # context "$312 of $300" / "102% of cap" from the ROW, not from live config
2017
+ # that may have changed since (Codex P0-4). A mid-week reset re-anchors
2018
+ # `week_start_at` (new window fresh rows under the UNIQUE key), the
2019
+ # budget-pattern reset handling hence NO `reset_event_id` column.
2020
+ # `alerted_at` is stamped BEFORE the osascript Popen (set-then-dispatch).
2021
+ # Schema owned by migration 011_budget_milestone_period_keys (the `period`
2022
+ # column + the period-inclusive UNIQUE; see _cctally_db.py). `period` is
2023
+ # NULL for pre-011 rows.
2024
+ conn.execute(
2025
+ """
2026
+ CREATE TABLE IF NOT EXISTS projected_milestones (
2027
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
2028
+ week_start_at TEXT NOT NULL, -- period-start instant (subscription-week OR calendar period-start; back-compat name)
2029
+ period TEXT, -- configured period at crossing; NULL = pre-011 unknown (migration 011)
2030
+ metric TEXT NOT NULL, -- 'weekly_pct' | 'budget_usd' | 'codex_budget_usd'
2031
+ threshold INTEGER NOT NULL, -- 90 | 100
2032
+ projected_value REAL NOT NULL,
2033
+ denominator REAL NOT NULL, -- target_usd (budget / codex_budget) | 100.0 (weekly)
2034
+ crossed_at_utc TEXT NOT NULL,
2035
+ alerted_at TEXT,
2036
+ account_key TEXT NOT NULL DEFAULT '*', -- '*' for vendor-budget metrics; real account for weekly_pct (Task 3)
2037
+ UNIQUE(account_key, week_start_at, period, metric, threshold)
2038
+ )
2039
+ """
1913
2040
  )
1914
- """
1915
- )
1916
- add_column_if_missing(
1917
- conn, "projected_milestones", "account_key", "TEXT NOT NULL DEFAULT '*'")
1918
-
1919
- # ── project_budget_milestones (per-project equiv-$ budget crossings) ──────
1920
- # Plain CREATE TABLE IF NOT EXISTS, NO migration handler / backfill the
1921
- # same posture as `budget_milestones` / `projected_milestones` (write-once,
1922
- # forward-only, framework-untracked). `project_key` is the NEW dimension in
1923
- # the UNIQUE key: each project crosses each threshold once per week,
1924
- # independently of every other project (issue #19 / #121, spec §5.1). It
1925
- # stores the canonical git-root (`ProjectKey.bucket_path`), matched by string
1926
- # equality against each session entry's resolved git-root. `budget_usd`
1927
- # snapshots the project's target AT crossing time so the dashboard renders
1928
- # "$26 of $25" from the ROW, not from live config that may have changed since
1929
- # (the Codex P0-4 lesson, already baked into `budget_milestones` /
1930
- # `projected_milestones`). A mid-week quota reset re-anchors `week_start_at`
1931
- # (new window fresh rows under the UNIQUE key) — budget-pattern reset
1932
- # handling, hence NO `reset_event_id` segment column. `alerted_at` is stamped
1933
- # BEFORE dispatch (set-then-dispatch invariant); NULL = "recorded without
1934
- # dispatch" (forward-only-from-set reconcile) OR "not yet dispatched", never
1935
- # "delivery failed". Lives BEFORE the migration dispatcher: a plain CREATE on
1936
- # a framework-untracked table never touches `schema_migrations`, so the
1937
- # dispatcher's fresh-install snapshot is unaffected.
1938
- conn.execute(
1939
- """
1940
- CREATE TABLE IF NOT EXISTS project_budget_milestones (
1941
- id INTEGER PRIMARY KEY AUTOINCREMENT,
1942
- week_start_at TEXT NOT NULL,
1943
- project_key TEXT NOT NULL, -- canonical git-root (bucket_path)
1944
- threshold INTEGER NOT NULL,
1945
- budget_usd REAL NOT NULL, -- project's target snapshotted AT crossing
1946
- spent_usd REAL NOT NULL,
1947
- consumption_pct REAL NOT NULL,
1948
- crossed_at_utc TEXT NOT NULL,
1949
- alerted_at TEXT,
1950
- account_key TEXT NOT NULL DEFAULT '*', -- account-blind this epic (spec §6): always '*'
1951
- UNIQUE(account_key, week_start_at, project_key, threshold)
2041
+ add_column_if_missing(
2042
+ conn, "projected_milestones", "account_key", "TEXT NOT NULL DEFAULT '*'")
2043
+
2044
+ # ── project_budget_milestones (per-project equiv-$ budget crossings) ──────
2045
+ # Plain CREATE TABLE IF NOT EXISTS, NO migration handler / backfill — the
2046
+ # same posture as `budget_milestones` / `projected_milestones` (write-once,
2047
+ # forward-only, framework-untracked). `project_key` is the NEW dimension in
2048
+ # the UNIQUE key: each project crosses each threshold once per week,
2049
+ # independently of every other project (issue #19 / #121, spec §5.1). It
2050
+ # stores the canonical git-root (`ProjectKey.bucket_path`), matched by string
2051
+ # equality against each session entry's resolved git-root. `budget_usd`
2052
+ # snapshots the project's target AT crossing time so the dashboard renders
2053
+ # "$26 of $25" from the ROW, not from live config that may have changed since
2054
+ # (the Codex P0-4 lesson, already baked into `budget_milestones` /
2055
+ # `projected_milestones`). A mid-week quota reset re-anchors `week_start_at`
2056
+ # (new window fresh rows under the UNIQUE key) — budget-pattern reset
2057
+ # handling, hence NO `reset_event_id` segment column. `alerted_at` is stamped
2058
+ # BEFORE dispatch (set-then-dispatch invariant); NULL = "recorded without
2059
+ # dispatch" (forward-only-from-set reconcile) OR "not yet dispatched", never
2060
+ # "delivery failed". Lives BEFORE the migration dispatcher: a plain CREATE on
2061
+ # a framework-untracked table never touches `schema_migrations`, so the
2062
+ # dispatcher's fresh-install snapshot is unaffected.
2063
+ conn.execute(
2064
+ """
2065
+ CREATE TABLE IF NOT EXISTS project_budget_milestones (
2066
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
2067
+ week_start_at TEXT NOT NULL,
2068
+ project_key TEXT NOT NULL, -- canonical git-root (bucket_path)
2069
+ threshold INTEGER NOT NULL,
2070
+ budget_usd REAL NOT NULL, -- project's target snapshotted AT crossing
2071
+ spent_usd REAL NOT NULL,
2072
+ consumption_pct REAL NOT NULL,
2073
+ crossed_at_utc TEXT NOT NULL,
2074
+ alerted_at TEXT,
2075
+ account_key TEXT NOT NULL DEFAULT '*', -- account-blind this epic (spec §6): always '*'
2076
+ UNIQUE(account_key, week_start_at, project_key, threshold)
2077
+ )
2078
+ """
1952
2079
  )
1953
- """
1954
- )
1955
- add_column_if_missing(
1956
- conn, "project_budget_milestones", "account_key",
1957
- "TEXT NOT NULL DEFAULT '*'")
1958
-
1959
- # In-place weekly partial-credit floor (issue #209, record-credit M2).
1960
- # Plain CREATE TABLE IF NOT EXISTS, NO migration handler / NO user_version
1961
- # bump the same framework-untracked posture as `project_budget_milestones`
1962
- # above. A `record-credit` invocation records a weekly credit (e.g.
1963
- # 46% -> 31%) WITHOUT writing a `week_reset_events` row: a credit lowers the
1964
- # current-7d clamp floor only and must NOT re-anchor the week window (the
1965
- # `week_reset_events`-driven window-resolution code would otherwise show a
1966
- # spurious "new week" and corrupt the forecast rate). `_reset_aware_floor`
1967
- # (below) unions this table with `week_reset_events` so the four MAX-clamp
1968
- # sites floor the current % to the post-credit value while the window stays
1969
- # put. `effective_at_utc` is `floor_to_hour(at)` in UTC; `applied_at_utc` is
1970
- # audit-only (kept out of goldens). Lives BEFORE the migration dispatcher: a
1971
- # plain CREATE on a framework-untracked table never touches
1972
- # `schema_migrations`, so the dispatcher's fresh-install snapshot is
1973
- # unaffected. See docs/superpowers/specs/2026-06-19-record-credit-weekly-design.md §2/§4a.
1974
- conn.execute(
1975
- """
1976
- CREATE TABLE IF NOT EXISTS weekly_credit_floors (
1977
- id INTEGER PRIMARY KEY AUTOINCREMENT,
1978
- week_start_date TEXT NOT NULL,
1979
- effective_at_utc TEXT NOT NULL,
1980
- observed_pre_credit_pct REAL NOT NULL,
1981
- applied_at_utc TEXT NOT NULL,
1982
- account_key TEXT NOT NULL DEFAULT 'unattributed',
1983
- UNIQUE(account_key, week_start_date, effective_at_utc)
2080
+ add_column_if_missing(
2081
+ conn, "project_budget_milestones", "account_key",
2082
+ "TEXT NOT NULL DEFAULT '*'")
2083
+
2084
+ # In-place weekly partial-credit floor (issue #209, record-credit M2).
2085
+ # Plain CREATE TABLE IF NOT EXISTS, NO migration handler / NO user_version
2086
+ # bump the same framework-untracked posture as `project_budget_milestones`
2087
+ # above. A `record-credit` invocation records a weekly credit (e.g.
2088
+ # 46% -> 31%) WITHOUT writing a `week_reset_events` row: a credit lowers the
2089
+ # current-7d clamp floor only and must NOT re-anchor the week window (the
2090
+ # `week_reset_events`-driven window-resolution code would otherwise show a
2091
+ # spurious "new week" and corrupt the forecast rate). `_reset_aware_floor`
2092
+ # (below) unions this table with `week_reset_events` so the four MAX-clamp
2093
+ # sites floor the current % to the post-credit value while the window stays
2094
+ # put. `effective_at_utc` is `floor_to_hour(at)` in UTC; `applied_at_utc` is
2095
+ # audit-only (kept out of goldens). Lives BEFORE the migration dispatcher: a
2096
+ # plain CREATE on a framework-untracked table never touches
2097
+ # `schema_migrations`, so the dispatcher's fresh-install snapshot is
2098
+ # unaffected. See docs/superpowers/specs/2026-06-19-record-credit-weekly-design.md §2/§4a.
2099
+ conn.execute(
2100
+ """
2101
+ CREATE TABLE IF NOT EXISTS weekly_credit_floors (
2102
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
2103
+ week_start_date TEXT NOT NULL,
2104
+ effective_at_utc TEXT NOT NULL,
2105
+ observed_pre_credit_pct REAL NOT NULL,
2106
+ applied_at_utc TEXT NOT NULL,
2107
+ account_key TEXT NOT NULL DEFAULT 'unattributed',
2108
+ UNIQUE(account_key, week_start_date, effective_at_utc)
2109
+ )
2110
+ """
1984
2111
  )
1985
- """
1986
- )
1987
- add_column_if_missing(
1988
- conn, "weekly_credit_floors", "account_key",
1989
- "TEXT NOT NULL DEFAULT 'unattributed'")
1990
-
1991
- # ── accounts registry (multi-account epic #341, spec §1/§2) ───────────────
1992
- # Derived from the journal like all stats.db state: `account_observe` op
1993
- # lines fold into rows here (via _apply_op_account_observe), `account_label`
1994
- # ops set the user label. Framework-untracked (plain CREATE TABLE IF NOT
1995
- # EXISTS, no migration / no user_version bump same posture as
1996
- # weekly_credit_floors above), so it never touches `schema_migrations` and
1997
- # the stats-schema change rides the STATS_INDEX_EPOCH bump + rebuild, not a
1998
- # new stats migration (the frozen-registry rule). `last_seen_utc` is derived
1999
- # at fold time from the max `at` of any account-stamped line, NOT carried by
2000
- # the observe record. `label_source` records provenance for the
2001
- # user > switcher > auto precedence rule. A new empty table is byte-invisible
2002
- # to every existing render, preserving the R8 byte-stability contract.
2003
- conn.execute(
2004
- """
2005
- CREATE TABLE IF NOT EXISTS accounts (
2006
- account_key TEXT PRIMARY KEY,
2007
- provider TEXT NOT NULL,
2008
- natural_id TEXT,
2009
- email TEXT,
2010
- label TEXT,
2011
- plan_type TEXT,
2012
- label_source TEXT NOT NULL DEFAULT 'auto',
2013
- first_seen_utc TEXT,
2014
- last_seen_utc TEXT
2112
+ add_column_if_missing(
2113
+ conn, "weekly_credit_floors", "account_key",
2114
+ "TEXT NOT NULL DEFAULT 'unattributed'")
2115
+
2116
+ # ── accounts registry (multi-account epic #341, spec §1/§2) ───────────────
2117
+ # Derived from the journal like all stats.db state: `account_observe` op
2118
+ # lines fold into rows here (via _apply_op_account_observe), `account_label`
2119
+ # ops set the user label. Framework-untracked (plain CREATE TABLE IF NOT
2120
+ # EXISTS, no migration / no user_version bump — same posture as
2121
+ # weekly_credit_floors above), so it never touches `schema_migrations` and
2122
+ # the stats-schema change rides the STATS_INDEX_EPOCH bump + rebuild, not a
2123
+ # new stats migration (the frozen-registry rule). `last_seen_utc` is derived
2124
+ # at fold time from the max `at` of any account-stamped line, NOT carried by
2125
+ # the observe record. `label_source` records provenance for the
2126
+ # user > switcher > auto precedence rule. A new empty table is byte-invisible
2127
+ # to every existing render, preserving the R8 byte-stability contract.
2128
+ conn.execute(
2129
+ """
2130
+ CREATE TABLE IF NOT EXISTS accounts (
2131
+ account_key TEXT PRIMARY KEY,
2132
+ provider TEXT NOT NULL,
2133
+ natural_id TEXT,
2134
+ email TEXT,
2135
+ label TEXT,
2136
+ plan_type TEXT,
2137
+ label_source TEXT NOT NULL DEFAULT 'auto',
2138
+ first_seen_utc TEXT,
2139
+ last_seen_utc TEXT
2140
+ )
2141
+ """
2015
2142
  )
2016
- """
2017
- )
2018
2143
 
2019
- # Stats migration 013 owns durable quota interpretation. Keep the current
2020
- # schema in the fresh-install path before the dispatcher, exactly like the
2021
- # existing live CREATE tables; its handler calls this same idempotent helper
2022
- # for an older stats.db and the dispatcher central-stamps on clean return.
2023
- # §6.2 backfill gate (Task 8): the quota-projection schema apply is one of
2024
- # the three open-time backfills — skipped once the fixups marker is stamped
2025
- # (the marker is set only AFTER this ran, so marker-present ⇒ tables present).
2026
- # Fresh installs read False and apply it here (the dispatcher fast-stamps 013
2027
- # without running its handler, so this open-time call is the sole creator).
2028
- if not _fixups_current:
2029
- _apply_quota_projection_schema(conn)
2030
-
2031
- # Migration framework dispatcher. Replaces the prior inline gate stack
2032
- # (has_blocks + _migration_done) with the framework's _run_pending_-
2033
- # migrations entry point. See spec §2.3, §5.2 + the migration handlers
2034
- # decorated with @stats_migration further down in this file.
2035
- #
2036
- # MUST run BEFORE any DDL or write that touches `schema_migrations`
2037
- # (Codex P1 #1 fix on c3625ee + e7fdcc8): the dispatcher's fresh-install
2038
- # detection snapshots `schema_migrations`'s existence in sqlite_master
2039
- # BEFORE its own CREATE TABLE IF NOT EXISTS. Pre-creating the table
2040
- # earlier in open_db() (or letting `_backfill_five_hour_blocks` insert
2041
- # markers first) flips that snapshot to True on a brand-new DB and
2042
- # dead-codes the stamp-only fast path. The dispatcher is now the sole
2043
- # creator of `schema_migrations` + `schema_migrations_skipped`.
2044
- _run_pending_migrations(
2045
- conn, registry=_STATS_MIGRATIONS, db_label="stats.db",
2046
- )
2144
+ # Stats migration 013 owns durable quota interpretation. Keep the current
2145
+ # schema in the fresh-install path before the dispatcher, exactly like the
2146
+ # existing live CREATE tables; its handler calls this same idempotent helper
2147
+ # for an older stats.db and the dispatcher central-stamps on clean return.
2148
+ # §6.2 backfill gate (Task 8): the quota-projection schema apply is one of
2149
+ # the three open-time backfills — skipped once the fixups marker is stamped
2150
+ # (the marker is set only AFTER this ran, so marker-present ⇒ tables present).
2151
+ # Fresh installs read False and apply it here (the dispatcher fast-stamps 013
2152
+ # without running its handler, so this open-time call is the sole creator).
2153
+ if not _fixups_current:
2154
+ _apply_quota_projection_schema(conn)
2155
+
2156
+ # Migration framework dispatcher. Replaces the prior inline gate stack
2157
+ # (has_blocks + _migration_done) with the framework's _run_pending_-
2158
+ # migrations entry point. See spec §2.3, §5.2 + the migration handlers
2159
+ # decorated with @stats_migration further down in this file.
2160
+ #
2161
+ # MUST run BEFORE any DDL or write that touches `schema_migrations`
2162
+ # (Codex P1 #1 fix on c3625ee + e7fdcc8): the dispatcher's fresh-install
2163
+ # detection snapshots `schema_migrations`'s existence in sqlite_master
2164
+ # BEFORE its own CREATE TABLE IF NOT EXISTS. Pre-creating the table
2165
+ # earlier in open_db() (or letting `_backfill_five_hour_blocks` insert
2166
+ # markers first) flips that snapshot to True on a brand-new DB and
2167
+ # dead-codes the stamp-only fast path. The dispatcher is now the sole
2168
+ # creator of `schema_migrations` + `schema_migrations_skipped`.
2169
+ _run_pending_migrations(
2170
+ conn, registry=_STATS_MIGRATIONS, db_label="stats.db",
2171
+ )
2047
2172
 
2048
- # One-time historical backfill of five_hour_blocks (rollup only;
2049
- # milestones are forward-only per spec §4.3 / [Write-once milestones]).
2050
- # Idempotent via UNIQUE(five_hour_window_key) + INSERT OR IGNORE.
2051
- # Runs AFTER the dispatcher so `schema_migrations` exists for the
2052
- # marker INSERTs inside the backfill body, and so any fresh-install
2053
- # stamp-only path the dispatcher took above is already committed.
2054
- # §6.2 backfill gate (Task 8): the two probe SELECTs + the backfill + its
2055
- # migration-003 re-invocation are open-time backfill work — skipped once the
2056
- # fixups marker is stamped. Dead in the journal world (the ingest cycle writes
2057
- # blocks with their snapshots), so it fires only on a pre-journal upgrade DB's
2058
- # first open; after that the marker gates it out permanently.
2059
- if not _fixups_current:
2060
- existing = conn.execute(
2061
- "SELECT 1 FROM five_hour_blocks LIMIT 1"
2062
- ).fetchone()
2063
- has_snapshots = conn.execute(
2064
- "SELECT 1 FROM weekly_usage_snapshots "
2065
- "WHERE five_hour_window_key IS NOT NULL "
2066
- " AND five_hour_percent IS NOT NULL "
2067
- "LIMIT 1"
2068
- ).fetchone()
2069
- if not existing and has_snapshots:
2070
- inserted = _backfill_five_hour_blocks(conn)
2071
- # Re-run the 5h dedup migration AFTER backfill creates parents.
2072
- # The dispatcher above ran while five_hour_blocks was empty, so
2073
- # the dedup handler no-op'd and stamped its marker. Snapshot
2074
- # keys can carry jitter beyond the 600s canonical floor (the
2075
- # 003_* migration handles up to 1800s grouping), so the
2076
- # backfill's `DISTINCT five_hour_window_key` over those keys
2077
- # can produce duplicate parent rows for one physical 5h
2078
- # window. Without this re-invocation those duplicates persist
2079
- # forever — the marker says it ran. Handler owns its own
2080
- # BEGIN/COMMIT and is idempotent (no groups → no-op).
2081
- #
2082
- # Honor `db skip` here as well: if the operator marked 003 as
2083
- # skipped (e.g., poison pill on their machine), we must NOT
2084
- # back-door run the handler. Duplicates introduced by the
2085
- # backfill will persist until they `db unskip` — which is the
2086
- # explicit choice the skip records. Failure path mirrors the
2087
- # dispatcher's contract: route through _log_migration_error so
2088
- # the next interactive command renders the banner, and clear
2089
- # the log entry on success so the banner auto-dismisses.
2090
- if inserted > 0:
2091
- target_name = "003_merge_5h_block_duplicates_v1"
2092
- try:
2093
- skipped = {
2094
- row[0] for row in conn.execute(
2095
- "SELECT name FROM schema_migrations_skipped"
2096
- ).fetchall()
2097
- }
2098
- except sqlite3.OperationalError:
2099
- skipped = set()
2100
- if target_name not in skipped:
2101
- for _m in _STATS_MIGRATIONS:
2102
- if _m.name == target_name:
2103
- qualified = f"stats.db:{target_name}"
2104
- try:
2105
- _m.handler(conn)
2106
- _clear_migration_error_log_entries(qualified)
2107
- except Exception as exc:
2108
- _log_migration_error(
2109
- name=qualified,
2110
- exc=exc,
2111
- tb=traceback.format_exc(),
2112
- )
2113
- eprint(f"[migration {qualified}] failed: {exc}")
2114
- break
2115
-
2116
- # ── Append-only journal replay-identity columns + ingest cursor ──
2117
- # (2026-07-22 DB journal redesign, spec §4.2 / §5.2). Every row a
2118
- # journal fold materializes carries the originating line's stable `id`
2119
- # in `journal_id`, with a partial UNIQUE index so the ingester's
2120
- # INSERT OR IGNORE fold is idempotent under replay/re-ingest. Runs AFTER
2121
- # the migration dispatcher so the columns land on the final (migrated)
2122
- # table shape — migrations 005/006 recreate percent/5h milestone tables
2123
- # and must not drop the column. All additive (add_column_if_missing /
2124
- # CREATE ... IF NOT EXISTS), framework-untracked — same posture as
2125
- # weekly_credit_floors / project_budget_milestones. Task 9 folds this
2126
- # under the STATS_INDEX_EPOCH version gate; until then it is idempotent
2127
- # per open (add_column_if_missing / IF NOT EXISTS no-op once present).
2128
- for _jtable in (
2129
- "weekly_usage_snapshots", "weekly_cost_snapshots", "week_reset_events",
2130
- "five_hour_reset_events", "five_hour_blocks", "weekly_credit_floors",
2131
- "percent_milestones", "five_hour_milestones", "budget_milestones",
2132
- "projected_milestones", "project_budget_milestones",
2133
- ):
2134
- add_column_if_missing(conn, _jtable, "journal_id", "TEXT")
2173
+ # One-time historical backfill of five_hour_blocks (rollup only;
2174
+ # milestones are forward-only per spec §4.3 / [Write-once milestones]).
2175
+ # Idempotent via UNIQUE(five_hour_window_key) + INSERT OR IGNORE.
2176
+ # Runs AFTER the dispatcher so `schema_migrations` exists for the
2177
+ # marker INSERTs inside the backfill body, and so any fresh-install
2178
+ # stamp-only path the dispatcher took above is already committed.
2179
+ # §6.2 backfill gate (Task 8): the two probe SELECTs + the backfill + its
2180
+ # migration-003 re-invocation are open-time backfill work — skipped once the
2181
+ # fixups marker is stamped. Dead in the journal world (the ingest cycle writes
2182
+ # blocks with their snapshots), so it fires only on a pre-journal upgrade DB's
2183
+ # first open; after that the marker gates it out permanently.
2184
+ if not _fixups_current:
2185
+ existing = conn.execute(
2186
+ "SELECT 1 FROM five_hour_blocks LIMIT 1"
2187
+ ).fetchone()
2188
+ has_snapshots = conn.execute(
2189
+ "SELECT 1 FROM weekly_usage_snapshots "
2190
+ "WHERE five_hour_window_key IS NOT NULL "
2191
+ " AND five_hour_percent IS NOT NULL "
2192
+ "LIMIT 1"
2193
+ ).fetchone()
2194
+ if not existing and has_snapshots:
2195
+ inserted = _backfill_five_hour_blocks(conn)
2196
+ # Re-run the 5h dedup migration AFTER backfill creates parents.
2197
+ # The dispatcher above ran while five_hour_blocks was empty, so
2198
+ # the dedup handler no-op'd and stamped its marker. Snapshot
2199
+ # keys can carry jitter beyond the 600s canonical floor (the
2200
+ # 003_* migration handles up to 1800s grouping), so the
2201
+ # backfill's `DISTINCT five_hour_window_key` over those keys
2202
+ # can produce duplicate parent rows for one physical 5h
2203
+ # window. Without this re-invocation those duplicates persist
2204
+ # forever — the marker says it ran. Handler owns its own
2205
+ # BEGIN/COMMIT and is idempotent (no groups → no-op).
2206
+ #
2207
+ # Honor `db skip` here as well: if the operator marked 003 as
2208
+ # skipped (e.g., poison pill on their machine), we must NOT
2209
+ # back-door run the handler. Duplicates introduced by the
2210
+ # backfill will persist until they `db unskip` — which is the
2211
+ # explicit choice the skip records. Failure path mirrors the
2212
+ # dispatcher's contract: route through _log_migration_error so
2213
+ # the next interactive command renders the banner, and clear
2214
+ # the log entry on success so the banner auto-dismisses.
2215
+ if inserted > 0:
2216
+ target_name = "003_merge_5h_block_duplicates_v1"
2217
+ try:
2218
+ skipped = {
2219
+ row[0] for row in conn.execute(
2220
+ "SELECT name FROM schema_migrations_skipped"
2221
+ ).fetchall()
2222
+ }
2223
+ except sqlite3.OperationalError:
2224
+ skipped = set()
2225
+ if target_name not in skipped:
2226
+ for _m in _STATS_MIGRATIONS:
2227
+ if _m.name == target_name:
2228
+ qualified = f"stats.db:{target_name}"
2229
+ try:
2230
+ _m.handler(conn)
2231
+ _clear_migration_error_log_entries(qualified)
2232
+ except Exception as exc:
2233
+ _log_migration_error(
2234
+ name=qualified,
2235
+ exc=exc,
2236
+ tb=traceback.format_exc(),
2237
+ )
2238
+ eprint(f"[migration {qualified}] failed: {exc}")
2239
+ break
2240
+
2241
+ # ── Append-only journal replay-identity columns + ingest cursor ──
2242
+ # (2026-07-22 DB journal redesign, spec §4.2 / §5.2). Every row a
2243
+ # journal fold materializes carries the originating line's stable `id`
2244
+ # in `journal_id`, with a partial UNIQUE index so the ingester's
2245
+ # INSERT OR IGNORE fold is idempotent under replay/re-ingest. Runs AFTER
2246
+ # the migration dispatcher so the columns land on the final (migrated)
2247
+ # table shape — migrations 005/006 recreate percent/5h milestone tables
2248
+ # and must not drop the column. All additive (add_column_if_missing /
2249
+ # CREATE ... IF NOT EXISTS), framework-untracked — same posture as
2250
+ # weekly_credit_floors / project_budget_milestones. Task 9 folds this
2251
+ # under the STATS_INDEX_EPOCH version gate; until then it is idempotent
2252
+ # per open (add_column_if_missing / IF NOT EXISTS no-op once present).
2253
+ for _jtable in (
2254
+ "weekly_usage_snapshots", "weekly_cost_snapshots", "week_reset_events",
2255
+ "five_hour_reset_events", "five_hour_blocks", "weekly_credit_floors",
2256
+ "percent_milestones", "five_hour_milestones", "budget_milestones",
2257
+ "projected_milestones", "project_budget_milestones",
2258
+ ):
2259
+ add_column_if_missing(conn, _jtable, "journal_id", "TEXT")
2260
+ conn.execute(
2261
+ f"CREATE UNIQUE INDEX IF NOT EXISTS idx_{_jtable}_journal_id "
2262
+ f"ON {_jtable}(journal_id) WHERE journal_id IS NOT NULL"
2263
+ )
2264
+ # Companion partial index (spec §5.3 harvest / Task 6 gate P2): the ingest
2265
+ # cycle's harvest scans every natural-keyed family for rows the pipeline
2266
+ # inserted this cycle (`WHERE journal_id IS NULL`). A partial index over
2267
+ # exactly those un-stamped rows keeps that scan O(this-cycle inserts), not
2268
+ # O(table) — at the 10x envelope the stamped rows are ~all of the table, so
2269
+ # a full scan would be pathological. Only the 8 HARVEST families need it
2270
+ # (the Model-A / op-fold tables — weekly_usage_snapshots, weekly_cost_
2271
+ # snapshots, weekly_credit_floors — are never harvest-scanned).
2272
+ for _htable in (
2273
+ "week_reset_events", "five_hour_reset_events", "five_hour_blocks",
2274
+ "percent_milestones", "five_hour_milestones", "budget_milestones",
2275
+ "projected_milestones", "project_budget_milestones",
2276
+ ):
2277
+ conn.execute(
2278
+ f"CREATE INDEX IF NOT EXISTS idx_{_htable}_journal_id_null "
2279
+ f"ON {_htable}(id) WHERE journal_id IS NULL"
2280
+ )
2281
+ # Single-row segment+offset consumption watermark (spec §5.2). The
2282
+ # applied_* pair is written atomically with the materialized rows and
2283
+ # acts as the trusted prefix when a cursor-only hand edit advances the
2284
+ # public pair without applying the skipped journal bytes (#410 Task B).
2285
+ conn.execute(
2286
+ "CREATE TABLE IF NOT EXISTS journal_cursor ("
2287
+ "id INTEGER PRIMARY KEY CHECK (id = 1), "
2288
+ "segment TEXT NOT NULL, "
2289
+ "offset INTEGER NOT NULL, "
2290
+ "applied_segment TEXT, "
2291
+ "applied_offset INTEGER)"
2292
+ )
2293
+ add_column_if_missing(
2294
+ conn, "journal_cursor", "applied_segment", "TEXT"
2295
+ )
2296
+ add_column_if_missing(
2297
+ conn, "journal_cursor", "applied_offset", "INTEGER"
2298
+ )
2299
+ # Schema-apply compatibility for legacy/test-mode paths that reach this
2300
+ # DDL with a pre-pair cursor row. A released epoch-1003 index does NOT
2301
+ # use this as an upgrade shortcut: the epoch mismatch rebuilds it into
2302
+ # the complete epoch-1004 schema first.
2135
2303
  conn.execute(
2136
- f"CREATE UNIQUE INDEX IF NOT EXISTS idx_{_jtable}_journal_id "
2137
- f"ON {_jtable}(journal_id) WHERE journal_id IS NOT NULL"
2304
+ "UPDATE journal_cursor "
2305
+ "SET applied_segment = segment, applied_offset = offset "
2306
+ "WHERE applied_segment IS NULL AND applied_offset IS NULL"
2138
2307
  )
2139
- # Companion partial index (spec §5.3 harvest / Task 6 gate P2): the ingest
2140
- # cycle's harvest scans every natural-keyed family for rows the pipeline
2141
- # inserted this cycle (`WHERE journal_id IS NULL`). A partial index over
2142
- # exactly those un-stamped rows keeps that scan O(this-cycle inserts), not
2143
- # O(table) — at the 10x envelope the stamped rows are ~all of the table, so
2144
- # a full scan would be pathological. Only the 8 HARVEST families need it
2145
- # (the Model-A / op-fold tables — weekly_usage_snapshots, weekly_cost_
2146
- # snapshots, weekly_credit_floors — are never harvest-scanned).
2147
- for _htable in (
2148
- "week_reset_events", "five_hour_reset_events", "five_hour_blocks",
2149
- "percent_milestones", "five_hour_milestones", "budget_milestones",
2150
- "projected_milestones", "project_budget_milestones",
2151
- ):
2308
+ # Disposable effective-event summary (#372 Task A). Durable truth remains
2309
+ # the append-only journal; rebuild repopulates this table from the shared
2310
+ # pure selector. The table lets live replay detect completed corrections
2311
+ # without inventing family-specific inverse operations.
2152
2312
  conn.execute(
2153
- f"CREATE INDEX IF NOT EXISTS idx_{_htable}_journal_id_null "
2154
- f"ON {_htable}(id) WHERE journal_id IS NULL"
2313
+ "CREATE TABLE IF NOT EXISTS journal_effective_events ("
2314
+ "event_id TEXT PRIMARY KEY, "
2315
+ "rev INTEGER NOT NULL CHECK (rev >= 0), "
2316
+ "status TEXT NOT NULL CHECK (status IN ('active','tombstone')), "
2317
+ "content_hash TEXT NOT NULL, "
2318
+ "batch_id TEXT, "
2319
+ "event_json TEXT)"
2320
+ )
2321
+ # Disposable selector diagnostics (#402 Task A). The append-only journal
2322
+ # remains authoritative; rebuild/live preflight replace this bounded
2323
+ # summary after a complete correction-prefix selection.
2324
+ conn.execute(
2325
+ "CREATE TABLE IF NOT EXISTS journal_protocol_violations ("
2326
+ "fingerprint TEXT PRIMARY KEY, "
2327
+ "batch_id TEXT NOT NULL, "
2328
+ "kind TEXT NOT NULL, "
2329
+ "violation_json TEXT NOT NULL)"
2155
2330
  )
2156
- # Single-row segment+offset consumption watermark (spec §5.2). The cursor
2157
- # never advances past a byte range the ingest cycle did not read.
2158
- conn.execute(
2159
- "CREATE TABLE IF NOT EXISTS journal_cursor ("
2160
- "id INTEGER PRIMARY KEY CHECK (id = 1), "
2161
- "segment TEXT NOT NULL, "
2162
- "offset INTEGER NOT NULL)"
2163
- )
2164
2331
 
2165
- # §6.2 backfill gate (Task 8): stamp the one-shot marker AFTER the three
2166
- # open-time backfills ran, so the next open skips them (and their probes)
2167
- # entirely. The marker table DDL runs ONLY on this "fixups ran" path, never
2168
- # on the steady-state open. A crash before this commit leaves the marker
2169
- # unset and everything re-runs idempotently next open (invariant).
2170
- if not _fixups_current:
2171
- _cctally_store.mark_stats_open_fixups_done(conn)
2172
-
2173
- conn.commit()
2174
-
2175
- # ── Epoch stamp / in-place cutover (DB journal redesign §7.1/§8) ──
2176
- # The full schema is now applied (head 13 + journal_id columns + cursor). A
2177
- # ``_target_path`` build (rebuild scratch) stamps the epoch DIRECTLY (no
2178
- # export — the rebuild folds the journal itself). A real legacy install runs
2179
- # the cutover: export history to a bootstrap segment, stamp ``journal_id`` on
2180
- # every row, advance the cursor, and stamp the epoch — all atomic (spec §8).
2181
- # Under test mode (epoch disabled) neither runs, so the DB stays at
2182
- # len(registry) for the migration-framework harness.
2183
- if _epoch_engaged:
2184
- if _target_path is not None:
2185
- conn.execute(f"PRAGMA user_version = {STATS_INDEX_EPOCH}")
2186
- conn.commit()
2187
- elif conn.execute("PRAGMA user_version").fetchone()[0] == LEGACY_STATS_HEAD:
2188
- # Cut over ONLY once the legacy dispatcher reached the export baseline
2189
- # (head 13). A DEFERRED migration (MigrationGateNotMet — e.g. the
2190
- # 008/009/010 recompute gate) leaves user_version < 13; skip the
2191
- # cutover so the next open retries the dispatcher first, then cuts over
2192
- # (spec §8 step 1: "run any pending legacy stats migrations, reaching
2193
- # the export baseline"). Never journal a pre-recompute stats shape.
2194
- importlib.import_module("_cctally_journal").run_cutover(conn)
2195
- return conn
2332
+ # §6.2 backfill gate (Task 8): stamp the one-shot marker AFTER the three
2333
+ # open-time backfills ran, so the next open skips them (and their probes)
2334
+ # entirely. The marker table DDL runs ONLY on this "fixups ran" path, never
2335
+ # on the steady-state open. A crash before this commit leaves the marker
2336
+ # unset and everything re-runs idempotently next open (invariant).
2337
+ if not _fixups_current:
2338
+ _cctally_store.mark_stats_open_fixups_done(conn)
2339
+
2340
+ conn.commit()
2341
+
2342
+ # ── Epoch stamp / in-place cutover (DB journal redesign §7.1/§8) ──
2343
+ # The full schema is now applied (head 13 + journal_id columns + cursor). A
2344
+ # ``_target_path`` build (rebuild scratch) stamps the epoch DIRECTLY (no
2345
+ # export — the rebuild folds the journal itself). A real legacy install runs
2346
+ # the cutover: export history to a bootstrap segment, stamp ``journal_id`` on
2347
+ # every row, advance the cursor, and stamp the epoch — all atomic (spec §8).
2348
+ # Under test mode (epoch disabled) neither runs, so the DB stays at
2349
+ # len(registry) for the migration-framework harness.
2350
+ if _epoch_engaged:
2351
+ if _target_path is not None:
2352
+ conn.execute(f"PRAGMA user_version = {STATS_INDEX_EPOCH}")
2353
+ conn.commit()
2354
+ elif conn.execute("PRAGMA user_version").fetchone()[0] == LEGACY_STATS_HEAD:
2355
+ # Cut over ONLY once the legacy dispatcher reached the export baseline
2356
+ # (head 13). A DEFERRED migration (MigrationGateNotMet — e.g. the
2357
+ # 008/009/010 recompute gate) leaves user_version < 13; skip the
2358
+ # cutover so the next open retries the dispatcher first, then cuts over
2359
+ # (spec §8 step 1: "run any pending legacy stats migrations, reaching
2360
+ # the export baseline"). Never journal a pre-recompute stats shape.
2361
+ try:
2362
+ importlib.import_module("_cctally_journal").run_cutover(conn)
2363
+ except BaseException:
2364
+ try:
2365
+ conn.close()
2366
+ except Exception:
2367
+ pass
2368
+ raise
2369
+ return conn
2196
2370
 
2197
2371
 
2198
2372
  # === WeekRef cluster ================================================