cctally 1.82.0 → 1.83.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -0
- package/README.md +52 -74
- package/bin/_cctally_alerts.py +8 -1
- package/bin/_cctally_cache.py +963 -149
- package/bin/_cctally_config.py +43 -4
- package/bin/_cctally_core.py +933 -759
- package/bin/_cctally_dashboard.py +157 -47
- package/bin/_cctally_dashboard_cache_report.py +13 -6
- package/bin/_cctally_dashboard_conversation.py +1 -0
- package/bin/_cctally_dashboard_envelope.py +186 -8
- package/bin/_cctally_dashboard_share.py +60 -20
- package/bin/_cctally_dashboard_sources.py +427 -128
- package/bin/_cctally_db.py +605 -128
- package/bin/_cctally_doctor.py +413 -28
- package/bin/_cctally_five_hour.py +12 -5
- package/bin/_cctally_journal.py +2050 -156
- package/bin/_cctally_journal_repair.py +519 -0
- package/bin/_cctally_milestone_history.py +142 -56
- package/bin/_cctally_milestones.py +179 -111
- package/bin/_cctally_parser.py +42 -0
- package/bin/_cctally_project.py +24 -18
- package/bin/_cctally_quota.py +139 -25
- package/bin/_cctally_record.py +279 -108
- package/bin/_cctally_rederive.py +1052 -0
- package/bin/_cctally_reporting.py +58 -53
- package/bin/_cctally_setup.py +1 -0
- package/bin/_cctally_source_analytics.py +4 -1
- package/bin/_cctally_statusline.py +11 -11
- package/bin/_cctally_store.py +1039 -31
- package/bin/_cctally_sync_week.py +17 -8
- package/bin/_cctally_tui.py +421 -54
- package/bin/_cctally_update.py +133 -8
- package/bin/_cctally_weekrefs.py +14 -0
- package/bin/_lib_aggregators.py +10 -6
- package/bin/_lib_cache_report.py +101 -9
- package/bin/_lib_codex_pools.py +82 -0
- package/bin/_lib_conversation_query.py +126 -33
- package/bin/_lib_dashboard_sources.py +126 -1
- package/bin/_lib_diff_kernel.py +28 -15
- package/bin/_lib_doctor.py +342 -4
- package/bin/_lib_journal.py +924 -2
- package/bin/_lib_jsonl.py +43 -14
- package/bin/_lib_pricing.py +140 -21
- package/bin/_lib_readme_refresh.py +401 -0
- package/bin/_lib_rederive.py +395 -0
- package/bin/_lib_share.py +58 -2
- package/bin/cctally +56 -8
- package/dashboard/static/assets/{index-DJP4gEB7.js → index-3bgCMVHb.js} +52 -52
- package/dashboard/static/assets/index-D27EIHEI.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +6 -1
- package/dashboard/static/assets/index-Dk1nplOz.css +0 -1
package/bin/_cctally_core.py
CHANGED
|
@@ -11,6 +11,7 @@ and the only legal monkeypatch target for the 23 promoted globals listed
|
|
|
11
11
|
below. See docs/superpowers/specs/2026-05-22-cctally-core-data-globals.md.
|
|
12
12
|
"""
|
|
13
13
|
from __future__ import annotations
|
|
14
|
+
import contextvars
|
|
14
15
|
import datetime as dt
|
|
15
16
|
import math
|
|
16
17
|
import os
|
|
@@ -338,7 +339,17 @@ STATS_WAL_SIZE_LIMIT_BYTES = 16 * 1024 * 1024 # 16777216
|
|
|
338
339
|
# epoch-transition coordinator (``_cctally_journal.run_epoch_transition``):
|
|
339
340
|
# resolve the cutover identity, append the canonical cutover op, then rebuild
|
|
340
341
|
# account-scoped. See docs/superpowers/specs/2026-07-23-multi-account-design.md §2.
|
|
341
|
-
|
|
342
|
+
# 1001 -> 1002 (#372 Task A): the disposable index gains the effective-event
|
|
343
|
+
# revision summary consumed by correction planning/live replay. Rebuild selects
|
|
344
|
+
# the highest completed revision before folding; the legacy migration registry
|
|
345
|
+
# remains frozen.
|
|
346
|
+
# 1002 -> 1003 (#402 Task A): persist the selector's bounded structural
|
|
347
|
+
# correction-batch violations in the disposable index so shallow Dashboard/TUI
|
|
348
|
+
# Doctor gathers cannot report false health without rescanning the whole journal.
|
|
349
|
+
# 1003 -> 1004 (#410 Task B): pair the public journal cursor with the exact
|
|
350
|
+
# prefix atomically applied to the materialized index. A cursor-only hand edit
|
|
351
|
+
# can no longer skip an already-durable event and make its natural key look new.
|
|
352
|
+
STATS_INDEX_EPOCH = 1004
|
|
342
353
|
LEGACY_STATS_HEAD = 13
|
|
343
354
|
|
|
344
355
|
|
|
@@ -656,6 +667,93 @@ def ensure_dirs() -> None:
|
|
|
656
667
|
eprint(f"[core] could not chmod data dir 0700 ({exc}); continuing")
|
|
657
668
|
|
|
658
669
|
|
|
670
|
+
# === stats.db maintenance-hold tracking (#386) ======================
|
|
671
|
+
#
|
|
672
|
+
# `flock` conflicts are per open-file-DESCRIPTION and apply WITHIN a process:
|
|
673
|
+
# holding LOCK_EX on one fd and then requesting LOCK_SH on a second fd of the
|
|
674
|
+
# same file blocks forever. That matters because `_cctally_store`'s #386 opener
|
|
675
|
+
# protocol takes `stats.db.maintenance.lock` SHARED around every live stats
|
|
676
|
+
# open, while `run_stats_ingest`'s legacy/fresh branch already holds it
|
|
677
|
+
# EXCLUSIVE when it calls `open_db()` (bin/_cctally_journal.py:2788). Without a
|
|
678
|
+
# re-entrancy signal that is an unconditional self-deadlock on first open of a
|
|
679
|
+
# pre-cutover install.
|
|
680
|
+
#
|
|
681
|
+
# A ContextVar, not a module global, and deliberately so: the suppressor must
|
|
682
|
+
# fire only for the execution context that actually owns the lock. A dashboard
|
|
683
|
+
# thread that does NOT own it and requests SHARED while another thread holds
|
|
684
|
+
# EXCLUSIVE is correctly made to WAIT — that is the protocol working, not a
|
|
685
|
+
# deadlock — and a process-global flag would wrongly wave it straight through
|
|
686
|
+
# into a family being replaced underneath it.
|
|
687
|
+
#
|
|
688
|
+
# Every acquisition of STATS_LOCK_MAINTENANCE_PATH in bin/ that is HELD ACROSS
|
|
689
|
+
# other work pairs with these (a site that takes the flock and releases it
|
|
690
|
+
# before returning does not, and must not — see `stats_open_guarded`):
|
|
691
|
+
# bin/_cctally_journal.py _acquire_maintenance_{shared,exclusive} / _release
|
|
692
|
+
# bin/_cctally_store.py _heal_flock_blocking, reached through
|
|
693
|
+
# _acquire_stats_maintenance_reentrant by the heal
|
|
694
|
+
# hook and the epoch resolver
|
|
695
|
+
# bin/_cctally_db.py cmd_db_rebuild, _acquire_db_admin_writer_flocks
|
|
696
|
+
# (db skip / db unskip), _cmd_db_repair_exclusive,
|
|
697
|
+
# _vacuum_one_db
|
|
698
|
+
# bin/_cctally_rederive.py _rederive_locks
|
|
699
|
+
# Adding another acquisition site without noting it here reintroduces the hang.
|
|
700
|
+
#
|
|
701
|
+
# The opener (`_cctally_store.stats_open_guarded`) takes the lock SHARED and
|
|
702
|
+
# releases it before handing the connection back, so it deliberately does NOT
|
|
703
|
+
# note a hold — but it DOES consult `holds_stats_maintenance()` to skip the
|
|
704
|
+
# acquire entirely when this context already owns the exclusive side.
|
|
705
|
+
|
|
706
|
+
_STATS_MAINTENANCE_HELD = contextvars.ContextVar(
|
|
707
|
+
"cctally_stats_maintenance_held", default=0
|
|
708
|
+
)
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
def holds_stats_maintenance() -> bool:
|
|
712
|
+
"""True when THIS execution context already holds stats.db.maintenance.lock."""
|
|
713
|
+
return _STATS_MAINTENANCE_HELD.get() > 0
|
|
714
|
+
|
|
715
|
+
|
|
716
|
+
# === stats.db sanctioned-write scope (#386) =========================
|
|
717
|
+
#
|
|
718
|
+
# The state behind `_cctally_store.stats_write_scope` / `in_stats_write_scope`
|
|
719
|
+
# / `holds_ingest_lock`. It lives HERE, beside the maintenance tracker, rather
|
|
720
|
+
# than in `_cctally_store` for one concrete reason: `tests/conftest.py`'s
|
|
721
|
+
# `load_script()` drops every cached `_cctally_*` sibling from `sys.modules`
|
|
722
|
+
# (deliberately — see its docstring) but KEEPS `_cctally_core`. A ContextVar
|
|
723
|
+
# owned by `_cctally_store` would therefore be silently replaced by a fresh,
|
|
724
|
+
# empty one halfway through a test, so a scope entered before the reload would
|
|
725
|
+
# stop counting and an authorized write would be denied. `_cctally_core` is the
|
|
726
|
+
# kernel and is never reloaded, so the sanction survives.
|
|
727
|
+
#
|
|
728
|
+
# ContextVars, NOT module globals: the dashboard is threaded and a global would
|
|
729
|
+
# let one sanctioned thread authorize another (spec section 6.1).
|
|
730
|
+
|
|
731
|
+
_STATS_WRITE_SCOPE = contextvars.ContextVar(
|
|
732
|
+
"cctally_stats_write_scope", default=0
|
|
733
|
+
)
|
|
734
|
+
_STATS_INGEST_LOCK_HELD = contextvars.ContextVar(
|
|
735
|
+
"cctally_stats_ingest_held", default=0
|
|
736
|
+
)
|
|
737
|
+
_STATS_INTERRUPTED_RECOVERY_SUPPRESSED = contextvars.ContextVar(
|
|
738
|
+
"cctally_stats_interrupted_recovery_suppressed", default=0
|
|
739
|
+
)
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
def note_stats_maintenance_acquired() -> None:
|
|
743
|
+
"""Record that this context now holds stats.db.maintenance.lock."""
|
|
744
|
+
_STATS_MAINTENANCE_HELD.set(_STATS_MAINTENANCE_HELD.get() + 1)
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
def note_stats_maintenance_released() -> None:
|
|
748
|
+
"""Record that this context released stats.db.maintenance.lock.
|
|
749
|
+
|
|
750
|
+
Clamped at zero rather than asserting: an unbalanced release is a bug, but
|
|
751
|
+
turning it into an exception inside a ``finally`` would mask the original
|
|
752
|
+
failure that got us there.
|
|
753
|
+
"""
|
|
754
|
+
_STATS_MAINTENANCE_HELD.set(max(0, _STATS_MAINTENANCE_HELD.get() - 1))
|
|
755
|
+
|
|
756
|
+
|
|
659
757
|
# === Alerts validation cluster ======================================
|
|
660
758
|
|
|
661
759
|
|
|
@@ -1289,6 +1387,9 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
1289
1387
|
_STATS_MIGRATIONS = c._STATS_MIGRATIONS
|
|
1290
1388
|
_log_migration_error = c._log_migration_error
|
|
1291
1389
|
_clear_migration_error_log_entries = c._clear_migration_error_log_entries
|
|
1390
|
+
_reconcile_durable_applied_migration_errors = (
|
|
1391
|
+
c._reconcile_durable_applied_migration_errors
|
|
1392
|
+
)
|
|
1292
1393
|
# Unified opener policy (spec §6.1). Call-time import so the shared PRAGMA
|
|
1293
1394
|
# policy applies without a module-load cycle (_cctally_store imports this
|
|
1294
1395
|
# module). Routed through importlib.import_module rather than a bare
|
|
@@ -1303,16 +1404,16 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
1303
1404
|
import importlib
|
|
1304
1405
|
_cctally_store = importlib.import_module("_cctally_store")
|
|
1305
1406
|
|
|
1306
|
-
repair_marker = db_path.with_name("stats.db.repairing")
|
|
1307
|
-
if repair_marker.exists():
|
|
1308
|
-
raise c.StatsDbMaintenanceError()
|
|
1309
1407
|
ensure_dirs()
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1408
|
+
# #386: the opener half of the physical-replacement protocol. This replaces
|
|
1409
|
+
# three bare `repair_marker.exists()` checks around an unguarded connect —
|
|
1410
|
+
# which observed no quarantine-pending record and held no maintenance lock,
|
|
1411
|
+
# so a destructive maintenance path could publish its record, scan for
|
|
1412
|
+
# handles, and still be raced by an opener arriving before the first rename.
|
|
1413
|
+
# The guarded opener holds maintenance-SHARED across the marker/pending
|
|
1414
|
+
# checks AND the connect. Scratch (`_target_path`) opens keep the old
|
|
1415
|
+
# marker-only behaviour; see `_cctally_store.stats_open_guarded`.
|
|
1416
|
+
conn = _cctally_store.stats_open_guarded(db_path)
|
|
1316
1417
|
conn.row_factory = sqlite3.Row
|
|
1317
1418
|
# #279 S1 F4: probe connect + initial PRAGMAs so a corrupt stats.db (the
|
|
1318
1419
|
# non-re-derivable DB) surfaces as a one-line diagnosis + staged exit 3 instead of
|
|
@@ -1342,17 +1443,21 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
1342
1443
|
# POSITIVELY classified corruption of the REAL stats index (never a
|
|
1343
1444
|
# ``_target_path`` rebuild build — that path is disarmed to avoid
|
|
1344
1445
|
# recursion), hand off to the store's HEAL_HOOK: it re-checks under the
|
|
1345
|
-
# maintenance lock, writes the forensics bundle FIRST,
|
|
1346
|
-
#
|
|
1347
|
-
#
|
|
1348
|
-
# runs against the fresh index;
|
|
1349
|
-
# returns False (declined) for a
|
|
1350
|
-
# disk-full, permissions), the
|
|
1351
|
-
#
|
|
1352
|
-
#
|
|
1446
|
+
# maintenance lock, writes the forensics bundle FIRST, builds and
|
|
1447
|
+
# validates a fresh index while the damaged family remains in place,
|
|
1448
|
+
# then preserves that family and atomically publishes the replacement.
|
|
1449
|
+
# A single retry of the connect+probe then runs against the fresh index;
|
|
1450
|
+
# a second failure surfaces loudly. HEAL returns False (declined) for a
|
|
1451
|
+
# non-corruption ``DatabaseError`` (BUSY, disk-full, permissions), the
|
|
1452
|
+
# dev-checkout-on-prod guard, or when a concurrent healer already fixed
|
|
1453
|
+
# it — all of which fall through to the original guided
|
|
1454
|
+
# StatsDbCorruptError so the manual path still applies.
|
|
1353
1455
|
heal = getattr(_cctally_store, "HEAL_HOOK", None)
|
|
1354
1456
|
if _target_path is None and heal is not None and heal("stats", exc):
|
|
1355
|
-
|
|
1457
|
+
# #386: the post-heal retry is an opener too. The heal released the
|
|
1458
|
+
# maintenance lock before returning, so another maintenance path can
|
|
1459
|
+
# legitimately own the family by now.
|
|
1460
|
+
conn = _cctally_store.stats_open_guarded(db_path)
|
|
1356
1461
|
conn.row_factory = sqlite3.Row
|
|
1357
1462
|
try:
|
|
1358
1463
|
_cctally_store.apply_policy(conn, "stats")
|
|
@@ -1390,6 +1495,12 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
1390
1495
|
# STEADY STATE — zero schema work (spec §6.2/§7.1). Applies to a
|
|
1391
1496
|
# ``_target_path`` open too: a rebuilt/scratch index is stamped at the
|
|
1392
1497
|
# epoch, so opening it must be a pure connect + PRAGMA + version read.
|
|
1498
|
+
# Only the live path owns the live migration-error sentinel: a
|
|
1499
|
+
# scratch rebuild must not clear it before validated publication.
|
|
1500
|
+
if _target_path is None:
|
|
1501
|
+
_reconcile_durable_applied_migration_errors(
|
|
1502
|
+
conn, _STATS_MIGRATIONS, "stats.db"
|
|
1503
|
+
)
|
|
1393
1504
|
return conn
|
|
1394
1505
|
if _target_path is None:
|
|
1395
1506
|
if _uv > LEGACY_STATS_HEAD:
|
|
@@ -1421,778 +1532,841 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
1421
1532
|
# surrounding schema apply (CREATE TABLE IF NOT EXISTS / add_column_if_missing
|
|
1422
1533
|
# / dispatcher) still runs every open — Task 9 folds THAT under the
|
|
1423
1534
|
# STATS_INDEX_EPOCH gate; this task only removes the recurring backfill cost.
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1535
|
+
# ── #386: the OPEN-TIME MUTATION REGIME (spec §3.1 second clause) ──
|
|
1536
|
+
# Everything below mutates stats.db: the full schema DDL, the quota
|
|
1537
|
+
# projection schema, the migration dispatcher, two backfills, the fixups
|
|
1538
|
+
# marker and the in-place cutover. Before #386 it ran under NO lock,
|
|
1539
|
+
# reachable from any of the 57 production `open_db` call sites — so two
|
|
1540
|
+
# commands racing a first open, an upgrade or a cutover both ran DDL on
|
|
1541
|
+
# the same file. The guard takes `stats.db.maintenance.lock` EXCLUSIVE
|
|
1542
|
+
# (re-entrant: the common case arrives from a caller that already holds
|
|
1543
|
+
# it) and enters the sanctioned write scope the authorizer checks.
|
|
1544
|
+
#
|
|
1545
|
+
# It is BELOW the epoch gate deliberately: the steady-state open returns
|
|
1546
|
+
# at `_uv == STATS_INDEX_EPOCH` above and never reaches here, so the hot
|
|
1547
|
+
# path takes no exclusive lock at all.
|
|
1548
|
+
with _cctally_store.stats_open_time_guard(live=_target_path is None):
|
|
1549
|
+
_fixups_current = _cctally_store.stats_open_fixups_current(conn)
|
|
1550
|
+
conn.execute(
|
|
1551
|
+
"""
|
|
1552
|
+
CREATE TABLE IF NOT EXISTS weekly_usage_snapshots (
|
|
1553
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1554
|
+
captured_at_utc TEXT NOT NULL,
|
|
1555
|
+
week_start_date TEXT NOT NULL,
|
|
1556
|
+
week_end_date TEXT NOT NULL,
|
|
1557
|
+
week_start_at TEXT,
|
|
1558
|
+
week_end_at TEXT,
|
|
1559
|
+
weekly_percent REAL NOT NULL,
|
|
1560
|
+
page_url TEXT,
|
|
1561
|
+
source TEXT NOT NULL DEFAULT 'userscript',
|
|
1562
|
+
payload_json TEXT NOT NULL,
|
|
1563
|
+
account_key TEXT NOT NULL DEFAULT 'unattributed'
|
|
1564
|
+
)
|
|
1565
|
+
"""
|
|
1439
1566
|
)
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
ON weekly_usage_snapshots(week_start_date, captured_at_utc DESC, id DESC)
|
|
1446
|
-
"""
|
|
1447
|
-
)
|
|
1448
|
-
conn.execute(
|
|
1449
|
-
"""
|
|
1450
|
-
CREATE TABLE IF NOT EXISTS weekly_cost_snapshots (
|
|
1451
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1452
|
-
captured_at_utc TEXT NOT NULL,
|
|
1453
|
-
week_start_date TEXT NOT NULL,
|
|
1454
|
-
week_end_date TEXT NOT NULL,
|
|
1455
|
-
week_start_at TEXT,
|
|
1456
|
-
week_end_at TEXT,
|
|
1457
|
-
range_start_iso TEXT,
|
|
1458
|
-
range_end_iso TEXT,
|
|
1459
|
-
cost_usd REAL NOT NULL,
|
|
1460
|
-
source TEXT NOT NULL DEFAULT 'cctally-range-cost',
|
|
1461
|
-
mode TEXT NOT NULL DEFAULT 'auto',
|
|
1462
|
-
project TEXT,
|
|
1463
|
-
account_key TEXT NOT NULL DEFAULT 'unattributed'
|
|
1567
|
+
conn.execute(
|
|
1568
|
+
"""
|
|
1569
|
+
CREATE INDEX IF NOT EXISTS idx_usage_week_time
|
|
1570
|
+
ON weekly_usage_snapshots(week_start_date, captured_at_utc DESC, id DESC)
|
|
1571
|
+
"""
|
|
1464
1572
|
)
|
|
1465
|
-
"""
|
|
1466
|
-
)
|
|
1467
|
-
conn.execute(
|
|
1468
|
-
"""
|
|
1469
|
-
CREATE INDEX IF NOT EXISTS idx_cost_week_time
|
|
1470
|
-
ON weekly_cost_snapshots(week_start_date, captured_at_utc DESC, id DESC)
|
|
1471
|
-
"""
|
|
1472
|
-
)
|
|
1473
|
-
|
|
1474
|
-
add_column_if_missing(conn, "weekly_usage_snapshots", "week_start_at", "TEXT")
|
|
1475
|
-
add_column_if_missing(conn, "weekly_usage_snapshots", "week_end_at", "TEXT")
|
|
1476
|
-
# account_key (#341): the account dimension rides the STATS_INDEX_EPOCH bump
|
|
1477
|
-
# (a fresh rebuild carries it via the CREATE TABLE above); this backstop keeps
|
|
1478
|
-
# an already-1001 DB that predates the column consistent. DEFAULT
|
|
1479
|
-
# 'unattributed' — every production writer passes the key explicitly (rev 4.1
|
|
1480
|
-
# defensive-backstop rule), enforced by the structural writer-audit test.
|
|
1481
|
-
add_column_if_missing(
|
|
1482
|
-
conn, "weekly_usage_snapshots", "account_key",
|
|
1483
|
-
"TEXT NOT NULL DEFAULT 'unattributed'")
|
|
1484
|
-
add_column_if_missing(conn, "weekly_usage_snapshots", "five_hour_percent", "REAL")
|
|
1485
|
-
add_column_if_missing(conn, "weekly_usage_snapshots", "five_hour_resets_at", "TEXT")
|
|
1486
|
-
# five_hour_window_key — canonical (10-min-floored epoch) key for
|
|
1487
|
-
# jitter-tolerant equality. Anthropic's status-line API jitters
|
|
1488
|
-
# rate_limits.5h.resets_at by ~seconds within the same physical 5h
|
|
1489
|
-
# window; joining on the raw ISO string treats each jittered fetch as
|
|
1490
|
-
# a new window, escaping the monotonic clamp at cmd_record_usage.
|
|
1491
|
-
# Backfill is RESUMABLE: Python's sqlite3 auto-commits DDL,
|
|
1492
|
-
# so a process killed mid-loop would leave the column added with NULL
|
|
1493
|
-
# keys for unprocessed rows. The gating below detects that partial
|
|
1494
|
-
# state on the next open_db() call (`five_hour_resets_at IS NOT NULL
|
|
1495
|
-
# AND five_hour_window_key IS NULL`) and completes the backfill, so
|
|
1496
|
-
# the original Bug B can't silently re-emerge for half-migrated rows.
|
|
1497
|
-
needs_5h_key_backfill = add_column_if_missing(
|
|
1498
|
-
conn, "weekly_usage_snapshots", "five_hour_window_key", "INTEGER"
|
|
1499
|
-
)
|
|
1500
|
-
# §6.2 backfill gate (Task 8): the resumable-partial probe + the backfill
|
|
1501
|
-
# loop are open-time backfill work — skipped once the fixups marker is
|
|
1502
|
-
# stamped. (The `add_column_if_missing` above is schema apply, Task 9's.)
|
|
1503
|
-
if not _fixups_current:
|
|
1504
|
-
if not needs_5h_key_backfill and conn.execute(
|
|
1505
|
-
"SELECT 1 FROM weekly_usage_snapshots "
|
|
1506
|
-
"WHERE five_hour_resets_at IS NOT NULL "
|
|
1507
|
-
" AND five_hour_window_key IS NULL "
|
|
1508
|
-
"LIMIT 1"
|
|
1509
|
-
).fetchone() is not None:
|
|
1510
|
-
needs_5h_key_backfill = True
|
|
1511
|
-
else:
|
|
1512
|
-
needs_5h_key_backfill = False
|
|
1513
|
-
|
|
1514
|
-
if needs_5h_key_backfill:
|
|
1515
|
-
backfill_rows = conn.execute(
|
|
1516
|
-
"SELECT id, five_hour_resets_at FROM weekly_usage_snapshots "
|
|
1517
|
-
"WHERE five_hour_resets_at IS NOT NULL "
|
|
1518
|
-
" AND five_hour_window_key IS NULL"
|
|
1519
|
-
).fetchall()
|
|
1520
|
-
for row in backfill_rows:
|
|
1521
|
-
try:
|
|
1522
|
-
iso = row[1]
|
|
1523
|
-
d = parse_iso_datetime(iso, "five_hour_resets_at backfill")
|
|
1524
|
-
epoch = int(d.timestamp())
|
|
1525
|
-
key = _canonical_5h_window_key(epoch)
|
|
1526
|
-
conn.execute(
|
|
1527
|
-
"UPDATE weekly_usage_snapshots "
|
|
1528
|
-
"SET five_hour_window_key = ? WHERE id = ?",
|
|
1529
|
-
(key, row[0]),
|
|
1530
|
-
)
|
|
1531
|
-
except (ValueError, TypeError) as exc:
|
|
1532
|
-
eprint(f"[migration] skipped row {row[0]}: {exc}")
|
|
1533
1573
|
conn.execute(
|
|
1534
|
-
"
|
|
1535
|
-
|
|
1574
|
+
"""
|
|
1575
|
+
CREATE TABLE IF NOT EXISTS weekly_cost_snapshots (
|
|
1576
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1577
|
+
captured_at_utc TEXT NOT NULL,
|
|
1578
|
+
week_start_date TEXT NOT NULL,
|
|
1579
|
+
week_end_date TEXT NOT NULL,
|
|
1580
|
+
week_start_at TEXT,
|
|
1581
|
+
week_end_at TEXT,
|
|
1582
|
+
range_start_iso TEXT,
|
|
1583
|
+
range_end_iso TEXT,
|
|
1584
|
+
cost_usd REAL NOT NULL,
|
|
1585
|
+
source TEXT NOT NULL DEFAULT 'cctally-range-cost',
|
|
1586
|
+
mode TEXT NOT NULL DEFAULT 'auto',
|
|
1587
|
+
project TEXT,
|
|
1588
|
+
account_key TEXT NOT NULL DEFAULT 'unattributed'
|
|
1589
|
+
)
|
|
1590
|
+
"""
|
|
1591
|
+
)
|
|
1592
|
+
conn.execute(
|
|
1593
|
+
"""
|
|
1594
|
+
CREATE INDEX IF NOT EXISTS idx_cost_week_time
|
|
1595
|
+
ON weekly_cost_snapshots(week_start_date, captured_at_utc DESC, id DESC)
|
|
1596
|
+
"""
|
|
1536
1597
|
)
|
|
1537
|
-
conn.commit()
|
|
1538
1598
|
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1599
|
+
add_column_if_missing(conn, "weekly_usage_snapshots", "week_start_at", "TEXT")
|
|
1600
|
+
add_column_if_missing(conn, "weekly_usage_snapshots", "week_end_at", "TEXT")
|
|
1601
|
+
# account_key (#341): the account dimension rides the STATS_INDEX_EPOCH bump
|
|
1602
|
+
# (a fresh rebuild carries it via the CREATE TABLE above); this backstop keeps
|
|
1603
|
+
# an already-1001 DB that predates the column consistent. DEFAULT
|
|
1604
|
+
# 'unattributed' — every production writer passes the key explicitly (rev 4.1
|
|
1605
|
+
# defensive-backstop rule), enforced by the structural writer-audit test.
|
|
1606
|
+
add_column_if_missing(
|
|
1607
|
+
conn, "weekly_usage_snapshots", "account_key",
|
|
1608
|
+
"TEXT NOT NULL DEFAULT 'unattributed'")
|
|
1609
|
+
add_column_if_missing(conn, "weekly_usage_snapshots", "five_hour_percent", "REAL")
|
|
1610
|
+
add_column_if_missing(conn, "weekly_usage_snapshots", "five_hour_resets_at", "TEXT")
|
|
1611
|
+
# five_hour_window_key — canonical (10-min-floored epoch) key for
|
|
1612
|
+
# jitter-tolerant equality. Anthropic's status-line API jitters
|
|
1613
|
+
# rate_limits.5h.resets_at by ~seconds within the same physical 5h
|
|
1614
|
+
# window; joining on the raw ISO string treats each jittered fetch as
|
|
1615
|
+
# a new window, escaping the monotonic clamp at cmd_record_usage.
|
|
1616
|
+
# Backfill is RESUMABLE: Python's sqlite3 auto-commits DDL,
|
|
1617
|
+
# so a process killed mid-loop would leave the column added with NULL
|
|
1618
|
+
# keys for unprocessed rows. The gating below detects that partial
|
|
1619
|
+
# state on the next open_db() call (`five_hour_resets_at IS NOT NULL
|
|
1620
|
+
# AND five_hour_window_key IS NULL`) and completes the backfill, so
|
|
1621
|
+
# the original Bug B can't silently re-emerge for half-migrated rows.
|
|
1622
|
+
needs_5h_key_backfill = add_column_if_missing(
|
|
1623
|
+
conn, "weekly_usage_snapshots", "five_hour_window_key", "INTEGER"
|
|
1624
|
+
)
|
|
1625
|
+
# §6.2 backfill gate (Task 8): the resumable-partial probe + the backfill
|
|
1626
|
+
# loop are open-time backfill work — skipped once the fixups marker is
|
|
1627
|
+
# stamped. (The `add_column_if_missing` above is schema apply, Task 9's.)
|
|
1628
|
+
if not _fixups_current:
|
|
1629
|
+
if not needs_5h_key_backfill and conn.execute(
|
|
1630
|
+
"SELECT 1 FROM weekly_usage_snapshots "
|
|
1631
|
+
"WHERE five_hour_resets_at IS NOT NULL "
|
|
1632
|
+
" AND five_hour_window_key IS NULL "
|
|
1633
|
+
"LIMIT 1"
|
|
1634
|
+
).fetchone() is not None:
|
|
1635
|
+
needs_5h_key_backfill = True
|
|
1636
|
+
else:
|
|
1637
|
+
needs_5h_key_backfill = False
|
|
1638
|
+
|
|
1639
|
+
if needs_5h_key_backfill:
|
|
1640
|
+
backfill_rows = conn.execute(
|
|
1641
|
+
"SELECT id, five_hour_resets_at FROM weekly_usage_snapshots "
|
|
1642
|
+
"WHERE five_hour_resets_at IS NOT NULL "
|
|
1643
|
+
" AND five_hour_window_key IS NULL"
|
|
1644
|
+
).fetchall()
|
|
1645
|
+
for row in backfill_rows:
|
|
1646
|
+
try:
|
|
1647
|
+
iso = row[1]
|
|
1648
|
+
d = parse_iso_datetime(iso, "five_hour_resets_at backfill")
|
|
1649
|
+
epoch = int(d.timestamp())
|
|
1650
|
+
key = _canonical_5h_window_key(epoch)
|
|
1651
|
+
conn.execute(
|
|
1652
|
+
"UPDATE weekly_usage_snapshots "
|
|
1653
|
+
"SET five_hour_window_key = ? WHERE id = ?",
|
|
1654
|
+
(key, row[0]),
|
|
1655
|
+
)
|
|
1656
|
+
except (ValueError, TypeError) as exc:
|
|
1657
|
+
eprint(f"[migration] skipped row {row[0]}: {exc}")
|
|
1658
|
+
conn.execute(
|
|
1659
|
+
"CREATE INDEX IF NOT EXISTS idx_weekly_usage_snapshots_5h_window_key "
|
|
1660
|
+
"ON weekly_usage_snapshots(five_hour_window_key)"
|
|
1661
|
+
)
|
|
1662
|
+
conn.commit()
|
|
1546
1663
|
|
|
1547
|
-
|
|
1548
|
-
"""
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
"""
|
|
1555
|
-
CREATE INDEX IF NOT EXISTS idx_cost_week_start_at_time
|
|
1556
|
-
ON weekly_cost_snapshots(week_start_at, captured_at_utc DESC, id DESC)
|
|
1557
|
-
"""
|
|
1558
|
-
)
|
|
1664
|
+
add_column_if_missing(conn, "weekly_cost_snapshots", "week_start_at", "TEXT")
|
|
1665
|
+
add_column_if_missing(conn, "weekly_cost_snapshots", "week_end_at", "TEXT")
|
|
1666
|
+
add_column_if_missing(conn, "weekly_cost_snapshots", "range_start_iso", "TEXT")
|
|
1667
|
+
add_column_if_missing(conn, "weekly_cost_snapshots", "range_end_iso", "TEXT")
|
|
1668
|
+
add_column_if_missing(
|
|
1669
|
+
conn, "weekly_cost_snapshots", "account_key",
|
|
1670
|
+
"TEXT NOT NULL DEFAULT 'unattributed'")
|
|
1559
1671
|
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
marginal_cost_usd REAL,
|
|
1572
|
-
usage_snapshot_id INTEGER NOT NULL,
|
|
1573
|
-
cost_snapshot_id INTEGER NOT NULL,
|
|
1574
|
-
reset_event_id INTEGER NOT NULL DEFAULT 0,
|
|
1575
|
-
account_key TEXT NOT NULL DEFAULT 'unattributed',
|
|
1576
|
-
UNIQUE(account_key, week_start_date, percent_threshold, reset_event_id)
|
|
1672
|
+
conn.execute(
|
|
1673
|
+
"""
|
|
1674
|
+
CREATE INDEX IF NOT EXISTS idx_usage_week_start_at_time
|
|
1675
|
+
ON weekly_usage_snapshots(week_start_at, captured_at_utc DESC, id DESC)
|
|
1676
|
+
"""
|
|
1677
|
+
)
|
|
1678
|
+
conn.execute(
|
|
1679
|
+
"""
|
|
1680
|
+
CREATE INDEX IF NOT EXISTS idx_cost_week_start_at_time
|
|
1681
|
+
ON weekly_cost_snapshots(week_start_at, captured_at_utc DESC, id DESC)
|
|
1682
|
+
"""
|
|
1577
1683
|
)
|
|
1578
|
-
"""
|
|
1579
|
-
)
|
|
1580
1684
|
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
# below, since the table doesn't exist yet at this point in `open_db()`.
|
|
1601
|
-
add_column_if_missing(conn, "percent_milestones", "alerted_at", "TEXT")
|
|
1602
|
-
|
|
1603
|
-
# Mid-week reset events: when Anthropic advances `rate_limits.seven_day.
|
|
1604
|
-
# resets_at` before the previously-declared reset actually fires (i.e.,
|
|
1605
|
-
# gives the user a fresh weekly window before the old one naturally
|
|
1606
|
-
# expired), we record one row here so display + cost layers can treat
|
|
1607
|
-
# the effective reset moment as the old week's end AND the new week's
|
|
1608
|
-
# start — preventing the API's -7d-derived new week from overlapping
|
|
1609
|
-
# the old week. Inserted by cmd_record_usage on detection; read by
|
|
1610
|
-
# _apply_reset_events_to_weekrefs and the cost live-recompute path.
|
|
1611
|
-
conn.execute(
|
|
1612
|
-
"""
|
|
1613
|
-
CREATE TABLE IF NOT EXISTS week_reset_events (
|
|
1614
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1615
|
-
detected_at_utc TEXT NOT NULL,
|
|
1616
|
-
old_week_end_at TEXT NOT NULL,
|
|
1617
|
-
new_week_end_at TEXT NOT NULL,
|
|
1618
|
-
effective_reset_at_utc TEXT NOT NULL,
|
|
1619
|
-
observed_pre_credit_pct REAL,
|
|
1620
|
-
account_key TEXT NOT NULL DEFAULT 'unattributed',
|
|
1621
|
-
UNIQUE(account_key, old_week_end_at, new_week_end_at)
|
|
1685
|
+
conn.execute(
|
|
1686
|
+
"""
|
|
1687
|
+
CREATE TABLE IF NOT EXISTS percent_milestones (
|
|
1688
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1689
|
+
captured_at_utc TEXT NOT NULL,
|
|
1690
|
+
week_start_date TEXT NOT NULL,
|
|
1691
|
+
week_end_date TEXT NOT NULL,
|
|
1692
|
+
week_start_at TEXT,
|
|
1693
|
+
week_end_at TEXT,
|
|
1694
|
+
percent_threshold INTEGER NOT NULL,
|
|
1695
|
+
cumulative_cost_usd REAL NOT NULL,
|
|
1696
|
+
marginal_cost_usd REAL,
|
|
1697
|
+
usage_snapshot_id INTEGER NOT NULL,
|
|
1698
|
+
cost_snapshot_id INTEGER NOT NULL,
|
|
1699
|
+
reset_event_id INTEGER NOT NULL DEFAULT 0,
|
|
1700
|
+
account_key TEXT NOT NULL DEFAULT 'unattributed',
|
|
1701
|
+
UNIQUE(account_key, week_start_date, percent_threshold, reset_event_id)
|
|
1702
|
+
)
|
|
1703
|
+
"""
|
|
1622
1704
|
)
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1705
|
+
|
|
1706
|
+
add_column_if_missing(conn, "percent_milestones", "five_hour_percent_at_crossing", "REAL")
|
|
1707
|
+
add_column_if_missing(
|
|
1708
|
+
conn, "percent_milestones", "account_key",
|
|
1709
|
+
"TEXT NOT NULL DEFAULT 'unattributed'")
|
|
1710
|
+
# reset_event_id: segment column added by migration 005. Fresh-install
|
|
1711
|
+
# DBs get it via the live CREATE TABLE above + the dispatcher
|
|
1712
|
+
# fast-stamps the migration. Existing pre-005 DBs trip the migration's
|
|
1713
|
+
# rename-recreate-copy idiom (handler in _cctally_db.py); the handler's
|
|
1714
|
+
# fast-path probe stamps the marker when the column is already present
|
|
1715
|
+
# (covers the corner case where a partially-upgraded DB has the column
|
|
1716
|
+
# but not the new UNIQUE — re-run is safe).
|
|
1717
|
+
|
|
1718
|
+
# alerted_at: populated by the alert-dispatch path when a milestone-INSERT
|
|
1719
|
+
# row's threshold matches the user's configured alerts.weekly_thresholds /
|
|
1720
|
+
# alerts.five_hour_thresholds (and alerts.enabled is true). NULL means
|
|
1721
|
+
# "alerts were disabled at the moment of crossing OR the threshold wasn't
|
|
1722
|
+
# in the configured list" — never "alert delivery failed" (dispatch is
|
|
1723
|
+
# best-effort and write-once forward-only). The matching ALTER for
|
|
1724
|
+
# `five_hour_milestones` lives right after that table's CREATE block
|
|
1725
|
+
# below, since the table doesn't exist yet at this point in `open_db()`.
|
|
1726
|
+
add_column_if_missing(conn, "percent_milestones", "alerted_at", "TEXT")
|
|
1727
|
+
|
|
1728
|
+
# Mid-week reset events: when Anthropic advances `rate_limits.seven_day.
|
|
1729
|
+
# resets_at` before the previously-declared reset actually fires (i.e.,
|
|
1730
|
+
# gives the user a fresh weekly window before the old one naturally
|
|
1731
|
+
# expired), we record one row here so display + cost layers can treat
|
|
1732
|
+
# the effective reset moment as the old week's end AND the new week's
|
|
1733
|
+
# start — preventing the API's -7d-derived new week from overlapping
|
|
1734
|
+
# the old week. Inserted by cmd_record_usage on detection; read by
|
|
1735
|
+
# _apply_reset_events_to_weekrefs and the cost live-recompute path.
|
|
1736
|
+
conn.execute(
|
|
1737
|
+
"""
|
|
1738
|
+
CREATE TABLE IF NOT EXISTS week_reset_events (
|
|
1739
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1740
|
+
detected_at_utc TEXT NOT NULL,
|
|
1741
|
+
old_week_end_at TEXT NOT NULL,
|
|
1742
|
+
new_week_end_at TEXT NOT NULL,
|
|
1743
|
+
effective_reset_at_utc TEXT NOT NULL,
|
|
1744
|
+
observed_pre_credit_pct REAL,
|
|
1745
|
+
account_key TEXT NOT NULL DEFAULT 'unattributed',
|
|
1746
|
+
UNIQUE(account_key, old_week_end_at, new_week_end_at)
|
|
1747
|
+
)
|
|
1748
|
+
"""
|
|
1664
1749
|
)
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
conn
|
|
1669
|
-
|
|
1750
|
+
add_column_if_missing(
|
|
1751
|
+
conn, "week_reset_events", "account_key",
|
|
1752
|
+
"TEXT NOT NULL DEFAULT 'unattributed'")
|
|
1753
|
+
_backfill_week_reset_events(conn)
|
|
1754
|
+
|
|
1755
|
+
# ── five_hour_reset_events (Anthropic-issued in-place 5h credits) ──
|
|
1756
|
+
# Parallel concept to ``week_reset_events`` for the 5h dimension; lives
|
|
1757
|
+
# adjacent in ``_apply_schema`` because the two carry the same kind of
|
|
1758
|
+
# signal at different cadences. Diverges from weekly in that the payload
|
|
1759
|
+
# is the *percent values* (prior + post) rather than boundary keys,
|
|
1760
|
+
# because the 5h variant has a stable ``five_hour_window_key`` and only
|
|
1761
|
+
# the percent moves. See spec
|
|
1762
|
+
# docs/superpowers/specs/2026-05-16-5h-in-place-credit-detection.md §3.1
|
|
1763
|
+
# for rationale.
|
|
1764
|
+
#
|
|
1765
|
+
# UNIQUE(five_hour_window_key, effective_reset_at_utc) — supports stacked
|
|
1766
|
+
# credits across DISTINCT 10-min slots inside one block (see spec §2.3
|
|
1767
|
+
# "Bounded stacked-credit resolution" for the cap statement: ~30 distinct
|
|
1768
|
+
# slots per 5h block when floor matches ``_canonical_5h_window_key``'s
|
|
1769
|
+
# 600-second floor; same-slot collisions silently absorbed by
|
|
1770
|
+
# INSERT OR IGNORE — an intentional cap, not a bug).
|
|
1771
|
+
#
|
|
1772
|
+
# No FK per CLAUDE.md gotcha: FKs in this codebase are documentation-only
|
|
1773
|
+
# (``PRAGMA foreign_keys`` not enabled). ``five_hour_window_key`` provides
|
|
1774
|
+
# the join key without a formal FK.
|
|
1775
|
+
#
|
|
1776
|
+
# No ``_backfill_five_hour_reset_events`` call follows (forward-only ship
|
|
1777
|
+
# per spec Q5; historical backfill deferred to a future issue).
|
|
1778
|
+
conn.execute(
|
|
1779
|
+
"""
|
|
1780
|
+
CREATE TABLE IF NOT EXISTS five_hour_reset_events (
|
|
1781
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1782
|
+
detected_at_utc TEXT NOT NULL,
|
|
1783
|
+
five_hour_window_key INTEGER NOT NULL,
|
|
1784
|
+
prior_percent REAL NOT NULL,
|
|
1785
|
+
post_percent REAL NOT NULL,
|
|
1786
|
+
effective_reset_at_utc TEXT NOT NULL,
|
|
1787
|
+
account_key TEXT NOT NULL DEFAULT 'unattributed',
|
|
1788
|
+
UNIQUE(account_key, five_hour_window_key, effective_reset_at_utc)
|
|
1789
|
+
)
|
|
1790
|
+
"""
|
|
1791
|
+
)
|
|
1792
|
+
add_column_if_missing(
|
|
1793
|
+
conn, "five_hour_reset_events", "account_key",
|
|
1794
|
+
"TEXT NOT NULL DEFAULT 'unattributed'")
|
|
1670
1795
|
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1796
|
+
# ── five_hour_blocks (rollup, one row per API-anchored 5h block) ──
|
|
1797
|
+
conn.execute(
|
|
1798
|
+
"""
|
|
1799
|
+
CREATE TABLE IF NOT EXISTS five_hour_blocks (
|
|
1800
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1801
|
+
five_hour_window_key INTEGER NOT NULL,
|
|
1802
|
+
five_hour_resets_at TEXT NOT NULL,
|
|
1803
|
+
block_start_at TEXT NOT NULL,
|
|
1804
|
+
first_observed_at_utc TEXT NOT NULL,
|
|
1805
|
+
last_observed_at_utc TEXT NOT NULL,
|
|
1806
|
+
final_five_hour_percent REAL NOT NULL,
|
|
1807
|
+
seven_day_pct_at_block_start REAL,
|
|
1808
|
+
seven_day_pct_at_block_end REAL,
|
|
1809
|
+
crossed_seven_day_reset INTEGER NOT NULL DEFAULT 0,
|
|
1810
|
+
total_input_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1811
|
+
total_output_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1812
|
+
total_cache_create_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1813
|
+
total_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1814
|
+
total_cost_usd REAL NOT NULL DEFAULT 0,
|
|
1815
|
+
is_closed INTEGER NOT NULL DEFAULT 0,
|
|
1816
|
+
created_at_utc TEXT NOT NULL,
|
|
1817
|
+
last_updated_at_utc TEXT NOT NULL,
|
|
1818
|
+
account_key TEXT NOT NULL DEFAULT 'unattributed',
|
|
1819
|
+
UNIQUE(account_key, five_hour_window_key)
|
|
1820
|
+
)
|
|
1821
|
+
"""
|
|
1822
|
+
)
|
|
1823
|
+
add_column_if_missing(
|
|
1824
|
+
conn, "five_hour_blocks", "account_key",
|
|
1825
|
+
"TEXT NOT NULL DEFAULT 'unattributed'")
|
|
1826
|
+
conn.execute(
|
|
1827
|
+
"""
|
|
1828
|
+
CREATE INDEX IF NOT EXISTS idx_five_hour_blocks_block_start
|
|
1829
|
+
ON five_hour_blocks(block_start_at DESC)
|
|
1830
|
+
"""
|
|
1695
1831
|
)
|
|
1696
|
-
"""
|
|
1697
|
-
)
|
|
1698
|
-
add_column_if_missing(
|
|
1699
|
-
conn, "five_hour_blocks", "account_key",
|
|
1700
|
-
"TEXT NOT NULL DEFAULT 'unattributed'")
|
|
1701
|
-
conn.execute(
|
|
1702
|
-
"""
|
|
1703
|
-
CREATE INDEX IF NOT EXISTS idx_five_hour_blocks_block_start
|
|
1704
|
-
ON five_hour_blocks(block_start_at DESC)
|
|
1705
|
-
"""
|
|
1706
|
-
)
|
|
1707
1832
|
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1833
|
+
# ── five_hour_milestones (per-percent crossings inside a 5h block) ──
|
|
1834
|
+
conn.execute(
|
|
1835
|
+
"""
|
|
1836
|
+
CREATE TABLE IF NOT EXISTS five_hour_milestones (
|
|
1837
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1838
|
+
block_id INTEGER NOT NULL,
|
|
1839
|
+
five_hour_window_key INTEGER NOT NULL,
|
|
1840
|
+
percent_threshold INTEGER NOT NULL,
|
|
1841
|
+
captured_at_utc TEXT NOT NULL,
|
|
1842
|
+
usage_snapshot_id INTEGER NOT NULL,
|
|
1843
|
+
block_input_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1844
|
+
block_output_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1845
|
+
block_cache_create_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1846
|
+
block_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1847
|
+
block_cost_usd REAL NOT NULL DEFAULT 0,
|
|
1848
|
+
marginal_cost_usd REAL,
|
|
1849
|
+
seven_day_pct_at_crossing REAL,
|
|
1850
|
+
reset_event_id INTEGER NOT NULL DEFAULT 0,
|
|
1851
|
+
account_key TEXT NOT NULL DEFAULT 'unattributed',
|
|
1852
|
+
UNIQUE(account_key, five_hour_window_key, percent_threshold, reset_event_id),
|
|
1853
|
+
FOREIGN KEY (block_id) REFERENCES five_hour_blocks(id)
|
|
1854
|
+
)
|
|
1855
|
+
"""
|
|
1856
|
+
)
|
|
1857
|
+
conn.execute(
|
|
1858
|
+
"""
|
|
1859
|
+
CREATE INDEX IF NOT EXISTS idx_five_hour_milestones_block
|
|
1860
|
+
ON five_hour_milestones(block_id)
|
|
1861
|
+
"""
|
|
1729
1862
|
)
|
|
1730
|
-
"""
|
|
1731
|
-
)
|
|
1732
|
-
conn.execute(
|
|
1733
|
-
"""
|
|
1734
|
-
CREATE INDEX IF NOT EXISTS idx_five_hour_milestones_block
|
|
1735
|
-
ON five_hour_milestones(block_id)
|
|
1736
|
-
"""
|
|
1737
|
-
)
|
|
1738
1863
|
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1864
|
+
# alerted_at: see the matching ALTER on `percent_milestones` above for
|
|
1865
|
+
# rationale. Same write-once forward-only semantics: the alert-dispatch
|
|
1866
|
+
# path stamps this column on milestone-INSERT rows whose threshold
|
|
1867
|
+
# matches the user's configured `alerts.five_hour_thresholds`. NULL =
|
|
1868
|
+
# "alerts disabled at moment of crossing OR threshold not configured"
|
|
1869
|
+
# — never "delivery failed".
|
|
1870
|
+
add_column_if_missing(conn, "five_hour_milestones", "alerted_at", "TEXT")
|
|
1871
|
+
add_column_if_missing(
|
|
1872
|
+
conn, "five_hour_milestones", "account_key",
|
|
1873
|
+
"TEXT NOT NULL DEFAULT 'unattributed'")
|
|
1874
|
+
|
|
1875
|
+
# reset_event_id: segment column added by migration 006. Fresh-install
|
|
1876
|
+
# DBs get it via the live CREATE TABLE above + the dispatcher fast-stamps
|
|
1877
|
+
# the migration marker (the live DDL must carry the column AND the 3-col
|
|
1878
|
+
# UNIQUE for fast-stamp to be safe — see spec §3.2). Existing pre-006
|
|
1879
|
+
# DBs trip the migration's rename-recreate-copy idiom (handler in
|
|
1880
|
+
# bin/_cctally_db.py); the handler's fast-path probe stamps the marker
|
|
1881
|
+
# when the column is already present (covers the corner case where a
|
|
1882
|
+
# partially-upgraded DB has the column but not the new UNIQUE — re-run
|
|
1883
|
+
# is safe). Mirrors weekly migration 005 / `percent_milestones`.
|
|
1884
|
+
|
|
1885
|
+
# ── five_hour_block_models (per-(block, model) rollup-child) ──
|
|
1886
|
+
# MUST be created BEFORE the parent-backfill gate below, because
|
|
1887
|
+
# _backfill_five_hour_blocks writes into this table on the fresh-install
|
|
1888
|
+
# path. UNIQUE keyed on (five_hour_window_key, model) — durable across
|
|
1889
|
+
# parent rebuilds. Live writes use DELETE WHERE five_hour_window_key = ?.
|
|
1890
|
+
conn.execute(
|
|
1891
|
+
"""
|
|
1892
|
+
CREATE TABLE IF NOT EXISTS five_hour_block_models (
|
|
1893
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1894
|
+
block_id INTEGER NOT NULL,
|
|
1895
|
+
five_hour_window_key INTEGER NOT NULL,
|
|
1896
|
+
model TEXT NOT NULL,
|
|
1897
|
+
input_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1898
|
+
output_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1899
|
+
cache_create_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1900
|
+
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1901
|
+
cost_usd REAL NOT NULL DEFAULT 0,
|
|
1902
|
+
entry_count INTEGER NOT NULL DEFAULT 0,
|
|
1903
|
+
account_key TEXT NOT NULL DEFAULT 'unattributed',
|
|
1904
|
+
UNIQUE(account_key, five_hour_window_key, model),
|
|
1905
|
+
FOREIGN KEY (block_id) REFERENCES five_hour_blocks(id)
|
|
1906
|
+
)
|
|
1907
|
+
"""
|
|
1908
|
+
)
|
|
1909
|
+
add_column_if_missing(
|
|
1910
|
+
conn, "five_hour_block_models", "account_key",
|
|
1911
|
+
"TEXT NOT NULL DEFAULT 'unattributed'")
|
|
1912
|
+
conn.execute(
|
|
1913
|
+
"""
|
|
1914
|
+
CREATE INDEX IF NOT EXISTS idx_five_hour_block_models_block
|
|
1915
|
+
ON five_hour_block_models(block_id)
|
|
1916
|
+
"""
|
|
1917
|
+
)
|
|
1918
|
+
conn.execute(
|
|
1919
|
+
"""
|
|
1920
|
+
CREATE INDEX IF NOT EXISTS idx_five_hour_block_models_window
|
|
1921
|
+
ON five_hour_block_models(five_hour_window_key)
|
|
1922
|
+
"""
|
|
1781
1923
|
)
|
|
1782
|
-
"""
|
|
1783
|
-
)
|
|
1784
|
-
add_column_if_missing(
|
|
1785
|
-
conn, "five_hour_block_models", "account_key",
|
|
1786
|
-
"TEXT NOT NULL DEFAULT 'unattributed'")
|
|
1787
|
-
conn.execute(
|
|
1788
|
-
"""
|
|
1789
|
-
CREATE INDEX IF NOT EXISTS idx_five_hour_block_models_block
|
|
1790
|
-
ON five_hour_block_models(block_id)
|
|
1791
|
-
"""
|
|
1792
|
-
)
|
|
1793
|
-
conn.execute(
|
|
1794
|
-
"""
|
|
1795
|
-
CREATE INDEX IF NOT EXISTS idx_five_hour_block_models_window
|
|
1796
|
-
ON five_hour_block_models(five_hour_window_key)
|
|
1797
|
-
"""
|
|
1798
|
-
)
|
|
1799
1924
|
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1925
|
+
# ── five_hour_block_projects (per-(block, project_path) rollup-child) ──
|
|
1926
|
+
# NULL session_files.project_path → '(unknown)' sentinel at write time,
|
|
1927
|
+
# keeping reconcile invariant SUM(child.cost) == parent.total intact.
|
|
1928
|
+
conn.execute(
|
|
1929
|
+
"""
|
|
1930
|
+
CREATE TABLE IF NOT EXISTS five_hour_block_projects (
|
|
1931
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1932
|
+
block_id INTEGER NOT NULL,
|
|
1933
|
+
five_hour_window_key INTEGER NOT NULL,
|
|
1934
|
+
project_path TEXT NOT NULL,
|
|
1935
|
+
input_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1936
|
+
output_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1937
|
+
cache_create_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1938
|
+
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
|
|
1939
|
+
cost_usd REAL NOT NULL DEFAULT 0,
|
|
1940
|
+
entry_count INTEGER NOT NULL DEFAULT 0,
|
|
1941
|
+
account_key TEXT NOT NULL DEFAULT 'unattributed',
|
|
1942
|
+
UNIQUE(account_key, five_hour_window_key, project_path),
|
|
1943
|
+
FOREIGN KEY (block_id) REFERENCES five_hour_blocks(id)
|
|
1944
|
+
)
|
|
1945
|
+
"""
|
|
1946
|
+
)
|
|
1947
|
+
add_column_if_missing(
|
|
1948
|
+
conn, "five_hour_block_projects", "account_key",
|
|
1949
|
+
"TEXT NOT NULL DEFAULT 'unattributed'")
|
|
1950
|
+
conn.execute(
|
|
1951
|
+
"""
|
|
1952
|
+
CREATE INDEX IF NOT EXISTS idx_five_hour_block_projects_block
|
|
1953
|
+
ON five_hour_block_projects(block_id)
|
|
1954
|
+
"""
|
|
1955
|
+
)
|
|
1956
|
+
conn.execute(
|
|
1957
|
+
"""
|
|
1958
|
+
CREATE INDEX IF NOT EXISTS idx_five_hour_block_projects_window
|
|
1959
|
+
ON five_hour_block_projects(five_hour_window_key)
|
|
1960
|
+
"""
|
|
1819
1961
|
)
|
|
1820
|
-
"""
|
|
1821
|
-
)
|
|
1822
|
-
add_column_if_missing(
|
|
1823
|
-
conn, "five_hour_block_projects", "account_key",
|
|
1824
|
-
"TEXT NOT NULL DEFAULT 'unattributed'")
|
|
1825
|
-
conn.execute(
|
|
1826
|
-
"""
|
|
1827
|
-
CREATE INDEX IF NOT EXISTS idx_five_hour_block_projects_block
|
|
1828
|
-
ON five_hour_block_projects(block_id)
|
|
1829
|
-
"""
|
|
1830
|
-
)
|
|
1831
|
-
conn.execute(
|
|
1832
|
-
"""
|
|
1833
|
-
CREATE INDEX IF NOT EXISTS idx_five_hour_block_projects_window
|
|
1834
|
-
ON five_hour_block_projects(five_hour_window_key)
|
|
1835
|
-
"""
|
|
1836
|
-
)
|
|
1837
1962
|
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1963
|
+
# ── budget_milestones (equiv-$ budget threshold crossings — issue #19) ──
|
|
1964
|
+
# Write-once, forward-only (the exact posture of `five_hour_milestones`). A
|
|
1965
|
+
# mid-week quota reset re-anchors `week_start_at` (see
|
|
1966
|
+
# `_resolve_current_budget_window`), so the new window naturally gets
|
|
1967
|
+
# fresh rows under UNIQUE(week_start_at, period, threshold) — no
|
|
1968
|
+
# `reset_event_id` segment column needed (unlike the percent/5h tables).
|
|
1969
|
+
# `week_start_at` stores the effective/re-anchored ISO string from the
|
|
1970
|
+
# resolver (`isoformat(timespec="seconds")`); the resolver's
|
|
1971
|
+
# `parse_iso_datetime` returns a HOST-LOCAL tz-aware datetime, so this
|
|
1972
|
+
# dedup key carries the host's UTC offset (e.g. `…T07:00:00-07:00`) —
|
|
1973
|
+
# host-consistent, NOT portable across hosts, same posture as
|
|
1974
|
+
# `five_hour_blocks.block_start_at`. Firing + reconcile + the dashboard
|
|
1975
|
+
# envelope all read/write the identical string on a given host, so the
|
|
1976
|
+
# UNIQUE dedup is exact. `alerted_at` is stamped BEFORE the osascript Popen
|
|
1977
|
+
# (set-then-dispatch invariant); NULL = "recorded without dispatch" (the
|
|
1978
|
+
# forward-only-from-set reconcile path) OR "not yet dispatched", never
|
|
1979
|
+
# "delivery failed".
|
|
1980
|
+
# Unified vendor-tagged table (#143): one row per (vendor, period_start_at,
|
|
1981
|
+
# period, threshold). `vendor` ∈ 'claude'|'codex'. `period_start_at` is the
|
|
1982
|
+
# resolved period-window start instant (subscription-week OR calendar
|
|
1983
|
+
# period-start). `period` is the configured period at crossing; NULL = pre-012
|
|
1984
|
+
# unknown. Owned by migration 012_unify_budget_milestones_vendor (merge of the
|
|
1985
|
+
# former budget_milestones + codex_budget_milestones). The Codex table is NO
|
|
1986
|
+
# LONGER live-created here — migration 012 drops it and this CREATE must not
|
|
1987
|
+
# resurrect it; migration 011 is hardened to skip it when absent (#143).
|
|
1988
|
+
conn.execute(
|
|
1989
|
+
"""
|
|
1990
|
+
CREATE TABLE IF NOT EXISTS budget_milestones (
|
|
1991
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1992
|
+
vendor TEXT NOT NULL,
|
|
1993
|
+
period_start_at TEXT NOT NULL,
|
|
1994
|
+
period TEXT,
|
|
1995
|
+
threshold INTEGER NOT NULL,
|
|
1996
|
+
budget_usd REAL NOT NULL,
|
|
1997
|
+
spent_usd REAL NOT NULL,
|
|
1998
|
+
consumption_pct REAL NOT NULL,
|
|
1999
|
+
crossed_at_utc TEXT NOT NULL,
|
|
2000
|
+
alerted_at TEXT,
|
|
2001
|
+
account_key TEXT NOT NULL DEFAULT '*',
|
|
2002
|
+
UNIQUE(vendor, account_key, period_start_at, period, threshold)
|
|
2003
|
+
)
|
|
2004
|
+
"""
|
|
1878
2005
|
)
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
2006
|
+
add_column_if_missing(
|
|
2007
|
+
conn, "budget_milestones", "account_key", "TEXT NOT NULL DEFAULT '*'")
|
|
2008
|
+
|
|
2009
|
+
# ── projected_milestones (week-average-pace projection crossings — #121) ──
|
|
2010
|
+
# Write-once, forward-only — same posture as `budget_milestones` (no
|
|
2011
|
+
# `reset_event_id` segment column). Two metrics share the table, keyed by
|
|
2012
|
+
# `metric` ('weekly_pct' | 'budget_usd'); a level fires once the
|
|
2013
|
+
# WEEK-AVERAGE projection (not the displayed high-end verdict) crosses
|
|
2014
|
+
# `threshold`. `denominator` snapshots the target AT crossing (target_usd
|
|
2015
|
+
# for budget_usd, 100.0 for weekly_pct) so the dashboard envelope renders
|
|
2016
|
+
# context "$312 of $300" / "102% of cap" from the ROW, not from live config
|
|
2017
|
+
# that may have changed since (Codex P0-4). A mid-week reset re-anchors
|
|
2018
|
+
# `week_start_at` (new window → fresh rows under the UNIQUE key), the
|
|
2019
|
+
# budget-pattern reset handling — hence NO `reset_event_id` column.
|
|
2020
|
+
# `alerted_at` is stamped BEFORE the osascript Popen (set-then-dispatch).
|
|
2021
|
+
# Schema owned by migration 011_budget_milestone_period_keys (the `period`
|
|
2022
|
+
# column + the period-inclusive UNIQUE; see _cctally_db.py). `period` is
|
|
2023
|
+
# NULL for pre-011 rows.
|
|
2024
|
+
conn.execute(
|
|
2025
|
+
"""
|
|
2026
|
+
CREATE TABLE IF NOT EXISTS projected_milestones (
|
|
2027
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
2028
|
+
week_start_at TEXT NOT NULL, -- period-start instant (subscription-week OR calendar period-start; back-compat name)
|
|
2029
|
+
period TEXT, -- configured period at crossing; NULL = pre-011 unknown (migration 011)
|
|
2030
|
+
metric TEXT NOT NULL, -- 'weekly_pct' | 'budget_usd' | 'codex_budget_usd'
|
|
2031
|
+
threshold INTEGER NOT NULL, -- 90 | 100
|
|
2032
|
+
projected_value REAL NOT NULL,
|
|
2033
|
+
denominator REAL NOT NULL, -- target_usd (budget / codex_budget) | 100.0 (weekly)
|
|
2034
|
+
crossed_at_utc TEXT NOT NULL,
|
|
2035
|
+
alerted_at TEXT,
|
|
2036
|
+
account_key TEXT NOT NULL DEFAULT '*', -- '*' for vendor-budget metrics; real account for weekly_pct (Task 3)
|
|
2037
|
+
UNIQUE(account_key, week_start_at, period, metric, threshold)
|
|
2038
|
+
)
|
|
2039
|
+
"""
|
|
1913
2040
|
)
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
2041
|
+
add_column_if_missing(
|
|
2042
|
+
conn, "projected_milestones", "account_key", "TEXT NOT NULL DEFAULT '*'")
|
|
2043
|
+
|
|
2044
|
+
# ── project_budget_milestones (per-project equiv-$ budget crossings) ──────
|
|
2045
|
+
# Plain CREATE TABLE IF NOT EXISTS, NO migration handler / backfill — the
|
|
2046
|
+
# same posture as `budget_milestones` / `projected_milestones` (write-once,
|
|
2047
|
+
# forward-only, framework-untracked). `project_key` is the NEW dimension in
|
|
2048
|
+
# the UNIQUE key: each project crosses each threshold once per week,
|
|
2049
|
+
# independently of every other project (issue #19 / #121, spec §5.1). It
|
|
2050
|
+
# stores the canonical git-root (`ProjectKey.bucket_path`), matched by string
|
|
2051
|
+
# equality against each session entry's resolved git-root. `budget_usd`
|
|
2052
|
+
# snapshots the project's target AT crossing time so the dashboard renders
|
|
2053
|
+
# "$26 of $25" from the ROW, not from live config that may have changed since
|
|
2054
|
+
# (the Codex P0-4 lesson, already baked into `budget_milestones` /
|
|
2055
|
+
# `projected_milestones`). A mid-week quota reset re-anchors `week_start_at`
|
|
2056
|
+
# (new window → fresh rows under the UNIQUE key) — budget-pattern reset
|
|
2057
|
+
# handling, hence NO `reset_event_id` segment column. `alerted_at` is stamped
|
|
2058
|
+
# BEFORE dispatch (set-then-dispatch invariant); NULL = "recorded without
|
|
2059
|
+
# dispatch" (forward-only-from-set reconcile) OR "not yet dispatched", never
|
|
2060
|
+
# "delivery failed". Lives BEFORE the migration dispatcher: a plain CREATE on
|
|
2061
|
+
# a framework-untracked table never touches `schema_migrations`, so the
|
|
2062
|
+
# dispatcher's fresh-install snapshot is unaffected.
|
|
2063
|
+
conn.execute(
|
|
2064
|
+
"""
|
|
2065
|
+
CREATE TABLE IF NOT EXISTS project_budget_milestones (
|
|
2066
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
2067
|
+
week_start_at TEXT NOT NULL,
|
|
2068
|
+
project_key TEXT NOT NULL, -- canonical git-root (bucket_path)
|
|
2069
|
+
threshold INTEGER NOT NULL,
|
|
2070
|
+
budget_usd REAL NOT NULL, -- project's target snapshotted AT crossing
|
|
2071
|
+
spent_usd REAL NOT NULL,
|
|
2072
|
+
consumption_pct REAL NOT NULL,
|
|
2073
|
+
crossed_at_utc TEXT NOT NULL,
|
|
2074
|
+
alerted_at TEXT,
|
|
2075
|
+
account_key TEXT NOT NULL DEFAULT '*', -- account-blind this epic (spec §6): always '*'
|
|
2076
|
+
UNIQUE(account_key, week_start_at, project_key, threshold)
|
|
2077
|
+
)
|
|
2078
|
+
"""
|
|
1952
2079
|
)
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
2080
|
+
add_column_if_missing(
|
|
2081
|
+
conn, "project_budget_milestones", "account_key",
|
|
2082
|
+
"TEXT NOT NULL DEFAULT '*'")
|
|
2083
|
+
|
|
2084
|
+
# In-place weekly partial-credit floor (issue #209, record-credit M2).
|
|
2085
|
+
# Plain CREATE TABLE IF NOT EXISTS, NO migration handler / NO user_version
|
|
2086
|
+
# bump — the same framework-untracked posture as `project_budget_milestones`
|
|
2087
|
+
# above. A `record-credit` invocation records a weekly credit (e.g.
|
|
2088
|
+
# 46% -> 31%) WITHOUT writing a `week_reset_events` row: a credit lowers the
|
|
2089
|
+
# current-7d clamp floor only and must NOT re-anchor the week window (the
|
|
2090
|
+
# `week_reset_events`-driven window-resolution code would otherwise show a
|
|
2091
|
+
# spurious "new week" and corrupt the forecast rate). `_reset_aware_floor`
|
|
2092
|
+
# (below) unions this table with `week_reset_events` so the four MAX-clamp
|
|
2093
|
+
# sites floor the current % to the post-credit value while the window stays
|
|
2094
|
+
# put. `effective_at_utc` is `floor_to_hour(at)` in UTC; `applied_at_utc` is
|
|
2095
|
+
# audit-only (kept out of goldens). Lives BEFORE the migration dispatcher: a
|
|
2096
|
+
# plain CREATE on a framework-untracked table never touches
|
|
2097
|
+
# `schema_migrations`, so the dispatcher's fresh-install snapshot is
|
|
2098
|
+
# unaffected. See docs/superpowers/specs/2026-06-19-record-credit-weekly-design.md §2/§4a.
|
|
2099
|
+
conn.execute(
|
|
2100
|
+
"""
|
|
2101
|
+
CREATE TABLE IF NOT EXISTS weekly_credit_floors (
|
|
2102
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
2103
|
+
week_start_date TEXT NOT NULL,
|
|
2104
|
+
effective_at_utc TEXT NOT NULL,
|
|
2105
|
+
observed_pre_credit_pct REAL NOT NULL,
|
|
2106
|
+
applied_at_utc TEXT NOT NULL,
|
|
2107
|
+
account_key TEXT NOT NULL DEFAULT 'unattributed',
|
|
2108
|
+
UNIQUE(account_key, week_start_date, effective_at_utc)
|
|
2109
|
+
)
|
|
2110
|
+
"""
|
|
1984
2111
|
)
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2112
|
+
add_column_if_missing(
|
|
2113
|
+
conn, "weekly_credit_floors", "account_key",
|
|
2114
|
+
"TEXT NOT NULL DEFAULT 'unattributed'")
|
|
2115
|
+
|
|
2116
|
+
# ── accounts registry (multi-account epic #341, spec §1/§2) ───────────────
|
|
2117
|
+
# Derived from the journal like all stats.db state: `account_observe` op
|
|
2118
|
+
# lines fold into rows here (via _apply_op_account_observe), `account_label`
|
|
2119
|
+
# ops set the user label. Framework-untracked (plain CREATE TABLE IF NOT
|
|
2120
|
+
# EXISTS, no migration / no user_version bump — same posture as
|
|
2121
|
+
# weekly_credit_floors above), so it never touches `schema_migrations` and
|
|
2122
|
+
# the stats-schema change rides the STATS_INDEX_EPOCH bump + rebuild, not a
|
|
2123
|
+
# new stats migration (the frozen-registry rule). `last_seen_utc` is derived
|
|
2124
|
+
# at fold time from the max `at` of any account-stamped line, NOT carried by
|
|
2125
|
+
# the observe record. `label_source` records provenance for the
|
|
2126
|
+
# user > switcher > auto precedence rule. A new empty table is byte-invisible
|
|
2127
|
+
# to every existing render, preserving the R8 byte-stability contract.
|
|
2128
|
+
conn.execute(
|
|
2129
|
+
"""
|
|
2130
|
+
CREATE TABLE IF NOT EXISTS accounts (
|
|
2131
|
+
account_key TEXT PRIMARY KEY,
|
|
2132
|
+
provider TEXT NOT NULL,
|
|
2133
|
+
natural_id TEXT,
|
|
2134
|
+
email TEXT,
|
|
2135
|
+
label TEXT,
|
|
2136
|
+
plan_type TEXT,
|
|
2137
|
+
label_source TEXT NOT NULL DEFAULT 'auto',
|
|
2138
|
+
first_seen_utc TEXT,
|
|
2139
|
+
last_seen_utc TEXT
|
|
2140
|
+
)
|
|
2141
|
+
"""
|
|
2015
2142
|
)
|
|
2016
|
-
"""
|
|
2017
|
-
)
|
|
2018
2143
|
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2144
|
+
# Stats migration 013 owns durable quota interpretation. Keep the current
|
|
2145
|
+
# schema in the fresh-install path before the dispatcher, exactly like the
|
|
2146
|
+
# existing live CREATE tables; its handler calls this same idempotent helper
|
|
2147
|
+
# for an older stats.db and the dispatcher central-stamps on clean return.
|
|
2148
|
+
# §6.2 backfill gate (Task 8): the quota-projection schema apply is one of
|
|
2149
|
+
# the three open-time backfills — skipped once the fixups marker is stamped
|
|
2150
|
+
# (the marker is set only AFTER this ran, so marker-present ⇒ tables present).
|
|
2151
|
+
# Fresh installs read False and apply it here (the dispatcher fast-stamps 013
|
|
2152
|
+
# without running its handler, so this open-time call is the sole creator).
|
|
2153
|
+
if not _fixups_current:
|
|
2154
|
+
_apply_quota_projection_schema(conn)
|
|
2155
|
+
|
|
2156
|
+
# Migration framework dispatcher. Replaces the prior inline gate stack
|
|
2157
|
+
# (has_blocks + _migration_done) with the framework's _run_pending_-
|
|
2158
|
+
# migrations entry point. See spec §2.3, §5.2 + the migration handlers
|
|
2159
|
+
# decorated with @stats_migration further down in this file.
|
|
2160
|
+
#
|
|
2161
|
+
# MUST run BEFORE any DDL or write that touches `schema_migrations`
|
|
2162
|
+
# (Codex P1 #1 fix on c3625ee + e7fdcc8): the dispatcher's fresh-install
|
|
2163
|
+
# detection snapshots `schema_migrations`'s existence in sqlite_master
|
|
2164
|
+
# BEFORE its own CREATE TABLE IF NOT EXISTS. Pre-creating the table
|
|
2165
|
+
# earlier in open_db() (or letting `_backfill_five_hour_blocks` insert
|
|
2166
|
+
# markers first) flips that snapshot to True on a brand-new DB and
|
|
2167
|
+
# dead-codes the stamp-only fast path. The dispatcher is now the sole
|
|
2168
|
+
# creator of `schema_migrations` + `schema_migrations_skipped`.
|
|
2169
|
+
_run_pending_migrations(
|
|
2170
|
+
conn, registry=_STATS_MIGRATIONS, db_label="stats.db",
|
|
2171
|
+
)
|
|
2047
2172
|
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2173
|
+
# One-time historical backfill of five_hour_blocks (rollup only;
|
|
2174
|
+
# milestones are forward-only per spec §4.3 / [Write-once milestones]).
|
|
2175
|
+
# Idempotent via UNIQUE(five_hour_window_key) + INSERT OR IGNORE.
|
|
2176
|
+
# Runs AFTER the dispatcher so `schema_migrations` exists for the
|
|
2177
|
+
# marker INSERTs inside the backfill body, and so any fresh-install
|
|
2178
|
+
# stamp-only path the dispatcher took above is already committed.
|
|
2179
|
+
# §6.2 backfill gate (Task 8): the two probe SELECTs + the backfill + its
|
|
2180
|
+
# migration-003 re-invocation are open-time backfill work — skipped once the
|
|
2181
|
+
# fixups marker is stamped. Dead in the journal world (the ingest cycle writes
|
|
2182
|
+
# blocks with their snapshots), so it fires only on a pre-journal upgrade DB's
|
|
2183
|
+
# first open; after that the marker gates it out permanently.
|
|
2184
|
+
if not _fixups_current:
|
|
2185
|
+
existing = conn.execute(
|
|
2186
|
+
"SELECT 1 FROM five_hour_blocks LIMIT 1"
|
|
2187
|
+
).fetchone()
|
|
2188
|
+
has_snapshots = conn.execute(
|
|
2189
|
+
"SELECT 1 FROM weekly_usage_snapshots "
|
|
2190
|
+
"WHERE five_hour_window_key IS NOT NULL "
|
|
2191
|
+
" AND five_hour_percent IS NOT NULL "
|
|
2192
|
+
"LIMIT 1"
|
|
2193
|
+
).fetchone()
|
|
2194
|
+
if not existing and has_snapshots:
|
|
2195
|
+
inserted = _backfill_five_hour_blocks(conn)
|
|
2196
|
+
# Re-run the 5h dedup migration AFTER backfill creates parents.
|
|
2197
|
+
# The dispatcher above ran while five_hour_blocks was empty, so
|
|
2198
|
+
# the dedup handler no-op'd and stamped its marker. Snapshot
|
|
2199
|
+
# keys can carry jitter beyond the 600s canonical floor (the
|
|
2200
|
+
# 003_* migration handles up to 1800s grouping), so the
|
|
2201
|
+
# backfill's `DISTINCT five_hour_window_key` over those keys
|
|
2202
|
+
# can produce duplicate parent rows for one physical 5h
|
|
2203
|
+
# window. Without this re-invocation those duplicates persist
|
|
2204
|
+
# forever — the marker says it ran. Handler owns its own
|
|
2205
|
+
# BEGIN/COMMIT and is idempotent (no groups → no-op).
|
|
2206
|
+
#
|
|
2207
|
+
# Honor `db skip` here as well: if the operator marked 003 as
|
|
2208
|
+
# skipped (e.g., poison pill on their machine), we must NOT
|
|
2209
|
+
# back-door run the handler. Duplicates introduced by the
|
|
2210
|
+
# backfill will persist until they `db unskip` — which is the
|
|
2211
|
+
# explicit choice the skip records. Failure path mirrors the
|
|
2212
|
+
# dispatcher's contract: route through _log_migration_error so
|
|
2213
|
+
# the next interactive command renders the banner, and clear
|
|
2214
|
+
# the log entry on success so the banner auto-dismisses.
|
|
2215
|
+
if inserted > 0:
|
|
2216
|
+
target_name = "003_merge_5h_block_duplicates_v1"
|
|
2217
|
+
try:
|
|
2218
|
+
skipped = {
|
|
2219
|
+
row[0] for row in conn.execute(
|
|
2220
|
+
"SELECT name FROM schema_migrations_skipped"
|
|
2221
|
+
).fetchall()
|
|
2222
|
+
}
|
|
2223
|
+
except sqlite3.OperationalError:
|
|
2224
|
+
skipped = set()
|
|
2225
|
+
if target_name not in skipped:
|
|
2226
|
+
for _m in _STATS_MIGRATIONS:
|
|
2227
|
+
if _m.name == target_name:
|
|
2228
|
+
qualified = f"stats.db:{target_name}"
|
|
2229
|
+
try:
|
|
2230
|
+
_m.handler(conn)
|
|
2231
|
+
_clear_migration_error_log_entries(qualified)
|
|
2232
|
+
except Exception as exc:
|
|
2233
|
+
_log_migration_error(
|
|
2234
|
+
name=qualified,
|
|
2235
|
+
exc=exc,
|
|
2236
|
+
tb=traceback.format_exc(),
|
|
2237
|
+
)
|
|
2238
|
+
eprint(f"[migration {qualified}] failed: {exc}")
|
|
2239
|
+
break
|
|
2240
|
+
|
|
2241
|
+
# ── Append-only journal replay-identity columns + ingest cursor ──
|
|
2242
|
+
# (2026-07-22 DB journal redesign, spec §4.2 / §5.2). Every row a
|
|
2243
|
+
# journal fold materializes carries the originating line's stable `id`
|
|
2244
|
+
# in `journal_id`, with a partial UNIQUE index so the ingester's
|
|
2245
|
+
# INSERT OR IGNORE fold is idempotent under replay/re-ingest. Runs AFTER
|
|
2246
|
+
# the migration dispatcher so the columns land on the final (migrated)
|
|
2247
|
+
# table shape — migrations 005/006 recreate percent/5h milestone tables
|
|
2248
|
+
# and must not drop the column. All additive (add_column_if_missing /
|
|
2249
|
+
# CREATE ... IF NOT EXISTS), framework-untracked — same posture as
|
|
2250
|
+
# weekly_credit_floors / project_budget_milestones. Task 9 folds this
|
|
2251
|
+
# under the STATS_INDEX_EPOCH version gate; until then it is idempotent
|
|
2252
|
+
# per open (add_column_if_missing / IF NOT EXISTS no-op once present).
|
|
2253
|
+
for _jtable in (
|
|
2254
|
+
"weekly_usage_snapshots", "weekly_cost_snapshots", "week_reset_events",
|
|
2255
|
+
"five_hour_reset_events", "five_hour_blocks", "weekly_credit_floors",
|
|
2256
|
+
"percent_milestones", "five_hour_milestones", "budget_milestones",
|
|
2257
|
+
"projected_milestones", "project_budget_milestones",
|
|
2258
|
+
):
|
|
2259
|
+
add_column_if_missing(conn, _jtable, "journal_id", "TEXT")
|
|
2260
|
+
conn.execute(
|
|
2261
|
+
f"CREATE UNIQUE INDEX IF NOT EXISTS idx_{_jtable}_journal_id "
|
|
2262
|
+
f"ON {_jtable}(journal_id) WHERE journal_id IS NOT NULL"
|
|
2263
|
+
)
|
|
2264
|
+
# Companion partial index (spec §5.3 harvest / Task 6 gate P2): the ingest
|
|
2265
|
+
# cycle's harvest scans every natural-keyed family for rows the pipeline
|
|
2266
|
+
# inserted this cycle (`WHERE journal_id IS NULL`). A partial index over
|
|
2267
|
+
# exactly those un-stamped rows keeps that scan O(this-cycle inserts), not
|
|
2268
|
+
# O(table) — at the 10x envelope the stamped rows are ~all of the table, so
|
|
2269
|
+
# a full scan would be pathological. Only the 8 HARVEST families need it
|
|
2270
|
+
# (the Model-A / op-fold tables — weekly_usage_snapshots, weekly_cost_
|
|
2271
|
+
# snapshots, weekly_credit_floors — are never harvest-scanned).
|
|
2272
|
+
for _htable in (
|
|
2273
|
+
"week_reset_events", "five_hour_reset_events", "five_hour_blocks",
|
|
2274
|
+
"percent_milestones", "five_hour_milestones", "budget_milestones",
|
|
2275
|
+
"projected_milestones", "project_budget_milestones",
|
|
2276
|
+
):
|
|
2277
|
+
conn.execute(
|
|
2278
|
+
f"CREATE INDEX IF NOT EXISTS idx_{_htable}_journal_id_null "
|
|
2279
|
+
f"ON {_htable}(id) WHERE journal_id IS NULL"
|
|
2280
|
+
)
|
|
2281
|
+
# Single-row segment+offset consumption watermark (spec §5.2). The
|
|
2282
|
+
# applied_* pair is written atomically with the materialized rows and
|
|
2283
|
+
# acts as the trusted prefix when a cursor-only hand edit advances the
|
|
2284
|
+
# public pair without applying the skipped journal bytes (#410 Task B).
|
|
2285
|
+
conn.execute(
|
|
2286
|
+
"CREATE TABLE IF NOT EXISTS journal_cursor ("
|
|
2287
|
+
"id INTEGER PRIMARY KEY CHECK (id = 1), "
|
|
2288
|
+
"segment TEXT NOT NULL, "
|
|
2289
|
+
"offset INTEGER NOT NULL, "
|
|
2290
|
+
"applied_segment TEXT, "
|
|
2291
|
+
"applied_offset INTEGER)"
|
|
2292
|
+
)
|
|
2293
|
+
add_column_if_missing(
|
|
2294
|
+
conn, "journal_cursor", "applied_segment", "TEXT"
|
|
2295
|
+
)
|
|
2296
|
+
add_column_if_missing(
|
|
2297
|
+
conn, "journal_cursor", "applied_offset", "INTEGER"
|
|
2298
|
+
)
|
|
2299
|
+
# Schema-apply compatibility for legacy/test-mode paths that reach this
|
|
2300
|
+
# DDL with a pre-pair cursor row. A released epoch-1003 index does NOT
|
|
2301
|
+
# use this as an upgrade shortcut: the epoch mismatch rebuilds it into
|
|
2302
|
+
# the complete epoch-1004 schema first.
|
|
2135
2303
|
conn.execute(
|
|
2136
|
-
|
|
2137
|
-
|
|
2304
|
+
"UPDATE journal_cursor "
|
|
2305
|
+
"SET applied_segment = segment, applied_offset = offset "
|
|
2306
|
+
"WHERE applied_segment IS NULL AND applied_offset IS NULL"
|
|
2138
2307
|
)
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
# O(table) — at the 10x envelope the stamped rows are ~all of the table, so
|
|
2144
|
-
# a full scan would be pathological. Only the 8 HARVEST families need it
|
|
2145
|
-
# (the Model-A / op-fold tables — weekly_usage_snapshots, weekly_cost_
|
|
2146
|
-
# snapshots, weekly_credit_floors — are never harvest-scanned).
|
|
2147
|
-
for _htable in (
|
|
2148
|
-
"week_reset_events", "five_hour_reset_events", "five_hour_blocks",
|
|
2149
|
-
"percent_milestones", "five_hour_milestones", "budget_milestones",
|
|
2150
|
-
"projected_milestones", "project_budget_milestones",
|
|
2151
|
-
):
|
|
2308
|
+
# Disposable effective-event summary (#372 Task A). Durable truth remains
|
|
2309
|
+
# the append-only journal; rebuild repopulates this table from the shared
|
|
2310
|
+
# pure selector. The table lets live replay detect completed corrections
|
|
2311
|
+
# without inventing family-specific inverse operations.
|
|
2152
2312
|
conn.execute(
|
|
2153
|
-
|
|
2154
|
-
|
|
2313
|
+
"CREATE TABLE IF NOT EXISTS journal_effective_events ("
|
|
2314
|
+
"event_id TEXT PRIMARY KEY, "
|
|
2315
|
+
"rev INTEGER NOT NULL CHECK (rev >= 0), "
|
|
2316
|
+
"status TEXT NOT NULL CHECK (status IN ('active','tombstone')), "
|
|
2317
|
+
"content_hash TEXT NOT NULL, "
|
|
2318
|
+
"batch_id TEXT, "
|
|
2319
|
+
"event_json TEXT)"
|
|
2320
|
+
)
|
|
2321
|
+
# Disposable selector diagnostics (#402 Task A). The append-only journal
|
|
2322
|
+
# remains authoritative; rebuild/live preflight replace this bounded
|
|
2323
|
+
# summary after a complete correction-prefix selection.
|
|
2324
|
+
conn.execute(
|
|
2325
|
+
"CREATE TABLE IF NOT EXISTS journal_protocol_violations ("
|
|
2326
|
+
"fingerprint TEXT PRIMARY KEY, "
|
|
2327
|
+
"batch_id TEXT NOT NULL, "
|
|
2328
|
+
"kind TEXT NOT NULL, "
|
|
2329
|
+
"violation_json TEXT NOT NULL)"
|
|
2155
2330
|
)
|
|
2156
|
-
# Single-row segment+offset consumption watermark (spec §5.2). The cursor
|
|
2157
|
-
# never advances past a byte range the ingest cycle did not read.
|
|
2158
|
-
conn.execute(
|
|
2159
|
-
"CREATE TABLE IF NOT EXISTS journal_cursor ("
|
|
2160
|
-
"id INTEGER PRIMARY KEY CHECK (id = 1), "
|
|
2161
|
-
"segment TEXT NOT NULL, "
|
|
2162
|
-
"offset INTEGER NOT NULL)"
|
|
2163
|
-
)
|
|
2164
2331
|
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2332
|
+
# §6.2 backfill gate (Task 8): stamp the one-shot marker AFTER the three
|
|
2333
|
+
# open-time backfills ran, so the next open skips them (and their probes)
|
|
2334
|
+
# entirely. The marker table DDL runs ONLY on this "fixups ran" path, never
|
|
2335
|
+
# on the steady-state open. A crash before this commit leaves the marker
|
|
2336
|
+
# unset and everything re-runs idempotently next open (invariant).
|
|
2337
|
+
if not _fixups_current:
|
|
2338
|
+
_cctally_store.mark_stats_open_fixups_done(conn)
|
|
2339
|
+
|
|
2340
|
+
conn.commit()
|
|
2341
|
+
|
|
2342
|
+
# ── Epoch stamp / in-place cutover (DB journal redesign §7.1/§8) ──
|
|
2343
|
+
# The full schema is now applied (head 13 + journal_id columns + cursor). A
|
|
2344
|
+
# ``_target_path`` build (rebuild scratch) stamps the epoch DIRECTLY (no
|
|
2345
|
+
# export — the rebuild folds the journal itself). A real legacy install runs
|
|
2346
|
+
# the cutover: export history to a bootstrap segment, stamp ``journal_id`` on
|
|
2347
|
+
# every row, advance the cursor, and stamp the epoch — all atomic (spec §8).
|
|
2348
|
+
# Under test mode (epoch disabled) neither runs, so the DB stays at
|
|
2349
|
+
# len(registry) for the migration-framework harness.
|
|
2350
|
+
if _epoch_engaged:
|
|
2351
|
+
if _target_path is not None:
|
|
2352
|
+
conn.execute(f"PRAGMA user_version = {STATS_INDEX_EPOCH}")
|
|
2353
|
+
conn.commit()
|
|
2354
|
+
elif conn.execute("PRAGMA user_version").fetchone()[0] == LEGACY_STATS_HEAD:
|
|
2355
|
+
# Cut over ONLY once the legacy dispatcher reached the export baseline
|
|
2356
|
+
# (head 13). A DEFERRED migration (MigrationGateNotMet — e.g. the
|
|
2357
|
+
# 008/009/010 recompute gate) leaves user_version < 13; skip the
|
|
2358
|
+
# cutover so the next open retries the dispatcher first, then cuts over
|
|
2359
|
+
# (spec §8 step 1: "run any pending legacy stats migrations, reaching
|
|
2360
|
+
# the export baseline"). Never journal a pre-recompute stats shape.
|
|
2361
|
+
try:
|
|
2362
|
+
importlib.import_module("_cctally_journal").run_cutover(conn)
|
|
2363
|
+
except BaseException:
|
|
2364
|
+
try:
|
|
2365
|
+
conn.close()
|
|
2366
|
+
except Exception:
|
|
2367
|
+
pass
|
|
2368
|
+
raise
|
|
2369
|
+
return conn
|
|
2196
2370
|
|
|
2197
2371
|
|
|
2198
2372
|
# === WeekRef cluster ================================================
|