cctally 1.80.4 → 1.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/bin/_cctally_account.py +397 -0
- package/bin/_cctally_alerts.py +58 -2
- package/bin/_cctally_cache.py +696 -166
- package/bin/_cctally_cache_report.py +21 -3
- package/bin/_cctally_config.py +119 -8
- package/bin/_cctally_core.py +343 -48
- package/bin/_cctally_dashboard.py +32 -7
- package/bin/_cctally_dashboard_conversation.py +6 -2
- package/bin/_cctally_dashboard_envelope.py +49 -0
- package/bin/_cctally_dashboard_share.py +78 -1
- package/bin/_cctally_dashboard_sources.py +434 -48
- package/bin/_cctally_db.py +659 -152
- package/bin/_cctally_diff.py +9 -0
- package/bin/_cctally_doctor.py +201 -26
- package/bin/_cctally_five_hour.py +110 -69
- package/bin/_cctally_forecast.py +112 -43
- package/bin/_cctally_journal.py +591 -80
- package/bin/_cctally_milestones.py +180 -67
- package/bin/_cctally_parser.py +87 -0
- package/bin/_cctally_percent_breakdown.py +24 -15
- package/bin/_cctally_project.py +12 -1
- package/bin/_cctally_quota.py +150 -39
- package/bin/_cctally_record.py +491 -141
- package/bin/_cctally_reporting.py +66 -14
- package/bin/_cctally_setup.py +9 -1
- package/bin/_cctally_source_analytics.py +56 -7
- package/bin/_cctally_statusline.py +55 -8
- package/bin/_cctally_store.py +20 -8
- package/bin/_cctally_sync_week.py +30 -10
- package/bin/_cctally_tui.py +99 -18
- package/bin/_cctally_weekrefs.py +38 -15
- package/bin/_lib_accounts.py +325 -0
- package/bin/_lib_alerts_payload.py +25 -4
- package/bin/_lib_cache_writer_lock.py +77 -0
- package/bin/_lib_codex_hooks.py +40 -1
- package/bin/_lib_diff_kernel.py +28 -14
- package/bin/_lib_doctor.py +160 -0
- package/bin/_lib_journal.py +65 -2
- package/bin/_lib_quota.py +81 -4
- package/bin/_lib_render.py +19 -5
- package/bin/_lib_share.py +25 -0
- package/bin/_lib_snapshot_cache.py +8 -0
- package/bin/_lib_subscription_weeks.py +16 -2
- package/bin/_lib_view_models.py +18 -9
- package/bin/cctally +43 -4
- package/dashboard/static/assets/index-DJP4gEB7.js +92 -0
- package/dashboard/static/assets/index-Dk1nplOz.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +4 -1
- package/dashboard/static/assets/index-BzWujBS3.js +0 -92
- package/dashboard/static/assets/index-Cc2ykqF_.css +0 -1
package/bin/_cctally_journal.py
CHANGED
|
@@ -27,6 +27,7 @@ from __future__ import annotations
|
|
|
27
27
|
|
|
28
28
|
import datetime as dt
|
|
29
29
|
import fcntl
|
|
30
|
+
import json
|
|
30
31
|
import os
|
|
31
32
|
import pathlib
|
|
32
33
|
import sqlite3
|
|
@@ -35,6 +36,7 @@ import time
|
|
|
35
36
|
from dataclasses import dataclass, field
|
|
36
37
|
|
|
37
38
|
import _cctally_core
|
|
39
|
+
import _lib_accounts
|
|
38
40
|
import _lib_journal
|
|
39
41
|
import _lib_record
|
|
40
42
|
|
|
@@ -408,8 +410,9 @@ def journal_high_water() -> tuple[str, int] | None:
|
|
|
408
410
|
# QUOTA_APPLIER the Codex quota cache leg (Task 7; wired to `_quota_applier`
|
|
409
411
|
# below). Contract: (decoded) -> stop_index | None. `decoded`
|
|
410
412
|
# is the ordered list of (record, segment, offset); a non-None
|
|
411
|
-
# int is a prefix-stop boundary (busy
|
|
412
|
-
# a quota line): the cycle processes
|
|
413
|
+
# int is a prefix-stop boundary (busy global or Codex cache
|
|
414
|
+
# writer flock at a quota line): the cycle processes
|
|
415
|
+
# decoded[:stop] and
|
|
413
416
|
# advances the cursor to decoded[stop]'s offset (spec §5.2
|
|
414
417
|
# step 3). Always-on: a Claude-only batch scans + returns None.
|
|
415
418
|
# codex_apply per-cycle `(ctx) -> None` closure (Task 7, a `run_stats_
|
|
@@ -690,10 +693,11 @@ def _read_range(cursor, hw) -> list[tuple[str, int, bytes]]:
|
|
|
690
693
|
# --------------------------------------------------------------------------
|
|
691
694
|
# cache leg — Codex quota obs -> cache.db quota_window_snapshots (spec §5.2
|
|
692
695
|
# step 3, Task 7 Item 2). Runs BEFORE the stats BEGIN IMMEDIATE, under the
|
|
693
|
-
# `cache.db.codex.lock`
|
|
694
|
-
# SQLite write transactions). The journal Codex quota obs are the
|
|
695
|
-
# (§1 latent data-loss hole — the source rollout JSONL
|
|
696
|
-
# re-materializes the disposable cache.db index from them,
|
|
696
|
+
# global cache writer flock followed by `cache.db.codex.lock` (lock-order law:
|
|
697
|
+
# flocks precede SQLite write transactions). The journal Codex quota obs are the
|
|
698
|
+
# DURABLE truth (§1 latent data-loss hole — the source rollout JSONL
|
|
699
|
+
# evaporates); this leg re-materializes the disposable cache.db index from them,
|
|
700
|
+
# idempotently
|
|
697
701
|
# (INSERT OR IGNORE on the natural key). Distinct from the direct cache write in
|
|
698
702
|
# `sync_codex_cache._write_codex_file_batch` (kept byte-identical, Item 1) — the
|
|
699
703
|
# two converge on the same rows.
|
|
@@ -706,8 +710,8 @@ _QUOTA_SNAPSHOT_INSERT = (
|
|
|
706
710
|
"(source, source_root_key, source_path, line_offset, captured_at_utc, "
|
|
707
711
|
" observed_slot, logical_limit_key, limit_id, limit_name, window_minutes, "
|
|
708
712
|
" used_percent, resets_at_utc, plan_type, individual_limit_json, "
|
|
709
|
-
" reached_type, observed_model) "
|
|
710
|
-
"VALUES (
|
|
713
|
+
" reached_type, observed_model, account_key) "
|
|
714
|
+
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
|
711
715
|
)
|
|
712
716
|
|
|
713
717
|
_QUOTA_SNAPSHOT_COLS = (
|
|
@@ -718,6 +722,17 @@ _QUOTA_SNAPSHOT_COLS = (
|
|
|
718
722
|
)
|
|
719
723
|
|
|
720
724
|
|
|
725
|
+
def _quota_snapshot_values(rec: dict) -> tuple:
|
|
726
|
+
"""Build the INSERT values tuple for one Codex quota obs. account_key (#341)
|
|
727
|
+
rides the obs TOP-LEVEL ``account`` field (obs stamp shape), not the payload,
|
|
728
|
+
so an unstamped/sentinel obs re-materializes cache.db with NULL account_key
|
|
729
|
+
(``NULL ≡ unattributed`` on the read path). first-stamp-wins via the
|
|
730
|
+
``INSERT OR IGNORE`` natural key — account_key is a stamped attribute, never
|
|
731
|
+
part of the identity."""
|
|
732
|
+
p = rec.get("payload") or {}
|
|
733
|
+
return tuple(p.get(col) for col in _QUOTA_SNAPSHOT_COLS) + (rec.get("account"),)
|
|
734
|
+
|
|
735
|
+
|
|
721
736
|
def _is_codex_quota_obs(rec: dict) -> bool:
|
|
722
737
|
return (
|
|
723
738
|
rec.get("t") == "obs"
|
|
@@ -728,12 +743,13 @@ def _is_codex_quota_obs(rec: dict) -> bool:
|
|
|
728
743
|
|
|
729
744
|
def _quota_applier(decoded) -> int | None:
|
|
730
745
|
"""Cache leg (spec §5.2 step 3): materialize this batch's Codex quota obs
|
|
731
|
-
into cache.db `quota_window_snapshots`, under
|
|
732
|
-
`cache.db.codex.lock`. Contract (journal seam):
|
|
733
|
-
`decoded = [(record, segment, offset), ...]` in
|
|
746
|
+
into cache.db `quota_window_snapshots`, under the NON-BLOCKING global cache
|
|
747
|
+
writer lock followed by `cache.db.codex.lock`. Contract (journal seam):
|
|
748
|
+
`(decoded) -> stop | None`, `decoded = [(record, segment, offset), ...]` in
|
|
749
|
+
canonical order.
|
|
734
750
|
|
|
735
751
|
- No Codex quota obs in the batch → return None (no flock taken).
|
|
736
|
-
- Busy
|
|
752
|
+
- Busy global/provider flock, OR a cache write it cannot complete → PREFIX-STOP:
|
|
737
753
|
return the index of the FIRST codex quota obs, so the cycle processes only
|
|
738
754
|
`decoded[:stop]` and advances the cursor to `decoded[stop]`'s offset,
|
|
739
755
|
retrying the remainder next cycle (the scalar cursor never advances past an
|
|
@@ -744,14 +760,22 @@ def _quota_applier(decoded) -> int | None:
|
|
|
744
760
|
if _is_codex_quota_obs(rec)]
|
|
745
761
|
if not quota_idx:
|
|
746
762
|
return None
|
|
763
|
+
from _lib_cache_writer_lock import (
|
|
764
|
+
acquire_cache_writer_flocks,
|
|
765
|
+
release_cache_writer_flocks,
|
|
766
|
+
)
|
|
767
|
+
|
|
747
768
|
_cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
|
|
748
|
-
fd = os.open(str(_cctally_core.CACHE_LOCK_CODEX_PATH),
|
|
749
|
-
os.O_RDWR | os.O_CREAT, 0o600)
|
|
750
769
|
try:
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
770
|
+
held = acquire_cache_writer_flocks(
|
|
771
|
+
_cctally_core.CACHE_LOCK_PATH,
|
|
772
|
+
_cctally_core.CACHE_LOCK_CODEX_PATH,
|
|
773
|
+
)
|
|
774
|
+
except OSError:
|
|
775
|
+
return quota_idx[0]
|
|
776
|
+
if held is None:
|
|
777
|
+
return quota_idx[0]
|
|
778
|
+
try:
|
|
755
779
|
try:
|
|
756
780
|
cache = sqlite3.connect(str(_cctally_core.CACHE_DB_PATH), timeout=15.0)
|
|
757
781
|
except sqlite3.Error as exc: # pragma: no cover — cache.db unopenable
|
|
@@ -761,9 +785,8 @@ def _quota_applier(decoded) -> int | None:
|
|
|
761
785
|
cache.execute("PRAGMA busy_timeout=15000")
|
|
762
786
|
cache.execute("BEGIN IMMEDIATE")
|
|
763
787
|
for i in quota_idx:
|
|
764
|
-
p = decoded[i][0]["payload"]
|
|
765
788
|
cache.execute(_QUOTA_SNAPSHOT_INSERT,
|
|
766
|
-
|
|
789
|
+
_quota_snapshot_values(decoded[i][0]))
|
|
767
790
|
cache.commit()
|
|
768
791
|
except sqlite3.Error as exc:
|
|
769
792
|
try:
|
|
@@ -778,10 +801,7 @@ def _quota_applier(decoded) -> int | None:
|
|
|
778
801
|
cache.close()
|
|
779
802
|
return None
|
|
780
803
|
finally:
|
|
781
|
-
|
|
782
|
-
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
783
|
-
finally:
|
|
784
|
-
os.close(fd)
|
|
804
|
+
release_cache_writer_flocks(held)
|
|
785
805
|
|
|
786
806
|
|
|
787
807
|
# Wire the seam (declared None near the top as the contract stub). Always-on:
|
|
@@ -869,14 +889,22 @@ def _usage_snapshot_fold_decision(conn, payload) -> tuple[bool, object]:
|
|
|
869
889
|
weekly_percent = float(payload["weekly_percent"])
|
|
870
890
|
five_hour_percent = payload.get("five_hour_percent")
|
|
871
891
|
five_hour_window_key = payload.get("five_hour_window_key")
|
|
892
|
+
# Account dimension (#341, review finding 11): every clamp/dedup query below
|
|
893
|
+
# is scoped to the account being processed so two accounts writing into the
|
|
894
|
+
# same week / same physical 5h window never clamp or dedup against each other.
|
|
895
|
+
# Defaults to the reserved sentinel when the caller omits it (byte-stable on a
|
|
896
|
+
# single-account install where every row shares one key).
|
|
897
|
+
account_key = payload.get("account_key") or _lib_accounts.UNATTRIBUTED
|
|
872
898
|
|
|
873
899
|
clamp_floor_iso = _cctally_core._reset_aware_floor(
|
|
874
|
-
conn, week_start_date, week_start_at, week_end_at
|
|
900
|
+
conn, week_start_date, week_start_at, week_end_at,
|
|
901
|
+
account_key=account_key,
|
|
875
902
|
) or "1970-01-01T00:00:00Z"
|
|
876
903
|
max_row = conn.execute(
|
|
877
904
|
"SELECT MAX(weekly_percent) FROM weekly_usage_snapshots "
|
|
878
|
-
"WHERE week_start_date = ? AND
|
|
879
|
-
(
|
|
905
|
+
"WHERE week_start_date = ? AND account_key = ? "
|
|
906
|
+
" AND unixepoch(captured_at_utc) >= unixepoch(?)",
|
|
907
|
+
(week_start_date, account_key, clamp_floor_iso),
|
|
880
908
|
).fetchone()
|
|
881
909
|
max_v = max_row[0] if max_row else None
|
|
882
910
|
if _lib_record.hwm_clamp_applies(weekly_percent, max_v):
|
|
@@ -886,12 +914,14 @@ def _usage_snapshot_fold_decision(conn, payload) -> tuple[bool, object]:
|
|
|
886
914
|
if five_hour_percent is not None and five_hour_window_key is not None:
|
|
887
915
|
max_5h_row = conn.execute(
|
|
888
916
|
"SELECT MAX(five_hour_percent) FROM weekly_usage_snapshots "
|
|
889
|
-
"WHERE five_hour_window_key = ? "
|
|
917
|
+
"WHERE five_hour_window_key = ? AND account_key = ? "
|
|
890
918
|
" AND unixepoch(captured_at_utc) >= unixepoch(COALESCE("
|
|
891
919
|
" (SELECT effective_reset_at_utc FROM five_hour_reset_events "
|
|
892
|
-
" WHERE five_hour_window_key = ?
|
|
920
|
+
" WHERE five_hour_window_key = ? AND account_key = ? "
|
|
921
|
+
" ORDER BY id DESC LIMIT 1),"
|
|
893
922
|
" '1970-01-01T00:00:00Z'))",
|
|
894
|
-
(int(five_hour_window_key),
|
|
923
|
+
(int(five_hour_window_key), account_key,
|
|
924
|
+
int(five_hour_window_key), account_key),
|
|
895
925
|
).fetchone()
|
|
896
926
|
max_5h = max_5h_row[0] if max_5h_row else None
|
|
897
927
|
if _lib_record.hwm_clamp_applies(float(five_hour_percent), max_5h):
|
|
@@ -899,8 +929,9 @@ def _usage_snapshot_fold_decision(conn, payload) -> tuple[bool, object]:
|
|
|
899
929
|
|
|
900
930
|
last = conn.execute(
|
|
901
931
|
"SELECT weekly_percent, five_hour_percent FROM weekly_usage_snapshots "
|
|
902
|
-
"WHERE week_start_date = ?
|
|
903
|
-
|
|
932
|
+
"WHERE week_start_date = ? AND account_key = ? "
|
|
933
|
+
"ORDER BY captured_at_utc DESC, id DESC LIMIT 1",
|
|
934
|
+
(week_start_date, account_key),
|
|
904
935
|
).fetchone()
|
|
905
936
|
if last is not None and float(last[0]) == weekly_percent:
|
|
906
937
|
last_5h = last[1]
|
|
@@ -929,13 +960,286 @@ def _apply_op_weekly_credit_floor(conn, record) -> None:
|
|
|
929
960
|
"effective_at_utc": payload["effective_at_utc"],
|
|
930
961
|
"observed_pre_credit_pct": float(payload["observed_pre_credit_pct"]),
|
|
931
962
|
"applied_at_utc": payload.get("applied_at_utc", record["at"]),
|
|
963
|
+
# Two-shaped stamp (#341 rev 4.1): evt/op carry account_key in the
|
|
964
|
+
# payload. Default to the sentinel for legacy ops written pre-#341.
|
|
965
|
+
"account_key": payload.get("account_key") or _lib_accounts.UNATTRIBUTED,
|
|
932
966
|
})
|
|
933
967
|
|
|
934
968
|
|
|
969
|
+
# --------------------------------------------------------------------------
|
|
970
|
+
# accounts registry fold (#341, spec §1/§2). `account_observe` / `account_label`
|
|
971
|
+
# op lines fold into the `accounts` registry. Registered here so BOTH the live
|
|
972
|
+
# ingest (`_pipeline_op_fold`) AND `rebuild_stats_index` (its op-fold stream)
|
|
973
|
+
# derive the registry deterministically. `last_seen_utc` is NOT set here — it
|
|
974
|
+
# derives from the max `at` of any account-stamped line via
|
|
975
|
+
# `_derive_account_last_seen`, run after the fold in both paths.
|
|
976
|
+
# --------------------------------------------------------------------------
|
|
977
|
+
|
|
978
|
+
_LABEL_RANK = {"auto": 0, "switcher": 1, "user": 2}
|
|
979
|
+
|
|
980
|
+
|
|
981
|
+
def _label_rank(source: str | None) -> int:
|
|
982
|
+
return _LABEL_RANK.get(source or "auto", 0)
|
|
983
|
+
|
|
984
|
+
|
|
985
|
+
def _apply_op_account_observe(conn, record) -> None:
|
|
986
|
+
"""Fold an `account_observe` op into the `accounts` registry. Idempotent:
|
|
987
|
+
INSERT OR IGNORE creates the row on first sight, then the identity fields
|
|
988
|
+
(provider/natural_id/email/plan_type) take the latest chronological value
|
|
989
|
+
(canonical fold order = chronological), `first_seen_utc` keeps the MIN `at`,
|
|
990
|
+
and an optional label is applied only when its provenance rank is >= the
|
|
991
|
+
stored one (user > switcher > auto — never override a user label)."""
|
|
992
|
+
p = record.get("payload") or {}
|
|
993
|
+
key = p.get("account_key")
|
|
994
|
+
at = record.get("at")
|
|
995
|
+
if not key:
|
|
996
|
+
return
|
|
997
|
+
conn.execute(
|
|
998
|
+
"INSERT OR IGNORE INTO accounts "
|
|
999
|
+
"(account_key, provider, label_source, first_seen_utc, last_seen_utc) "
|
|
1000
|
+
"VALUES (?, ?, 'auto', ?, ?)",
|
|
1001
|
+
(key, p.get("provider"), at, at),
|
|
1002
|
+
)
|
|
1003
|
+
conn.execute(
|
|
1004
|
+
"UPDATE accounts SET provider = COALESCE(?, provider), "
|
|
1005
|
+
" natural_id = COALESCE(?, natural_id), email = COALESCE(?, email), "
|
|
1006
|
+
" plan_type = COALESCE(?, plan_type) WHERE account_key = ?",
|
|
1007
|
+
(p.get("provider"), p.get("natural_id"), p.get("email"),
|
|
1008
|
+
p.get("plan_type"), key),
|
|
1009
|
+
)
|
|
1010
|
+
if at is not None:
|
|
1011
|
+
conn.execute(
|
|
1012
|
+
"UPDATE accounts SET first_seen_utc = ? WHERE account_key = ? "
|
|
1013
|
+
"AND (first_seen_utc IS NULL OR ? < first_seen_utc)",
|
|
1014
|
+
(at, key, at),
|
|
1015
|
+
)
|
|
1016
|
+
conn.execute(
|
|
1017
|
+
"UPDATE accounts SET last_seen_utc = ? WHERE account_key = ? "
|
|
1018
|
+
"AND (last_seen_utc IS NULL OR ? > last_seen_utc)",
|
|
1019
|
+
(at, key, at),
|
|
1020
|
+
)
|
|
1021
|
+
inc_label = p.get("label")
|
|
1022
|
+
if inc_label is not None:
|
|
1023
|
+
inc_src = p.get("label_source") or "auto"
|
|
1024
|
+
row = conn.execute(
|
|
1025
|
+
"SELECT label_source FROM accounts WHERE account_key = ?", (key,)
|
|
1026
|
+
).fetchone()
|
|
1027
|
+
cur_src = row[0] if row is not None else "auto"
|
|
1028
|
+
if _label_rank(inc_src) >= _label_rank(cur_src):
|
|
1029
|
+
conn.execute(
|
|
1030
|
+
"UPDATE accounts SET label = ?, label_source = ? "
|
|
1031
|
+
"WHERE account_key = ?",
|
|
1032
|
+
(inc_label, inc_src, key),
|
|
1033
|
+
)
|
|
1034
|
+
|
|
1035
|
+
|
|
1036
|
+
def _apply_op_account_label(conn, record) -> None:
|
|
1037
|
+
"""Fold an `account_label` op (a user rename) — always authoritative
|
|
1038
|
+
(label_source='user', the top of the precedence order)."""
|
|
1039
|
+
p = record.get("payload") or {}
|
|
1040
|
+
key = p.get("account_key")
|
|
1041
|
+
if not key:
|
|
1042
|
+
return
|
|
1043
|
+
at = record.get("at")
|
|
1044
|
+
conn.execute(
|
|
1045
|
+
"INSERT OR IGNORE INTO accounts "
|
|
1046
|
+
"(account_key, provider, label_source, first_seen_utc, last_seen_utc) "
|
|
1047
|
+
"VALUES (?, ?, 'auto', ?, ?)",
|
|
1048
|
+
(key, p.get("provider"), at, at),
|
|
1049
|
+
)
|
|
1050
|
+
conn.execute(
|
|
1051
|
+
"UPDATE accounts SET label = ?, label_source = 'user' "
|
|
1052
|
+
"WHERE account_key = ?",
|
|
1053
|
+
(p.get("label"), key),
|
|
1054
|
+
)
|
|
1055
|
+
|
|
1056
|
+
|
|
1057
|
+
def _account_of(record) -> str | None:
|
|
1058
|
+
"""The account_key a record contributes to `last_seen_utc`: the top-level
|
|
1059
|
+
`account` stamp on a data-bearing line, or an `account_observe`'s own key."""
|
|
1060
|
+
acct = record.get("account")
|
|
1061
|
+
if isinstance(acct, str) and acct:
|
|
1062
|
+
return acct
|
|
1063
|
+
if record.get("t") == "op":
|
|
1064
|
+
p = record.get("payload") or {}
|
|
1065
|
+
if p.get("kind") == "account_observe":
|
|
1066
|
+
k = p.get("account_key")
|
|
1067
|
+
if isinstance(k, str) and k:
|
|
1068
|
+
return k
|
|
1069
|
+
return None
|
|
1070
|
+
|
|
1071
|
+
|
|
1072
|
+
def _derive_account_last_seen(conn, records) -> None:
|
|
1073
|
+
"""Fold-time `last_seen_utc` derivation (spec §1): the MAX `at` of any
|
|
1074
|
+
account-stamped line advances an account's last-seen, so a stable account's
|
|
1075
|
+
last-seen keeps moving without extra observe records. Idempotent MAX update;
|
|
1076
|
+
runs after the fold in both the live cycle and rebuild. Only touches rows a
|
|
1077
|
+
prior observe already created (never invents an account row)."""
|
|
1078
|
+
latest: dict = {}
|
|
1079
|
+
for rec in records:
|
|
1080
|
+
key = _account_of(rec)
|
|
1081
|
+
at = rec.get("at")
|
|
1082
|
+
if not key or not at:
|
|
1083
|
+
continue
|
|
1084
|
+
prev = latest.get(key)
|
|
1085
|
+
if prev is None or at > prev:
|
|
1086
|
+
latest[key] = at
|
|
1087
|
+
for key, at in latest.items():
|
|
1088
|
+
conn.execute(
|
|
1089
|
+
"UPDATE accounts SET last_seen_utc = ? WHERE account_key = ? "
|
|
1090
|
+
"AND (last_seen_utc IS NULL OR ? > last_seen_utc)",
|
|
1091
|
+
(at, key, at),
|
|
1092
|
+
)
|
|
1093
|
+
|
|
1094
|
+
|
|
1095
|
+
# --------------------------------------------------------------------------
|
|
1096
|
+
# legacy classifier (#341, spec §2). A DATA-BEARING journal line that lacks an
|
|
1097
|
+
# `account` field is "legacy" (pre-cutover). Accounts-machinery records
|
|
1098
|
+
# (account_observe / account_label ops, the cutover op) are recognised by their
|
|
1099
|
+
# registered kinds and are NEITHER legacy NOR account-stamped data. This pure
|
|
1100
|
+
# classifier maps a legacy line to its provider, then `legacy_account_key` maps
|
|
1101
|
+
# that provider to the cutover mapping (Claude legacy -> the op's value; Codex
|
|
1102
|
+
# legacy -> unattributed). Used at rebuild + by the cache backfill migration to
|
|
1103
|
+
# normalise the missing account_key BEFORE insertion.
|
|
1104
|
+
# --------------------------------------------------------------------------
|
|
1105
|
+
|
|
1106
|
+
# Old evt lines carry no top-level provider, so evt kind -> provider is a fixed
|
|
1107
|
+
# table. `budget` is vendor-dependent (its payload `vendor` names the provider).
|
|
1108
|
+
_EVT_KIND_PROVIDER = {
|
|
1109
|
+
"snapshot_accept": "claude",
|
|
1110
|
+
"weekly_cost_snapshot": "claude",
|
|
1111
|
+
"week_reset": "claude",
|
|
1112
|
+
"five_hour_credit": "claude",
|
|
1113
|
+
"five_hour_block_close": "claude",
|
|
1114
|
+
"percent_milestone": "claude",
|
|
1115
|
+
"five_hour_milestone": "claude",
|
|
1116
|
+
"projected": "claude",
|
|
1117
|
+
"project_budget": "claude",
|
|
1118
|
+
"quota_alert_arming": "codex",
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
# Op kinds that are accounts-machinery (recognised, never classified as legacy).
|
|
1122
|
+
_ACCOUNTS_MACHINERY_KINDS = frozenset(
|
|
1123
|
+
("account_observe", "account_label", "accounts_cutover"))
|
|
1124
|
+
|
|
1125
|
+
# Legacy-classifier exhaustiveness guard (#341, review finding P2-1). EVERY evt
|
|
1126
|
+
# kind in `_EVT_SPECS` and every harvest kind in `_HARVEST_SPECS` must carry a
|
|
1127
|
+
# classifier disposition: a provider verdict (a data-bearing real-account or
|
|
1128
|
+
# `*`-family kind, via `_EVT_KIND_PROVIDER`), the vendor-tagged special case
|
|
1129
|
+
# (`budget`, provider read from the payload `vendor`), or an explicit EXEMPTION.
|
|
1130
|
+
# `weekly_credit_effects` is exempt because it is effects-only
|
|
1131
|
+
# (`_EvtSpec.table is None`): it inserts NO target row, so nothing carries
|
|
1132
|
+
# `account_key` to normalise — it only deletes stale-replica snapshots by their
|
|
1133
|
+
# globally-unique `journal_id` (an account-agnostic key) and force-writes the
|
|
1134
|
+
# account-agnostic hwm-7d statusline file. The exhaustiveness is asserted
|
|
1135
|
+
# STRUCTURALLY by tests/test_accounts_journal.py (iterating both spec registries),
|
|
1136
|
+
# so a future data-bearing kind cannot silently escape classification.
|
|
1137
|
+
_CLASSIFIER_VENDOR_TAGGED_KINDS = frozenset(("budget",))
|
|
1138
|
+
_CLASSIFIER_EXEMPT_KINDS = frozenset(("weekly_credit_effects",))
|
|
1139
|
+
|
|
1140
|
+
|
|
1141
|
+
def classify_legacy_provider(record) -> str | None:
|
|
1142
|
+
"""Return the provider ('claude'|'codex') of a DATA-BEARING legacy record
|
|
1143
|
+
(obs/op/evt lacking an account stamp), or None when the record is not
|
|
1144
|
+
legacy data — an already-account-stamped line, an accounts-machinery record,
|
|
1145
|
+
an effects-only exempt kind, or an unknown kind (additive-evolution
|
|
1146
|
+
tolerance).
|
|
1147
|
+
|
|
1148
|
+
Two-shaped already-stamped guard (#341 rev 4.1): obs carry the account on the
|
|
1149
|
+
top-level ``account`` field; evt/op carry it inside ``payload.account_key``.
|
|
1150
|
+
EITHER shape means the line is already account-stamped and is NOT legacy — a
|
|
1151
|
+
single-shape check would mis-classify a freshly account-stamped evt as legacy
|
|
1152
|
+
and re-normalise it."""
|
|
1153
|
+
if not isinstance(record, dict):
|
|
1154
|
+
return None
|
|
1155
|
+
payload = record.get("payload") or {}
|
|
1156
|
+
kind = payload.get("kind")
|
|
1157
|
+
# A line already carrying an account stamp (either shape) is not legacy.
|
|
1158
|
+
if isinstance(record.get("account"), str) and record.get("account"):
|
|
1159
|
+
return None
|
|
1160
|
+
if isinstance(payload.get("account_key"), str) and payload.get("account_key"):
|
|
1161
|
+
return None
|
|
1162
|
+
# Accounts-machinery records + effects-only exempt kinds carry no target row
|
|
1163
|
+
# to stamp — recognised by registration, never legacy data (review P3-D).
|
|
1164
|
+
if kind in _ACCOUNTS_MACHINERY_KINDS or kind in _CLASSIFIER_EXEMPT_KINDS:
|
|
1165
|
+
return None
|
|
1166
|
+
t = record.get("t")
|
|
1167
|
+
if t == "obs":
|
|
1168
|
+
prov = record.get("provider")
|
|
1169
|
+
return prov if prov in ("claude", "codex") else None
|
|
1170
|
+
if t == "op":
|
|
1171
|
+
# weekly_credit_floor is the only legacy op family; it is Claude.
|
|
1172
|
+
return "claude" if kind == "weekly_credit_floor" else None
|
|
1173
|
+
if t == "evt":
|
|
1174
|
+
if kind in _CLASSIFIER_VENDOR_TAGGED_KINDS:
|
|
1175
|
+
vendor = payload.get("vendor")
|
|
1176
|
+
return vendor if vendor in ("claude", "codex") else None
|
|
1177
|
+
return _EVT_KIND_PROVIDER.get(kind)
|
|
1178
|
+
return None
|
|
1179
|
+
|
|
1180
|
+
|
|
1181
|
+
def legacy_account_key(record, claude_legacy_account: str) -> str | None:
|
|
1182
|
+
"""Map a legacy record to the account_key to stamp: the cutover op's
|
|
1183
|
+
recorded Claude account for a Claude legacy line, `unattributed` for a Codex
|
|
1184
|
+
legacy line. Returns None when the record is not legacy data (caller leaves
|
|
1185
|
+
it untouched)."""
|
|
1186
|
+
prov = classify_legacy_provider(record)
|
|
1187
|
+
if prov is None:
|
|
1188
|
+
return None
|
|
1189
|
+
if prov == "claude":
|
|
1190
|
+
return claude_legacy_account
|
|
1191
|
+
return _lib_accounts.UNATTRIBUTED
|
|
1192
|
+
|
|
1193
|
+
|
|
1194
|
+
# The REAL-account evt/op kinds whose missing account_key is normalised to the
|
|
1195
|
+
# cutover mapping at rebuild (#341, handoff item 2). The `*`-families (`budget`,
|
|
1196
|
+
# `projected`, `project_budget`) are DELIBERATELY excluded — they take the
|
|
1197
|
+
# schema DEFAULT `'*'`, never the cutover account (spec §2 / scope matrix).
|
|
1198
|
+
_REAL_ACCOUNT_EVT_OP_KINDS = frozenset((
|
|
1199
|
+
"snapshot_accept", "weekly_cost_snapshot", "week_reset", "five_hour_credit",
|
|
1200
|
+
"five_hour_block_close", "percent_milestone", "five_hour_milestone",
|
|
1201
|
+
"weekly_credit_floor",
|
|
1202
|
+
))
|
|
1203
|
+
|
|
1204
|
+
|
|
1205
|
+
def _normalize_legacy_account_stamp(record, claude_legacy_account: str) -> None:
|
|
1206
|
+
"""In-place two-shaped account normalisation for a legacy (pre-#341) record
|
|
1207
|
+
at rebuild (spec §2, handoff item 2). obs get a top-level ``account``; a
|
|
1208
|
+
REAL-account evt/op gets ``payload.account_key``. Already-stamped records,
|
|
1209
|
+
``*``-families, and unknown/machinery kinds are left untouched — so a rebuild
|
|
1210
|
+
over a cutover-op'd journal reproduces pre-feature Claude data under the op's
|
|
1211
|
+
account and pre-feature Codex data under ``unattributed`` (acceptance 4)."""
|
|
1212
|
+
if not isinstance(record, dict):
|
|
1213
|
+
return
|
|
1214
|
+
t = record.get("t")
|
|
1215
|
+
if t == "obs":
|
|
1216
|
+
if isinstance(record.get("account"), str) and record.get("account"):
|
|
1217
|
+
return
|
|
1218
|
+
key = legacy_account_key(record, claude_legacy_account)
|
|
1219
|
+
if key is not None:
|
|
1220
|
+
record["account"] = key
|
|
1221
|
+
return
|
|
1222
|
+
if t in ("evt", "op"):
|
|
1223
|
+
payload = record.get("payload")
|
|
1224
|
+
if not isinstance(payload, dict):
|
|
1225
|
+
return
|
|
1226
|
+
if payload.get("kind") not in _REAL_ACCOUNT_EVT_OP_KINDS:
|
|
1227
|
+
return
|
|
1228
|
+
if isinstance(payload.get("account_key"), str) and payload.get("account_key"):
|
|
1229
|
+
return
|
|
1230
|
+
key = legacy_account_key(record, claude_legacy_account)
|
|
1231
|
+
if key is not None:
|
|
1232
|
+
payload["account_key"] = key
|
|
1233
|
+
|
|
1234
|
+
|
|
935
1235
|
# Obs/op fold registry (spec §5.3 "fold op"). Keyed by `payload.kind`; the
|
|
936
1236
|
# built-in `_pipeline_op_fold` pipeline hook dispatches through it. 6b may
|
|
937
1237
|
# register more op folds; obs no longer fold directly (see the NOTE above).
|
|
938
|
-
FOLD_APPLIERS = {
|
|
1238
|
+
FOLD_APPLIERS = {
|
|
1239
|
+
"weekly_credit_floor": _apply_op_weekly_credit_floor,
|
|
1240
|
+
"account_observe": _apply_op_account_observe,
|
|
1241
|
+
"account_label": _apply_op_account_label,
|
|
1242
|
+
}
|
|
939
1243
|
|
|
940
1244
|
|
|
941
1245
|
# --------------------------------------------------------------------------
|
|
@@ -974,12 +1278,23 @@ def _apply_generic_evt(conn, evt):
|
|
|
974
1278
|
cols[key] = value
|
|
975
1279
|
# Re-derive any projection-FK columns from a journaled natural-key column
|
|
976
1280
|
# (spec §5.3 — e.g. five_hour_milestones.block_id from five_hour_window_key,
|
|
977
|
-
# since the open block is a projection with no logical id).
|
|
1281
|
+
# since the open block is a projection with no logical id). Composite
|
|
1282
|
+
# (account_key, <lookup_col>) when the row carries an account (#341, review
|
|
1283
|
+
# finding 3): a shared physical 5h window resolves THIS account's block, so a
|
|
1284
|
+
# milestone child never attaches to another account's block. 0 when absent.
|
|
1285
|
+
acct = cols.get("account_key")
|
|
978
1286
|
for column, (ref_table, lookup_col) in spec.derived_fk.items():
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
1287
|
+
if acct is not None:
|
|
1288
|
+
row = conn.execute(
|
|
1289
|
+
f"SELECT id FROM {ref_table} "
|
|
1290
|
+
f"WHERE {lookup_col} = ? AND account_key = ?",
|
|
1291
|
+
(cols.get(lookup_col), acct),
|
|
1292
|
+
).fetchone()
|
|
1293
|
+
else:
|
|
1294
|
+
row = conn.execute(
|
|
1295
|
+
f"SELECT id FROM {ref_table} WHERE {lookup_col} = ?",
|
|
1296
|
+
(cols.get(lookup_col),),
|
|
1297
|
+
).fetchone()
|
|
983
1298
|
cols[column] = int(row[0]) if row is not None else 0
|
|
984
1299
|
return _insert_or_ignore(conn, spec.table, cols)
|
|
985
1300
|
|
|
@@ -1031,18 +1346,23 @@ def _apply_quota_alert_arming(conn, evt):
|
|
|
1031
1346
|
no `journal_id` column (it is not in the Task-4 additive list); idempotence
|
|
1032
1347
|
is the natural-key upsert, not a journal_id INSERT OR IGNORE."""
|
|
1033
1348
|
p = evt.get("payload") or {}
|
|
1349
|
+
# account_key (#341) is part of the arming identity/UNIQUE. A live-emitted evt
|
|
1350
|
+
# carries payload.account_key; a legacy (pre-#341) cutover-exported arming has
|
|
1351
|
+
# none -> normalise to the sentinel (Codex legacy -> unattributed) so the
|
|
1352
|
+
# NOT NULL column always receives a value.
|
|
1353
|
+
account_key = p.get("account_key") or _lib_accounts.UNATTRIBUTED
|
|
1034
1354
|
conn.execute(
|
|
1035
1355
|
"INSERT INTO quota_alert_arming "
|
|
1036
1356
|
"(source, source_root_key, logical_limit_key, observed_slot, "
|
|
1037
|
-
" window_minutes, rule_fingerprint, activated_at_utc) "
|
|
1038
|
-
"VALUES (
|
|
1039
|
-
"ON CONFLICT(source, source_root_key,
|
|
1040
|
-
" window_minutes) DO UPDATE SET "
|
|
1357
|
+
" window_minutes, rule_fingerprint, activated_at_utc, account_key) "
|
|
1358
|
+
"VALUES (?,?,?,?,?,?,?,?) "
|
|
1359
|
+
"ON CONFLICT(source, source_root_key, account_key, logical_limit_key, "
|
|
1360
|
+
" observed_slot, window_minutes) DO UPDATE SET "
|
|
1041
1361
|
" rule_fingerprint=excluded.rule_fingerprint, "
|
|
1042
1362
|
" activated_at_utc=excluded.activated_at_utc",
|
|
1043
1363
|
(p.get("source"), p.get("source_root_key"), p.get("logical_limit_key"),
|
|
1044
1364
|
p.get("observed_slot"), p.get("window_minutes"),
|
|
1045
|
-
p.get("rule_fingerprint"), p.get("activated_at_utc")),
|
|
1365
|
+
p.get("rule_fingerprint"), p.get("activated_at_utc"), account_key),
|
|
1046
1366
|
)
|
|
1047
1367
|
return None
|
|
1048
1368
|
|
|
@@ -1065,10 +1385,21 @@ def _apply_block_close(conn, evt):
|
|
|
1065
1385
|
continue
|
|
1066
1386
|
parent[key] = value
|
|
1067
1387
|
_insert_or_ignore(conn, "five_hour_blocks", parent)
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1388
|
+
# Composite (account_key, five_hour_window_key) parent resolution (#341,
|
|
1389
|
+
# review finding 3): a shared physical window resolves THIS account's block
|
|
1390
|
+
# so its rollup children attach to the right parent.
|
|
1391
|
+
p_acct = parent.get("account_key")
|
|
1392
|
+
if p_acct is not None:
|
|
1393
|
+
prow = conn.execute(
|
|
1394
|
+
"SELECT id FROM five_hour_blocks "
|
|
1395
|
+
"WHERE five_hour_window_key = ? AND account_key = ?",
|
|
1396
|
+
(parent.get("five_hour_window_key"), p_acct),
|
|
1397
|
+
).fetchone()
|
|
1398
|
+
else:
|
|
1399
|
+
prow = conn.execute(
|
|
1400
|
+
"SELECT id FROM five_hour_blocks WHERE five_hour_window_key = ?",
|
|
1401
|
+
(parent.get("five_hour_window_key"),),
|
|
1402
|
+
).fetchone()
|
|
1072
1403
|
if prow is None:
|
|
1073
1404
|
return None
|
|
1074
1405
|
block_id = int(prow[0])
|
|
@@ -1076,6 +1407,17 @@ def _apply_block_close(conn, evt):
|
|
|
1076
1407
|
for child in children.get(payload_key, []):
|
|
1077
1408
|
cols = dict(child)
|
|
1078
1409
|
cols["block_id"] = block_id
|
|
1410
|
+
# Force each child under the PARENT's account (#341 P2-2, 8a review):
|
|
1411
|
+
# a no-op on the live/already-stamped path (children already agree),
|
|
1412
|
+
# but on the legacy rebuild path _normalize_legacy_account_stamp
|
|
1413
|
+
# re-derives ONLY the parent's payload.account_key — the embedded
|
|
1414
|
+
# _models/_projects children stay unstamped and would otherwise take
|
|
1415
|
+
# the schema DEFAULT 'unattributed', mismatching their parent and
|
|
1416
|
+
# splitting the composite (account_key, window, model/project) UNIQUE
|
|
1417
|
+
# partition. Guarded on p_acct so a truly account-less rebuild (no
|
|
1418
|
+
# cutover mapping) leaves the DEFAULT untouched.
|
|
1419
|
+
if p_acct is not None:
|
|
1420
|
+
cols["account_key"] = p_acct
|
|
1079
1421
|
_insert_or_ignore(conn, child_table, cols)
|
|
1080
1422
|
return None
|
|
1081
1423
|
|
|
@@ -1126,26 +1468,32 @@ def _apply_evt(conn, evt):
|
|
|
1126
1468
|
# harvest registry (natural-keyed families, spec §5.3)
|
|
1127
1469
|
# --------------------------------------------------------------------------
|
|
1128
1470
|
|
|
1471
|
+
# Every harvest family's natural key now leads with `account_key` (#341): the
|
|
1472
|
+
# account is part of each table's extended UNIQUE, so the opaque evt id must
|
|
1473
|
+
# include it to stay a bijection with the row (two accounts sharing a physical
|
|
1474
|
+
# window / week / threshold produce DISTINCT evt ids). account_key is also a
|
|
1475
|
+
# plain payload column, so the generic fold round-trips it back onto the row.
|
|
1129
1476
|
_HARVEST_SPECS = [
|
|
1130
1477
|
_HarvestSpec(
|
|
1131
1478
|
"week_reset_events", "week_reset", "wr",
|
|
1132
|
-
id_parts=("old_week_end_at", "new_week_end_at"),
|
|
1479
|
+
id_parts=("account_key", "old_week_end_at", "new_week_end_at"),
|
|
1133
1480
|
at_column="detected_at_utc", order=30, suppression=True,
|
|
1134
1481
|
),
|
|
1135
1482
|
_HarvestSpec(
|
|
1136
1483
|
"five_hour_reset_events", "five_hour_credit", "fhc",
|
|
1137
|
-
id_parts=("five_hour_window_key", "effective_reset_at_utc"),
|
|
1484
|
+
id_parts=("account_key", "five_hour_window_key", "effective_reset_at_utc"),
|
|
1138
1485
|
at_column="detected_at_utc", order=30, suppression=True,
|
|
1139
1486
|
),
|
|
1140
1487
|
_HarvestSpec(
|
|
1141
1488
|
"five_hour_blocks", "five_hour_block_close", "fhbc",
|
|
1142
|
-
id_parts=("five_hour_window_key"
|
|
1489
|
+
id_parts=("account_key", "five_hour_window_key"),
|
|
1143
1490
|
at_column="last_updated_at_utc", order=40, closed_only=True,
|
|
1144
1491
|
children=_BLOCK_CHILDREN,
|
|
1145
1492
|
),
|
|
1146
1493
|
_HarvestSpec(
|
|
1147
1494
|
"percent_milestones", "percent_milestone", "pm",
|
|
1148
|
-
id_parts=("
|
|
1495
|
+
id_parts=("account_key", "week_start_date", "reset_event_id",
|
|
1496
|
+
"percent_threshold"),
|
|
1149
1497
|
fk_refs={
|
|
1150
1498
|
"usage_snapshot_id": ("weekly_usage_snapshots", "usage_snapshot_ref"),
|
|
1151
1499
|
"cost_snapshot_id": ("weekly_cost_snapshots", "cost_snapshot_ref"),
|
|
@@ -1155,30 +1503,33 @@ _HARVEST_SPECS = [
|
|
|
1155
1503
|
),
|
|
1156
1504
|
_HarvestSpec(
|
|
1157
1505
|
"five_hour_milestones", "five_hour_milestone", "fhm",
|
|
1158
|
-
id_parts=("
|
|
1506
|
+
id_parts=("account_key", "five_hour_window_key", "reset_event_id",
|
|
1507
|
+
"percent_threshold"),
|
|
1159
1508
|
fk_refs={
|
|
1160
1509
|
"usage_snapshot_id": ("weekly_usage_snapshots", "usage_snapshot_ref"),
|
|
1161
1510
|
"reset_event_id": ("five_hour_reset_events", "reset_event_ref"),
|
|
1162
1511
|
},
|
|
1163
1512
|
# block_id points at the OPEN five_hour_blocks row (a projection with no
|
|
1164
|
-
# journal_id) — re-derive it at fold from the journaled
|
|
1165
|
-
# instead of a broken logical FK
|
|
1513
|
+
# journal_id) — re-derive it at fold from the journaled (account_key,
|
|
1514
|
+
# window key) composite (#341) instead of a broken logical FK.
|
|
1166
1515
|
derived_fk={"block_id": ("five_hour_blocks", "five_hour_window_key")},
|
|
1167
1516
|
at_column="captured_at_utc", order=60,
|
|
1168
1517
|
),
|
|
1169
1518
|
_HarvestSpec(
|
|
1170
1519
|
"budget_milestones", "budget", "bm",
|
|
1171
|
-
id_parts=("
|
|
1520
|
+
id_parts=("account_key", "vendor", "period_start_at", "period",
|
|
1521
|
+
"threshold"),
|
|
1172
1522
|
at_column="crossed_at_utc", order=60,
|
|
1173
1523
|
),
|
|
1174
1524
|
_HarvestSpec(
|
|
1175
1525
|
"projected_milestones", "projected", "pjm",
|
|
1176
|
-
id_parts=("
|
|
1526
|
+
id_parts=("account_key", "week_start_at", "period", "metric",
|
|
1527
|
+
"threshold"),
|
|
1177
1528
|
at_column="crossed_at_utc", order=60,
|
|
1178
1529
|
),
|
|
1179
1530
|
_HarvestSpec(
|
|
1180
1531
|
"project_budget_milestones", "project_budget", "pbm",
|
|
1181
|
-
id_parts=("week_start_at", "project_key", "threshold"),
|
|
1532
|
+
id_parts=("account_key", "week_start_at", "project_key", "threshold"),
|
|
1182
1533
|
at_column="crossed_at_utc", order=60,
|
|
1183
1534
|
),
|
|
1184
1535
|
]
|
|
@@ -1543,8 +1894,8 @@ def _run_cycle(conn: sqlite3.Connection, *, reconcile_config=None,
|
|
|
1543
1894
|
decoded.append((rec, seg, off))
|
|
1544
1895
|
|
|
1545
1896
|
# Step 3: cache leg (Codex quota) BEFORE the stats txn (lock-order law).
|
|
1546
|
-
# QUOTA_APPLIER attempts the cache
|
|
1547
|
-
# flock it returns a prefix-stop index — the cycle processes
|
|
1897
|
+
# QUOTA_APPLIER attempts the global-then-Codex cache flock NB upsert; on
|
|
1898
|
+
# a busy flock it returns a prefix-stop index — the cycle processes
|
|
1548
1899
|
# decoded[:stop], sets the cursor to decoded[stop]'s offset, and retries
|
|
1549
1900
|
# the remainder next cycle (§5.2 step 3; prefix consumption keeps the
|
|
1550
1901
|
# scalar cursor sound).
|
|
@@ -1613,6 +1964,11 @@ def _run_cycle(conn: sqlite3.Connection, *, reconcile_config=None,
|
|
|
1613
1964
|
# O(this-cycle inserts) even on the accept-only common tick).
|
|
1614
1965
|
if conn.total_changes != pipeline_changes_before:
|
|
1615
1966
|
_harvest(ctx)
|
|
1967
|
+
# 4c'. Fold-time `last_seen_utc` derivation (#341): advance each account's
|
|
1968
|
+
# last-seen from the max `at` of any account-stamped line this cycle. A
|
|
1969
|
+
# no-op when the batch carries no account stamps (byte-stable on a
|
|
1970
|
+
# pre-multi-account single-account install).
|
|
1971
|
+
_derive_account_last_seen(conn, records)
|
|
1616
1972
|
# 4d. Advance the cursor (to HW, or to the cache-leg prefix boundary).
|
|
1617
1973
|
# `cursor_target is None` ONLY on a reconcile-only cycle over a still-
|
|
1618
1974
|
# empty journal (§5.2 above): there are no consumed lines to advance
|
|
@@ -1764,7 +2120,7 @@ _REBUILD_COUNT_TABLES = (
|
|
|
1764
2120
|
"five_hour_block_projects", "weekly_credit_floors", "percent_milestones",
|
|
1765
2121
|
"five_hour_milestones", "budget_milestones", "projected_milestones",
|
|
1766
2122
|
"project_budget_milestones", "quota_alert_arming", "quota_window_blocks",
|
|
1767
|
-
"quota_percent_milestones", "quota_threshold_events",
|
|
2123
|
+
"quota_percent_milestones", "quota_threshold_events", "accounts",
|
|
1768
2124
|
)
|
|
1769
2125
|
|
|
1770
2126
|
|
|
@@ -1800,20 +2156,35 @@ def _rebuild_quota_cache_leg(records) -> None:
|
|
|
1800
2156
|
quota obs (spec §5.4). The journal obs are the DURABLE source (§1 latent
|
|
1801
2157
|
data-loss hole — the rollout JSONL evaporates); this INSERT OR IGNOREs them
|
|
1802
2158
|
on the natural key, mirroring `_quota_applier`. Runs BEFORE any stats
|
|
1803
|
-
transaction, under the
|
|
1804
|
-
|
|
1805
|
-
|
|
2159
|
+
transaction, under the global cache writer lock followed by the
|
|
2160
|
+
`cache.db.codex.lock` provider flock (lock-order law). Best-effort: a
|
|
2161
|
+
missing/busy cache.db is a clean skip (the obs stay durable in the journal;
|
|
2162
|
+
the stats quota projection pass then degrades cleanly)."""
|
|
1806
2163
|
quota_obs = [r for r in records if _is_codex_quota_obs(r)]
|
|
1807
2164
|
if not quota_obs:
|
|
1808
2165
|
return
|
|
1809
2166
|
cache_path = _cctally_core.CACHE_DB_PATH
|
|
1810
2167
|
if not cache_path.exists():
|
|
1811
2168
|
return
|
|
2169
|
+
from _lib_cache_writer_lock import (
|
|
2170
|
+
acquire_cache_writer_flocks,
|
|
2171
|
+
release_cache_writer_flocks,
|
|
2172
|
+
)
|
|
2173
|
+
|
|
1812
2174
|
_cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
|
|
1813
|
-
fd = os.open(str(_cctally_core.CACHE_LOCK_CODEX_PATH),
|
|
1814
|
-
os.O_RDWR | os.O_CREAT, 0o600)
|
|
1815
2175
|
try:
|
|
1816
|
-
|
|
2176
|
+
held = acquire_cache_writer_flocks(
|
|
2177
|
+
_cctally_core.CACHE_LOCK_PATH,
|
|
2178
|
+
_cctally_core.CACHE_LOCK_CODEX_PATH,
|
|
2179
|
+
timeout=15.0,
|
|
2180
|
+
)
|
|
2181
|
+
except OSError as exc:
|
|
2182
|
+
print(f"[rebuild] quota cache leg lock failed: {exc}", file=sys.stderr)
|
|
2183
|
+
return
|
|
2184
|
+
if held is None:
|
|
2185
|
+
print("[rebuild] quota cache leg locks busy; skipping", file=sys.stderr)
|
|
2186
|
+
return
|
|
2187
|
+
try:
|
|
1817
2188
|
try:
|
|
1818
2189
|
cache = sqlite3.connect(str(cache_path), timeout=15.0)
|
|
1819
2190
|
except sqlite3.Error as exc: # pragma: no cover — cache.db unopenable
|
|
@@ -1823,9 +2194,8 @@ def _rebuild_quota_cache_leg(records) -> None:
|
|
|
1823
2194
|
cache.execute("PRAGMA busy_timeout=15000")
|
|
1824
2195
|
cache.execute("BEGIN IMMEDIATE")
|
|
1825
2196
|
for r in quota_obs:
|
|
1826
|
-
p = r["payload"]
|
|
1827
2197
|
cache.execute(_QUOTA_SNAPSHOT_INSERT,
|
|
1828
|
-
|
|
2198
|
+
_quota_snapshot_values(r))
|
|
1829
2199
|
cache.commit()
|
|
1830
2200
|
except sqlite3.Error as exc:
|
|
1831
2201
|
try:
|
|
@@ -1836,10 +2206,7 @@ def _rebuild_quota_cache_leg(records) -> None:
|
|
|
1836
2206
|
finally:
|
|
1837
2207
|
cache.close()
|
|
1838
2208
|
finally:
|
|
1839
|
-
|
|
1840
|
-
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
1841
|
-
finally:
|
|
1842
|
-
os.close(fd)
|
|
2209
|
+
release_cache_writer_flocks(held)
|
|
1843
2210
|
|
|
1844
2211
|
|
|
1845
2212
|
def rebuild_stats_index(*, target_path=None) -> RebuildResult:
|
|
@@ -1887,6 +2254,17 @@ def rebuild_stats_index(*, target_path=None) -> RebuildResult:
|
|
|
1887
2254
|
continue
|
|
1888
2255
|
decoded.append(rec)
|
|
1889
2256
|
|
|
2257
|
+
# Legacy account normalisation (#341, spec §2 / handoff item 2): a
|
|
2258
|
+
# pre-#341 real-account line lacks an account stamp — inject the cutover
|
|
2259
|
+
# mapping BEFORE the fold (Claude legacy -> the cutover op's account;
|
|
2260
|
+
# Codex legacy -> unattributed). `*`-families + already-stamped lines are
|
|
2261
|
+
# untouched. Resolved once from the journal's own cutover op (falls back
|
|
2262
|
+
# to `unattributed` when none is present), so a fresh single-account
|
|
2263
|
+
# rebuild is byte-neutral (everything is already `unattributed`).
|
|
2264
|
+
cutover_claude = resolve_cutover_claude_account()
|
|
2265
|
+
for rec in decoded:
|
|
2266
|
+
_normalize_legacy_account_stamp(rec, cutover_claude)
|
|
2267
|
+
|
|
1890
2268
|
# Cache leg BEFORE any stats txn (provider-flock lock-order): journal
|
|
1891
2269
|
# Codex quota obs -> cache.db quota_window_snapshots.
|
|
1892
2270
|
_rebuild_quota_cache_leg(decoded)
|
|
@@ -1950,6 +2328,10 @@ def rebuild_stats_index(*, target_path=None) -> RebuildResult:
|
|
|
1950
2328
|
for _order, _seq, _kind, rec in tail:
|
|
1951
2329
|
_apply_evt(conn, rec)
|
|
1952
2330
|
lines_folded += 1
|
|
2331
|
+
# Fold-time `last_seen_utc` derivation (#341): re-derive each
|
|
2332
|
+
# account's last-seen from the whole journal (the observe ops folded
|
|
2333
|
+
# in the structural phase already created the rows).
|
|
2334
|
+
_derive_account_last_seen(conn, decoded)
|
|
1953
2335
|
if hw is not None:
|
|
1954
2336
|
_write_cursor(conn, hw[0], hw[1])
|
|
1955
2337
|
conn.commit()
|
|
@@ -2103,8 +2485,9 @@ _CUTOVER_SPECS = (
|
|
|
2103
2485
|
# record and a later live re-emission for the same identity are ONE record.
|
|
2104
2486
|
_CutoverSpec("quota_alert_arming", "quota_alert_arming", "evt",
|
|
2105
2487
|
"activated_at_utc", stamp=False, natural_key_prefix="qaa",
|
|
2106
|
-
natural_key_id=("source", "source_root_key", "
|
|
2107
|
-
"
|
|
2488
|
+
natural_key_id=("source", "source_root_key", "account_key",
|
|
2489
|
+
"logical_limit_key", "observed_slot",
|
|
2490
|
+
"window_minutes")),
|
|
2108
2491
|
)
|
|
2109
2492
|
|
|
2110
2493
|
# Journal-covered stats tables whose rows get a `journal_id` stamp at cutover.
|
|
@@ -2126,8 +2509,14 @@ def _export_stats_table(conn, spec) -> list:
|
|
|
2126
2509
|
rowid = row["id"]
|
|
2127
2510
|
payload = {}
|
|
2128
2511
|
for key in row.keys():
|
|
2129
|
-
|
|
2130
|
-
|
|
2512
|
+
# account_key (#341) is NEVER exported — a legacy stats.db row carries
|
|
2513
|
+
# only the schema DEFAULT ('unattributed'/'*'), so exporting it would
|
|
2514
|
+
# make the bootstrap evt look already-stamped and defeat the rebuild's
|
|
2515
|
+
# legacy normalisation. Dropping it lets the fold re-derive the right
|
|
2516
|
+
# account (Claude legacy -> cutover op's account; Codex legacy ->
|
|
2517
|
+
# unattributed; `*`-families -> schema DEFAULT '*').
|
|
2518
|
+
if key in ("id", "journal_id", "account_key") \
|
|
2519
|
+
or key in spec.fk_refs or key in spec.exclude:
|
|
2131
2520
|
continue
|
|
2132
2521
|
payload[key] = row[key]
|
|
2133
2522
|
for col, (ref_table, ref_key) in spec.fk_refs.items():
|
|
@@ -2141,9 +2530,16 @@ def _export_stats_table(conn, spec) -> list:
|
|
|
2141
2530
|
for cr in child_rows]
|
|
2142
2531
|
if spec.natural_key_id:
|
|
2143
2532
|
# §5.3 "state" family: the evt id is the natural-key form (matching
|
|
2144
|
-
# the live emission), NOT the b:<table>:<rowid> bootstrap id.
|
|
2533
|
+
# the live emission), NOT the b:<table>:<rowid> bootstrap id. A legacy
|
|
2534
|
+
# (pre-#341) stats.db has no `account_key` column, so a natural-key
|
|
2535
|
+
# component absent from the row is the sentinel (#341): the exported
|
|
2536
|
+
# qaa id becomes `qaa:...:unattributed:...`, matching what a live
|
|
2537
|
+
# re-emission for the unattributed identity would build.
|
|
2538
|
+
row_cols = set(row.keys())
|
|
2145
2539
|
bid = _lib_journal.evt_id(
|
|
2146
|
-
spec.natural_key_prefix,
|
|
2540
|
+
spec.natural_key_prefix,
|
|
2541
|
+
*(row[c] if c in row_cols else _lib_accounts.UNATTRIBUTED
|
|
2542
|
+
for c in spec.natural_key_id))
|
|
2147
2543
|
else:
|
|
2148
2544
|
bid = _lib_journal.bootstrap_id(spec.table, rowid)
|
|
2149
2545
|
at = row[spec.at_col]
|
|
@@ -2172,9 +2568,20 @@ def _export_quota_obs() -> list:
|
|
|
2172
2568
|
except sqlite3.Error:
|
|
2173
2569
|
return []
|
|
2174
2570
|
try:
|
|
2571
|
+
# account_key (#341) rides the obs top-level ``account`` field, NOT the
|
|
2572
|
+
# payload — so a NULL/unattributed cache row exports an obs that OMITS the
|
|
2573
|
+
# field (byte-stable, invariant #1). Selected after the payload cols when
|
|
2574
|
+
# present; a pre-#341 cache lacking the column exports NULL (never loses
|
|
2575
|
+
# the durable quota obs).
|
|
2576
|
+
has_account = any(
|
|
2577
|
+
str(r[1]) == "account_key"
|
|
2578
|
+
for r in cache.execute("PRAGMA table_info(quota_window_snapshots)")
|
|
2579
|
+
)
|
|
2580
|
+
acct_sel = ", account_key" if has_account else ", NULL AS account_key"
|
|
2175
2581
|
cols = ", ".join(_QUOTA_SNAPSHOT_COLS)
|
|
2176
2582
|
rows = cache.execute(
|
|
2177
|
-
f"SELECT id, {cols} FROM quota_window_snapshots"
|
|
2583
|
+
f"SELECT id, {cols}{acct_sel} FROM quota_window_snapshots"
|
|
2584
|
+
).fetchall()
|
|
2178
2585
|
except sqlite3.Error:
|
|
2179
2586
|
return []
|
|
2180
2587
|
finally:
|
|
@@ -2185,9 +2592,10 @@ def _export_quota_obs() -> list:
|
|
|
2185
2592
|
payload = {"kind": _QUOTA_OBS_KIND}
|
|
2186
2593
|
for i, col in enumerate(_QUOTA_SNAPSHOT_COLS):
|
|
2187
2594
|
payload[col] = row[1 + i]
|
|
2595
|
+
account = row[1 + len(_QUOTA_SNAPSHOT_COLS)]
|
|
2188
2596
|
at = payload.get("captured_at_utc") or _now_iso()
|
|
2189
2597
|
rec = _lib_journal.make_obs(at=at, src="bootstrap", provider="codex",
|
|
2190
|
-
payload=payload)
|
|
2598
|
+
account=account, payload=payload)
|
|
2191
2599
|
rec["id"] = _lib_journal.bootstrap_id("quota_window_snapshots", rowid)
|
|
2192
2600
|
out.append(rec)
|
|
2193
2601
|
return out
|
|
@@ -2290,3 +2698,106 @@ def run_cutover(conn, *, now_utc: dt.datetime | None = None) -> "str | None":
|
|
|
2290
2698
|
except Exception:
|
|
2291
2699
|
pass
|
|
2292
2700
|
raise
|
|
2701
|
+
|
|
2702
|
+
|
|
2703
|
+
# ==========================================================================
|
|
2704
|
+
# Account epoch-transition coordinator (#341, spec §2)
|
|
2705
|
+
# ==========================================================================
|
|
2706
|
+
#
|
|
2707
|
+
# Epoch 1000 -> 1001 adds the account dimension. An existing epoch-1000 stats.db
|
|
2708
|
+
# reaches `resolve_stats_epoch_mismatch`, which runs this coordinator BEFORE the
|
|
2709
|
+
# rebuild, in exact order (spec §2, review finding 1):
|
|
2710
|
+
# (1) resolve the cutover identity WITHOUT opening stats.db — a stable-read of
|
|
2711
|
+
# ~/.claude.json; stably-absent / torn -> `unattributed` (never a guess);
|
|
2712
|
+
# (2) atomically check/append the canonical cutover op (stable semantic id
|
|
2713
|
+
# `accounts-cutover-v1`, timestamp-independent, so a retry cannot duplicate
|
|
2714
|
+
# it and replay is deterministic forever);
|
|
2715
|
+
# (3) only THEN capture the journal HW and rebuild — so the op is always inside
|
|
2716
|
+
# the rebuild's input and the legacy classifier can consume its recorded
|
|
2717
|
+
# Claude account.
|
|
2718
|
+
# The cache backfill migration (Task 2) consumes the SAME op value; it never
|
|
2719
|
+
# re-reads auth. The coordinator opens no stats.db until the rebuild's own
|
|
2720
|
+
# scratch, so no ordering circularity exists.
|
|
2721
|
+
|
|
2722
|
+
CUTOVER_OP_ID = "accounts-cutover-v1" # stable semantic id (timestamp-free)
|
|
2723
|
+
_CUTOVER_OP_KIND = "accounts_cutover"
|
|
2724
|
+
_CUTOVER_OP_SRC = "accounts-cutover"
|
|
2725
|
+
|
|
2726
|
+
|
|
2727
|
+
def _resolve_claude_cutover_identity(claude_json_path=None) -> str:
|
|
2728
|
+
"""Resolve the single legacy Claude account_key from ~/.claude.json without
|
|
2729
|
+
opening stats.db (spec §2 step 1). Identified -> account_key; stably-absent
|
|
2730
|
+
(missing file / no oauthAccount) or torn (unparseable mid-write) ->
|
|
2731
|
+
``unattributed`` (the op is always appended with whatever was resolvable, so
|
|
2732
|
+
replay is deterministic forever after)."""
|
|
2733
|
+
path = str(claude_json_path) if claude_json_path is not None \
|
|
2734
|
+
else str(_cctally_core.CLAUDE_JSON_PATH)
|
|
2735
|
+
|
|
2736
|
+
def _reader(data: bytes):
|
|
2737
|
+
try:
|
|
2738
|
+
obj = json.loads(data)
|
|
2739
|
+
except (ValueError, TypeError):
|
|
2740
|
+
raise _lib_accounts.TornRead()
|
|
2741
|
+
if not isinstance(obj, dict):
|
|
2742
|
+
raise _lib_accounts.TornRead()
|
|
2743
|
+
return _lib_accounts.claude_natural_id(obj.get("oauthAccount"))
|
|
2744
|
+
|
|
2745
|
+
result = _lib_accounts.stable_read_identity(path, _reader)
|
|
2746
|
+
if result.status == "identified":
|
|
2747
|
+
return _lib_accounts.account_key("claude", result.value)
|
|
2748
|
+
return _lib_accounts.UNATTRIBUTED
|
|
2749
|
+
|
|
2750
|
+
|
|
2751
|
+
def find_accounts_cutover_op():
|
|
2752
|
+
"""Scan the journal for the canonical cutover op; return its recorded
|
|
2753
|
+
``claude_legacy_account`` (spec §2 payload), or None when it has not been
|
|
2754
|
+
appended yet. Cheap enough for the one-time transition + the retry check."""
|
|
2755
|
+
for seg in list_segments():
|
|
2756
|
+
seg_path = _cctally_core.JOURNAL_DIR / seg
|
|
2757
|
+
try:
|
|
2758
|
+
size = os.path.getsize(seg_path)
|
|
2759
|
+
except OSError:
|
|
2760
|
+
continue
|
|
2761
|
+
for _name, _off, raw in _read_segment_lines(seg_path, 0, size):
|
|
2762
|
+
rec = _lib_journal.decode_line(raw)
|
|
2763
|
+
if rec is not None and rec.get("id") == CUTOVER_OP_ID:
|
|
2764
|
+
return (rec.get("payload") or {}).get("claude_legacy_account")
|
|
2765
|
+
return None
|
|
2766
|
+
|
|
2767
|
+
|
|
2768
|
+
def append_accounts_cutover_op(claude_legacy_account: str, *, at=None) -> str:
|
|
2769
|
+
"""Check-and-append the canonical cutover op (spec §2 step 2). Idempotent on
|
|
2770
|
+
the stable semantic id: if the op is already present, return its recorded
|
|
2771
|
+
value unchanged (a retry appends nothing). The real transition holds the
|
|
2772
|
+
maintenance lock, which serialises concurrent transitions; the stable id +
|
|
2773
|
+
identical payload make any residual race converge."""
|
|
2774
|
+
existing = find_accounts_cutover_op()
|
|
2775
|
+
if existing is not None:
|
|
2776
|
+
return existing
|
|
2777
|
+
rec = _lib_journal.make_op(
|
|
2778
|
+
at=(at or _now_iso()), src=_CUTOVER_OP_SRC,
|
|
2779
|
+
payload={"kind": _CUTOVER_OP_KIND,
|
|
2780
|
+
"claude_legacy_account": claude_legacy_account})
|
|
2781
|
+
rec["id"] = CUTOVER_OP_ID # override the content id with the stable token
|
|
2782
|
+
append_record(rec)
|
|
2783
|
+
return claude_legacy_account
|
|
2784
|
+
|
|
2785
|
+
|
|
2786
|
+
def resolve_cutover_claude_account() -> str:
|
|
2787
|
+
"""The single legacy Claude account the cutover op recorded, for the legacy
|
|
2788
|
+
classifier + cache backfill. Falls back to ``unattributed`` when no op is
|
|
2789
|
+
present (a fresh install with no legacy Claude history)."""
|
|
2790
|
+
value = find_accounts_cutover_op()
|
|
2791
|
+
return value if value is not None else _lib_accounts.UNATTRIBUTED
|
|
2792
|
+
|
|
2793
|
+
|
|
2794
|
+
def run_epoch_transition(*, claude_json_path=None) -> str:
|
|
2795
|
+
"""The account epoch-transition coordinator (spec §2). Resolve the cutover
|
|
2796
|
+
identity, check/append the canonical cutover op, THEN rebuild — in that exact
|
|
2797
|
+
order, so the op is inside the rebuild's input. Returns the resolved
|
|
2798
|
+
``claude_legacy_account``. Exposed for tests; the epoch-mismatch path calls
|
|
2799
|
+
it (under the maintenance + ingest locks) after quarantining the old index."""
|
|
2800
|
+
claude_key = _resolve_claude_cutover_identity(claude_json_path)
|
|
2801
|
+
recorded = append_accounts_cutover_op(claude_key)
|
|
2802
|
+
rebuild_stats_index()
|
|
2803
|
+
return recorded
|