cctally 1.83.0 → 1.84.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +2 -4
- package/bin/_cctally_account.py +144 -11
- package/bin/_cctally_alerts.py +3 -1
- package/bin/_cctally_cache.py +1396 -33
- package/bin/_cctally_dashboard.py +46 -2
- package/bin/_cctally_dashboard_conversation.py +16 -7
- package/bin/_cctally_dashboard_envelope.py +12 -0
- package/bin/_cctally_dashboard_share.py +58 -5
- package/bin/_cctally_dashboard_sources.py +641 -38
- package/bin/_cctally_db.py +173 -7
- package/bin/_cctally_doctor.py +112 -27
- package/bin/_cctally_journal.py +761 -74
- package/bin/_cctally_milestone_history.py +73 -16
- package/bin/_cctally_quota.py +313 -15
- package/bin/_cctally_source_analytics.py +4 -1
- package/bin/_cctally_tui.py +7 -3
- package/bin/_lib_doctor.py +67 -2
- package/bin/_lib_journal.py +48 -0
- package/bin/_lib_jsonl.py +119 -0
- package/bin/_lib_quota.py +222 -13
- package/bin/_lib_rederive.py +12 -0
- package/bin/_lib_source_analytics.py +13 -0
- package/bin/_lib_source_identity.py +24 -0
- package/dashboard/static/assets/index-DlVVJeS4.js +92 -0
- package/dashboard/static/assets/index-OYBkyglj.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +1 -1
- package/dashboard/static/assets/index-3bgCMVHb.js +0 -92
- package/dashboard/static/assets/index-D27EIHEI.css +0 -1
package/bin/_lib_doctor.py
CHANGED
|
@@ -184,6 +184,11 @@ class DoctorState:
|
|
|
184
184
|
# absent (pre-first-sync) or cache unreadable — check degrades OK.
|
|
185
185
|
parse_health_claude: Optional[dict] = None
|
|
186
186
|
parse_health_codex: Optional[dict] = None
|
|
187
|
+
# #416 review B4: `{"files": N, "at": iso}` written by a whole-tree Codex
|
|
188
|
+
# sync that deferred files because a `<root>/auth.json` read torn, deleted
|
|
189
|
+
# by the next whole-tree sync that defers none. None = key absent (the
|
|
190
|
+
# normal state) or cache unreadable — the check degrades OK.
|
|
191
|
+
codex_torn_deferred: Optional[dict] = None
|
|
187
192
|
# #279 S2 (F5b): PRAGMA quick_check(1) results, gathered ONLY under
|
|
188
193
|
# doctor_gather_state(deep=True) (CLI cmd_doctor) — the dashboard
|
|
189
194
|
# rebuild loop calls the gather every rebuild and quick_check on a
|
|
@@ -191,6 +196,7 @@ class DoctorState:
|
|
|
191
196
|
# "open failed: ..." | None = not run.
|
|
192
197
|
stats_db_quick_check: Optional[str] = None
|
|
193
198
|
cache_db_quick_check: Optional[str] = None
|
|
199
|
+
conversations_db_quick_check: Optional[str] = None
|
|
194
200
|
# #279 S2 (F5c): non-blocking flock probes on the two sync lock files
|
|
195
201
|
# (name -> True held / False free / None unreadable). Probe never
|
|
196
202
|
# creates files (doctor read-only contract). None = probe errored.
|
|
@@ -1899,7 +1905,9 @@ def _check_data_parse_health(s: DoctorState) -> CheckResult:
|
|
|
1899
1905
|
|
|
1900
1906
|
def _check_db_integrity(s: DoctorState) -> CheckResult:
|
|
1901
1907
|
details = {"stats_quick_check": s.stats_db_quick_check,
|
|
1902
|
-
"cache_quick_check": s.cache_db_quick_check
|
|
1908
|
+
"cache_quick_check": s.cache_db_quick_check,
|
|
1909
|
+
"conversations_quick_check":
|
|
1910
|
+
s.conversations_db_quick_check}
|
|
1903
1911
|
if s.stats_db_quick_check is not None and s.stats_db_quick_check != "ok":
|
|
1904
1912
|
return CheckResult(
|
|
1905
1913
|
id="db.integrity", title="Integrity", severity="fail",
|
|
@@ -1920,7 +1928,27 @@ def _check_db_integrity(s: DoctorState) -> CheckResult:
|
|
|
1920
1928
|
"`cctally cache-sync --rebuild`."),
|
|
1921
1929
|
details=details,
|
|
1922
1930
|
)
|
|
1923
|
-
if
|
|
1931
|
+
if (
|
|
1932
|
+
s.conversations_db_quick_check is not None
|
|
1933
|
+
and s.conversations_db_quick_check != "ok"
|
|
1934
|
+
):
|
|
1935
|
+
return CheckResult(
|
|
1936
|
+
id="db.integrity", title="Integrity", severity="warn",
|
|
1937
|
+
summary=(
|
|
1938
|
+
"conversations.db quick_check: "
|
|
1939
|
+
f"{s.conversations_db_quick_check}"
|
|
1940
|
+
),
|
|
1941
|
+
remediation=(
|
|
1942
|
+
"conversations.db is re-derivable — run "
|
|
1943
|
+
"`cctally cache-sync --rebuild`."
|
|
1944
|
+
),
|
|
1945
|
+
details=details,
|
|
1946
|
+
)
|
|
1947
|
+
if (
|
|
1948
|
+
s.stats_db_quick_check is None
|
|
1949
|
+
and s.cache_db_quick_check is None
|
|
1950
|
+
and s.conversations_db_quick_check is None
|
|
1951
|
+
):
|
|
1924
1952
|
return CheckResult(
|
|
1925
1953
|
id="db.integrity", title="Integrity", severity="ok",
|
|
1926
1954
|
summary="not checked (fast gather — run `cctally doctor`)",
|
|
@@ -2505,6 +2533,42 @@ def _check_accounts_freshness(s: DoctorState) -> CheckResult:
|
|
|
2505
2533
|
)
|
|
2506
2534
|
|
|
2507
2535
|
|
|
2536
|
+
def _check_accounts_codex_identity(s: DoctorState) -> CheckResult:
|
|
2537
|
+
"""WARN while a torn `<codex root>/auth.json` is deferring Codex ingest.
|
|
2538
|
+
|
|
2539
|
+
The defer is correct — spec §3.6's stable-read protocol never guesses an
|
|
2540
|
+
account — but since #416 a growing ALREADY-DECIDED rollout also consults
|
|
2541
|
+
`auth.json`, so a persistently torn file (truncated, half-written, replaced
|
|
2542
|
+
by a directory) halts every rollout under that root, not just the
|
|
2543
|
+
never-decided ones. `cache-sync` still exits 0, so this leg is the only
|
|
2544
|
+
standing signal that Codex spend and quota have stopped moving.
|
|
2545
|
+
|
|
2546
|
+
Not R8-gated: it names no account and adds no per-account column, so it
|
|
2547
|
+
reports at any account count (docs/accounts-gotchas.md).
|
|
2548
|
+
"""
|
|
2549
|
+
st = s.codex_torn_deferred or {}
|
|
2550
|
+
try:
|
|
2551
|
+
files = int(st.get("files") or 0)
|
|
2552
|
+
except (TypeError, ValueError):
|
|
2553
|
+
files = 0
|
|
2554
|
+
details = {"files": files, "at": st.get("at")}
|
|
2555
|
+
if files > 0:
|
|
2556
|
+
return CheckResult(
|
|
2557
|
+
id="accounts.codex_identity", title="Codex account identity",
|
|
2558
|
+
severity="warn",
|
|
2559
|
+
summary=(f"{files} Codex rollout(s) deferred — auth.json read torn; "
|
|
2560
|
+
"no usage attributed from them"),
|
|
2561
|
+
remediation=("Check the Codex auth.json (re-run `codex login` if it is "
|
|
2562
|
+
"truncated), then `cctally cache-sync --source codex`"),
|
|
2563
|
+
details=details,
|
|
2564
|
+
)
|
|
2565
|
+
return CheckResult(
|
|
2566
|
+
id="accounts.codex_identity", title="Codex account identity",
|
|
2567
|
+
severity="ok", summary="no deferred Codex ingest",
|
|
2568
|
+
remediation=None, details=details,
|
|
2569
|
+
)
|
|
2570
|
+
|
|
2571
|
+
|
|
2508
2572
|
def _check_accounts_attribution(s: DoctorState) -> CheckResult:
|
|
2509
2573
|
"""WARN when Claude usage is flowing but landing in `unattributed` despite a
|
|
2510
2574
|
resolved active account (the stamping pipeline is broken), or while the
|
|
@@ -2599,6 +2663,7 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
|
|
|
2599
2663
|
)),
|
|
2600
2664
|
("accounts", "Accounts", (
|
|
2601
2665
|
("accounts.identity", "_check_accounts_identity"),
|
|
2666
|
+
("accounts.codex_identity", "_check_accounts_codex_identity"),
|
|
2602
2667
|
("accounts.registry", "_check_accounts_registry"),
|
|
2603
2668
|
("accounts.freshness", "_check_accounts_freshness"),
|
|
2604
2669
|
("accounts.attribution", "_check_accounts_attribution"),
|
package/bin/_lib_journal.py
CHANGED
|
@@ -1118,6 +1118,54 @@ def make_account_label(
|
|
|
1118
1118
|
return make_op(at=at, src="account-label", payload=payload)
|
|
1119
1119
|
|
|
1120
1120
|
|
|
1121
|
+
def make_codex_file_account(
|
|
1122
|
+
at: str,
|
|
1123
|
+
*,
|
|
1124
|
+
root_scope: str,
|
|
1125
|
+
file_identity: str,
|
|
1126
|
+
incarnation: int,
|
|
1127
|
+
from_offset: int,
|
|
1128
|
+
account_key: "str | None" = None,
|
|
1129
|
+
) -> dict:
|
|
1130
|
+
"""Build a ``codex_file_account`` op — the DURABLE attribution decision for
|
|
1131
|
+
one byte range of one rollout incarnation (#416 spec §3.3).
|
|
1132
|
+
|
|
1133
|
+
Why this exists: the Codex account is otherwise re-derived from the live
|
|
1134
|
+
``auth.json`` on every ingest cycle, so a ``cache-sync --rebuild`` (which
|
|
1135
|
+
re-reads every rollout from offset 0, cache.db being fully re-derivable by
|
|
1136
|
+
design) re-stamps the entire history with whoever is logged in at that
|
|
1137
|
+
moment. A decision that is journaled ONCE and thereafter only replayed is
|
|
1138
|
+
immune to that.
|
|
1139
|
+
|
|
1140
|
+
The identity is ``(root_scope, file_identity, incarnation, from_offset)``,
|
|
1141
|
+
never ``(source_path, offset)`` (spec §3.2): discovery persists the first
|
|
1142
|
+
configured candidate spelling, so reordering ``$CODEX_HOME`` roots or
|
|
1143
|
+
respelling a symlink makes the same physical file miss a path-keyed map; and
|
|
1144
|
+
a truncation or root requalification resets the file to offset zero, so a
|
|
1145
|
+
permanent ``(path, offset)`` interval would overlap newly reused offsets and
|
|
1146
|
+
stamp a replacement file with the previous account. A truncation or
|
|
1147
|
+
requalification opens a NEW ``incarnation`` whose intervals can never
|
|
1148
|
+
overlap the old one.
|
|
1149
|
+
|
|
1150
|
+
Sentinel encoding follows the two-shaped stamp rule
|
|
1151
|
+
(``docs/accounts-gotchas.md``): this is an **op**, so a real account rides
|
|
1152
|
+
``payload.account_key`` and a stably-absent identity (no auth / api-key mode)
|
|
1153
|
+
is an explicit sentinel decision that OMITS the field. Never write the
|
|
1154
|
+
literal ``"unattributed"``. A **torn** read appends NOTHING at all — it is
|
|
1155
|
+
not a decision (spec §3.6).
|
|
1156
|
+
"""
|
|
1157
|
+
payload = {
|
|
1158
|
+
"kind": "codex_file_account",
|
|
1159
|
+
"root_scope": root_scope,
|
|
1160
|
+
"file_identity": file_identity,
|
|
1161
|
+
"incarnation": incarnation,
|
|
1162
|
+
"from_offset": from_offset,
|
|
1163
|
+
}
|
|
1164
|
+
if account_key is not None:
|
|
1165
|
+
payload["account_key"] = account_key
|
|
1166
|
+
return make_op(at=at, src="codex-file-account", payload=payload)
|
|
1167
|
+
|
|
1168
|
+
|
|
1121
1169
|
# --------------------------------------------------------------------------
|
|
1122
1170
|
# segment naming + canonical order
|
|
1123
1171
|
# --------------------------------------------------------------------------
|
package/bin/_lib_jsonl.py
CHANGED
|
@@ -641,6 +641,125 @@ def _codex_logical_limit_key(
|
|
|
641
641
|
return _codex_canonical_json(payload)
|
|
642
642
|
|
|
643
643
|
|
|
644
|
+
# --------------------------------------------------------------------------
|
|
645
|
+
# #416 spec §4.3 (review F8): `window_minutes` snapping.
|
|
646
|
+
#
|
|
647
|
+
# The provider occasionally reports a weekly window as `10081` rather than
|
|
648
|
+
# `10080`. `window_minutes` is a member of the logical limit key AND a column of
|
|
649
|
+
# its own, and both feed `QuotaWindowIdentity`, so one minute of jitter mints a
|
|
650
|
+
# second identity for one physical window — the second fragmentation axis behind
|
|
651
|
+
# spec §1.4 (392 live weekly blocks across 126 distinct reset-minutes, plus a
|
|
652
|
+
# stray `window_minutes = 10081`).
|
|
653
|
+
#
|
|
654
|
+
# Snapping is safe ONLY as a member-preserving replace. Rebuilding the key from
|
|
655
|
+
# limit/root/slot/minutes would drop `modelPool`, and
|
|
656
|
+
# `is_model_scoped_codex_quota` treats that member as an axis INDEPENDENT of the
|
|
657
|
+
# Spark `limit_name` — so a rebuild would file a Spark window under account
|
|
658
|
+
# weekly quota, which #373 forbids outright. Every other member (including one
|
|
659
|
+
# a future version adds) therefore survives verbatim.
|
|
660
|
+
#
|
|
661
|
+
# ±1 minute only. The next genuine boundary is 300 or 10080, so a wider
|
|
662
|
+
# tolerance buys nothing and a `10200` window is a different window, not jitter.
|
|
663
|
+
# --------------------------------------------------------------------------
|
|
664
|
+
|
|
665
|
+
CODEX_NATIVE_WINDOW_MINUTES: tuple[int, ...] = (300, 10080)
|
|
666
|
+
CODEX_WINDOW_MINUTES_SNAP_TOLERANCE = 1
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def snap_codex_window_minutes(window_minutes: object) -> object:
|
|
670
|
+
"""Snap a jittered Codex window length onto its native value.
|
|
671
|
+
|
|
672
|
+
Scalar half of the transform, used for the `window_minutes` COLUMN. Anything
|
|
673
|
+
that is not a plain positive int, or that is not within
|
|
674
|
+
``CODEX_WINDOW_MINUTES_SNAP_TOLERANCE`` of a native length, is returned
|
|
675
|
+
unchanged — the caller must be able to apply this unconditionally.
|
|
676
|
+
"""
|
|
677
|
+
if not isinstance(window_minutes, int) or isinstance(window_minutes, bool):
|
|
678
|
+
return window_minutes
|
|
679
|
+
for native in CODEX_NATIVE_WINDOW_MINUTES:
|
|
680
|
+
if abs(window_minutes - native) <= CODEX_WINDOW_MINUTES_SNAP_TOLERANCE:
|
|
681
|
+
return native
|
|
682
|
+
return window_minutes
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
def codex_snap_equivalent_limit_keys(logical_limit_key: str) -> tuple[str, ...]:
|
|
686
|
+
"""Every RAW logical limit key that ``snap_window_minutes`` maps onto the
|
|
687
|
+
same canonical key as ``logical_limit_key``.
|
|
688
|
+
|
|
689
|
+
The reset-anchor group (spec §4.2) must be the CANONICAL identity, but rows
|
|
690
|
+
are STORED under their raw key, so a group lookup has to enumerate the raw
|
|
691
|
+
spellings rather than snap in SQL. The set is bounded at three — the native
|
|
692
|
+
length plus/minus the tolerance — and contains the input itself, so a
|
|
693
|
+
non-snappable key resolves to a one-element tuple.
|
|
694
|
+
"""
|
|
695
|
+
snapped = snap_window_minutes(logical_limit_key)
|
|
696
|
+
try:
|
|
697
|
+
payload = json.loads(snapped)
|
|
698
|
+
except (json.JSONDecodeError, TypeError, ValueError):
|
|
699
|
+
return (logical_limit_key,)
|
|
700
|
+
if not isinstance(payload, dict):
|
|
701
|
+
return (logical_limit_key,)
|
|
702
|
+
native = payload.get("windowMinutes")
|
|
703
|
+
if native not in CODEX_NATIVE_WINDOW_MINUTES:
|
|
704
|
+
return (logical_limit_key,)
|
|
705
|
+
tol = CODEX_WINDOW_MINUTES_SNAP_TOLERANCE
|
|
706
|
+
out: list[str] = []
|
|
707
|
+
for minutes in range(native - tol, native + tol + 1):
|
|
708
|
+
payload["windowMinutes"] = minutes
|
|
709
|
+
out.append(_codex_canonical_json(payload))
|
|
710
|
+
return tuple(out)
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
def codex_snap_equivalent_window_minutes(window_minutes: object) -> tuple[object, ...]:
|
|
714
|
+
"""The scalar sibling of ``codex_snap_equivalent_limit_keys``: every RAW
|
|
715
|
+
``window_minutes`` value that ``snap_codex_window_minutes`` maps onto the
|
|
716
|
+
same native length as ``window_minutes``.
|
|
717
|
+
|
|
718
|
+
An identity carries the length in BOTH the logical limit key and a column of
|
|
719
|
+
its own, so a lookup that enumerates only the equivalent keys still misses a
|
|
720
|
+
stored row on the column predicate. Same bound (three values), same
|
|
721
|
+
self-inclusion for a non-snappable input.
|
|
722
|
+
"""
|
|
723
|
+
native = snap_codex_window_minutes(window_minutes)
|
|
724
|
+
if native not in CODEX_NATIVE_WINDOW_MINUTES:
|
|
725
|
+
return (window_minutes,)
|
|
726
|
+
tol = CODEX_WINDOW_MINUTES_SNAP_TOLERANCE
|
|
727
|
+
return tuple(range(native - tol, native + tol + 1))
|
|
728
|
+
|
|
729
|
+
|
|
730
|
+
def snap_window_minutes(logical_limit_key: str) -> str:
|
|
731
|
+
"""Replace ONLY the ``windowMinutes`` member of a serialized logical limit
|
|
732
|
+
key, preserving every other member verbatim.
|
|
733
|
+
|
|
734
|
+
Re-serialized through ``_codex_canonical_json`` so the byte form is exactly
|
|
735
|
+
what a natively-minted key produces: the key is a natural-key member on both
|
|
736
|
+
the journal (``_codex_quota_natural_key``) and the cache
|
|
737
|
+
(``UNIQUE(source, source_path, line_offset, logical_limit_key)``), so a
|
|
738
|
+
near-miss serialization would mint a second window rather than merge one.
|
|
739
|
+
|
|
740
|
+
Fails OPEN on shape: a key this cannot parse, or one whose ``windowMinutes``
|
|
741
|
+
is not a plain int, is returned exactly as it arrived rather than rebuilt
|
|
742
|
+
from guessed members.
|
|
743
|
+
"""
|
|
744
|
+
if not isinstance(logical_limit_key, str):
|
|
745
|
+
return logical_limit_key
|
|
746
|
+
try:
|
|
747
|
+
payload = json.loads(logical_limit_key)
|
|
748
|
+
except (json.JSONDecodeError, TypeError, ValueError):
|
|
749
|
+
return logical_limit_key
|
|
750
|
+
if not isinstance(payload, dict):
|
|
751
|
+
return logical_limit_key
|
|
752
|
+
minutes = payload.get("windowMinutes")
|
|
753
|
+
snapped = snap_codex_window_minutes(minutes)
|
|
754
|
+
if snapped == minutes and type(snapped) is type(minutes):
|
|
755
|
+
return logical_limit_key
|
|
756
|
+
payload["windowMinutes"] = snapped
|
|
757
|
+
try:
|
|
758
|
+
return _codex_canonical_json(payload)
|
|
759
|
+
except (TypeError, ValueError): # pragma: no cover — non-serializable member
|
|
760
|
+
return logical_limit_key
|
|
761
|
+
|
|
762
|
+
|
|
644
763
|
def _codex_quota_observations(
|
|
645
764
|
obj: dict[str, Any], payload: dict[str, Any], path_str: str, line_offset: int,
|
|
646
765
|
source_root_key: str | None, model: str | None,
|
package/bin/_lib_quota.py
CHANGED
|
@@ -38,6 +38,172 @@ def _integer_percent(value: float) -> int:
|
|
|
38
38
|
return math.floor(value + 1e-9)
|
|
39
39
|
|
|
40
40
|
|
|
41
|
+
# --------------------------------------------------------------------------
|
|
42
|
+
# #416 spec §4.2 — tolerance-anchored reset canonicalization (decision D4).
|
|
43
|
+
#
|
|
44
|
+
# The Codex quota path never received the jitter canonicalization the Claude 5h
|
|
45
|
+
# path has, so the RAW `resets_at` enters the window identity and one physical
|
|
46
|
+
# window splits into many, each with its own peak and milestone ladder (spec
|
|
47
|
+
# §1.4: one reset-minute observed split six ways).
|
|
48
|
+
#
|
|
49
|
+
# The tolerance is safe by construction: the next GENUINE reset of a 5h window
|
|
50
|
+
# is five hours out and of a weekly window seven days out, so nothing within
|
|
51
|
+
# 600s of an established anchor can be a different cycle.
|
|
52
|
+
#
|
|
53
|
+
# This kernel is pure and takes the established anchor set as an argument,
|
|
54
|
+
# because resolution must happen at INGEST over the complete population (spec
|
|
55
|
+
# §4.1 / review F7). A read-time anchor is deterministic only for a fixed input
|
|
56
|
+
# population, and the dashboard reads at most 35 days / 1,000 rows with those
|
|
57
|
+
# bounds applied in SQL — so omitting the earliest member of a jitter cluster
|
|
58
|
+
# would silently move its anchor and make the dashboard and the CLI disagree
|
|
59
|
+
# about window identity.
|
|
60
|
+
# --------------------------------------------------------------------------
|
|
61
|
+
|
|
62
|
+
CODEX_RESET_ANCHOR_TOLERANCE_SECONDS = 600
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def resolve_reset_anchor(
|
|
66
|
+
anchors: Iterable[dt.datetime], raw_reset: dt.datetime,
|
|
67
|
+
*, tolerance_seconds: int = CODEX_RESET_ANCHOR_TOLERANCE_SECONDS,
|
|
68
|
+
) -> dt.datetime:
|
|
69
|
+
"""Return the anchor ``raw_reset`` joins, or ``raw_reset`` itself when it
|
|
70
|
+
establishes a new one.
|
|
71
|
+
|
|
72
|
+
``anchors`` are the anchors already established for this observation's
|
|
73
|
+
identity (the identity MINUS the reset and MINUS the account — see
|
|
74
|
+
``_physical_window_key``, which excludes the account precisely so an
|
|
75
|
+
unidentified observation can be adopted by a same-window identified one).
|
|
76
|
+
|
|
77
|
+
First sight wins and an anchor NEVER moves: the returned value is always an
|
|
78
|
+
anchor already in ``anchors`` or the observation's own raw value, never a
|
|
79
|
+
recomputed centroid. Assignment picks the NEAREST anchor within tolerance,
|
|
80
|
+
breaking a tie on the earlier anchor, so for a given anchor set the answer
|
|
81
|
+
does not depend on the order the anchors were established in. (The anchor
|
|
82
|
+
SET can still depend on arrival order in the pathological case of a chain of
|
|
83
|
+
observations each within tolerance of its neighbour but not of the first;
|
|
84
|
+
real jitter is seconds wide, so a real cluster collapses to one anchor under
|
|
85
|
+
every order.)
|
|
86
|
+
"""
|
|
87
|
+
_require_aware(raw_reset, "raw_reset")
|
|
88
|
+
if (isinstance(anchors, ResetAnchorIndex)
|
|
89
|
+
and anchors.tolerance_seconds == tolerance_seconds):
|
|
90
|
+
# O(1) via the bucketed index; the linear body below stays the
|
|
91
|
+
# reference semantics for a caller holding a plain sequence, and
|
|
92
|
+
# `ResetAnchorIndex.resolve` is pinned against it by an equivalence
|
|
93
|
+
# test over a randomized population.
|
|
94
|
+
return anchors.resolve(raw_reset)
|
|
95
|
+
return _resolve_reset_anchor_linear(
|
|
96
|
+
anchors, raw_reset, tolerance_seconds=tolerance_seconds)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _resolve_reset_anchor_linear(
|
|
100
|
+
anchors: Iterable[dt.datetime], raw_reset: dt.datetime,
|
|
101
|
+
*, tolerance_seconds: int,
|
|
102
|
+
) -> dt.datetime:
|
|
103
|
+
best: dt.datetime | None = None
|
|
104
|
+
best_delta: float | None = None
|
|
105
|
+
for anchor in anchors:
|
|
106
|
+
_require_aware(anchor, "anchor")
|
|
107
|
+
delta = abs((raw_reset - anchor).total_seconds())
|
|
108
|
+
if delta > tolerance_seconds:
|
|
109
|
+
continue
|
|
110
|
+
if best is None or delta < best_delta or (
|
|
111
|
+
delta == best_delta and anchor < best
|
|
112
|
+
):
|
|
113
|
+
best, best_delta = anchor, delta
|
|
114
|
+
return raw_reset if best is None else best
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class ResetAnchorIndex:
|
|
118
|
+
"""An established-anchor set with O(1) ``resolve``.
|
|
119
|
+
|
|
120
|
+
Semantically identical to ``resolve_reset_anchor`` over the same anchors —
|
|
121
|
+
nearest within tolerance, ties broken on the earlier anchor, first sight
|
|
122
|
+
wins, the anchor never moves, and the pathological chain-of-neighbours
|
|
123
|
+
order-dependence that docstring documents is preserved exactly, because the
|
|
124
|
+
anchor SET is still whatever the caller established in whatever order.
|
|
125
|
+
|
|
126
|
+
Only the LOOKUP changes. The linear scan did a full ``datetime`` subtraction
|
|
127
|
+
against every established anchor, and a 5h group accumulates ~1,750 anchors
|
|
128
|
+
a year — so the scan is quadratic in the population. That is not a
|
|
129
|
+
background cost: cache migration 032 runs the resolver over the WHOLE
|
|
130
|
+
``quota_window_snapshots`` table synchronously on the first DB open after
|
|
131
|
+
upgrade, and ``cache-sync --rebuild`` runs it over the whole walk, so the
|
|
132
|
+
quadratic term lands on the operator's first post-upgrade command as a
|
|
133
|
+
multi-minute hang.
|
|
134
|
+
|
|
135
|
+
Anchors are bucketed by ``floor(epoch / tolerance)``. Anything within
|
|
136
|
+
``tolerance`` of a probe lies in the probe's own bucket or one on either
|
|
137
|
+
side, so three bucket lookups are EXHAUSTIVE — this is an exact index, not
|
|
138
|
+
an approximation, and no correctness argument depends on the bucket width
|
|
139
|
+
beyond that identity.
|
|
140
|
+
"""
|
|
141
|
+
|
|
142
|
+
__slots__ = ("_tolerance", "_buckets", "_order")
|
|
143
|
+
|
|
144
|
+
def __init__(
|
|
145
|
+
self, anchors: Iterable[dt.datetime] = (),
|
|
146
|
+
*, tolerance_seconds: int = CODEX_RESET_ANCHOR_TOLERANCE_SECONDS,
|
|
147
|
+
) -> None:
|
|
148
|
+
if not isinstance(tolerance_seconds, int) or isinstance(tolerance_seconds, bool):
|
|
149
|
+
raise ValueError("tolerance_seconds must be an int")
|
|
150
|
+
if tolerance_seconds <= 0:
|
|
151
|
+
raise ValueError("tolerance_seconds must be positive")
|
|
152
|
+
self._tolerance = tolerance_seconds
|
|
153
|
+
self._buckets: dict[int, list[dt.datetime]] = {}
|
|
154
|
+
self._order: list[dt.datetime] = []
|
|
155
|
+
for anchor in anchors:
|
|
156
|
+
self.add(anchor)
|
|
157
|
+
|
|
158
|
+
@property
|
|
159
|
+
def tolerance_seconds(self) -> int:
|
|
160
|
+
return self._tolerance
|
|
161
|
+
|
|
162
|
+
def _bucket(self, value: dt.datetime) -> int:
|
|
163
|
+
return int(value.timestamp() // self._tolerance)
|
|
164
|
+
|
|
165
|
+
def add(self, anchor: dt.datetime) -> bool:
|
|
166
|
+
"""Establish ``anchor``. Returns False when it was already present."""
|
|
167
|
+
_require_aware(anchor, "anchor")
|
|
168
|
+
bucket = self._buckets.setdefault(self._bucket(anchor), [])
|
|
169
|
+
if anchor in bucket:
|
|
170
|
+
return False
|
|
171
|
+
bucket.append(anchor)
|
|
172
|
+
self._order.append(anchor)
|
|
173
|
+
return True
|
|
174
|
+
|
|
175
|
+
def resolve(self, raw_reset: dt.datetime) -> dt.datetime:
|
|
176
|
+
"""The anchor ``raw_reset`` joins, or ``raw_reset`` when it establishes
|
|
177
|
+
a new one. Does NOT add it — the caller decides, exactly as with the
|
|
178
|
+
pure function."""
|
|
179
|
+
_require_aware(raw_reset, "raw_reset")
|
|
180
|
+
probe = self._bucket(raw_reset)
|
|
181
|
+
best: dt.datetime | None = None
|
|
182
|
+
best_delta: float | None = None
|
|
183
|
+
for offset in (-1, 0, 1):
|
|
184
|
+
for anchor in self._buckets.get(probe + offset, ()):
|
|
185
|
+
delta = abs((raw_reset - anchor).total_seconds())
|
|
186
|
+
if delta > self._tolerance:
|
|
187
|
+
continue
|
|
188
|
+
if best is None or delta < best_delta or (
|
|
189
|
+
delta == best_delta and anchor < best
|
|
190
|
+
):
|
|
191
|
+
best, best_delta = anchor, delta
|
|
192
|
+
return raw_reset if best is None else best
|
|
193
|
+
|
|
194
|
+
def __contains__(self, anchor: object) -> bool:
|
|
195
|
+
if not isinstance(anchor, dt.datetime):
|
|
196
|
+
return False
|
|
197
|
+
return anchor in self._buckets.get(self._bucket(anchor), ())
|
|
198
|
+
|
|
199
|
+
def __iter__(self):
|
|
200
|
+
"""Established anchors in the order they were established."""
|
|
201
|
+
return iter(self._order)
|
|
202
|
+
|
|
203
|
+
def __len__(self) -> int:
|
|
204
|
+
return len(self._order)
|
|
205
|
+
|
|
206
|
+
|
|
41
207
|
@dataclass(frozen=True)
|
|
42
208
|
class QuotaWindowIdentity:
|
|
43
209
|
"""One root-qualified native quota window identity.
|
|
@@ -87,10 +253,25 @@ class QuotaObservation:
|
|
|
87
253
|
plan_type: str | None = None
|
|
88
254
|
individual_limit_json: str | None = None
|
|
89
255
|
reached_type: str | None = None
|
|
256
|
+
# #416 spec §4.1/§4.2: the tolerance-anchored reset resolved at INGEST over
|
|
257
|
+
# the complete population and stored on the cache row. ``resets_at`` stays
|
|
258
|
+
# the RAW provider value, retained unchanged as evidence.
|
|
259
|
+
#
|
|
260
|
+
# Every reset-identity consumer reads THIS field, not ``resets_at``, and
|
|
261
|
+
# ``__post_init__`` fills it from ``resets_at`` when the caller has none —
|
|
262
|
+
# so a pre-migration row, an older binary's row, or any construction site
|
|
263
|
+
# that predates the column behaves exactly as it does today rather than
|
|
264
|
+
# failing. That default is what makes "route every consumer through the
|
|
265
|
+
# anchor" a safe blanket rule instead of a per-call-site judgement.
|
|
266
|
+
canonical_resets_at: dt.datetime | None = None
|
|
90
267
|
|
|
91
268
|
def __post_init__(self) -> None:
|
|
92
269
|
_require_aware(self.captured_at, "captured_at")
|
|
93
270
|
_require_aware(self.resets_at, "resets_at")
|
|
271
|
+
if self.canonical_resets_at is None:
|
|
272
|
+
object.__setattr__(self, "canonical_resets_at", self.resets_at)
|
|
273
|
+
else:
|
|
274
|
+
_require_aware(self.canonical_resets_at, "canonical_resets_at")
|
|
94
275
|
if not isinstance(self.used_percent, (int, float)) or isinstance(self.used_percent, bool):
|
|
95
276
|
raise ValueError("used_percent must be a number")
|
|
96
277
|
if not math.isfinite(self.used_percent) or not 0 <= self.used_percent <= 100:
|
|
@@ -221,10 +402,16 @@ def identity_sort_key(identity: QuotaWindowIdentity) -> tuple[str, str, str, str
|
|
|
221
402
|
|
|
222
403
|
|
|
223
404
|
def physical_order_key(observation: QuotaObservation) -> tuple[dt.datetime, dt.datetime, str, int]:
|
|
224
|
-
"""The frozen total order for physical rows within one identity.
|
|
405
|
+
"""The frozen total order for physical rows within one identity.
|
|
406
|
+
|
|
407
|
+
The reset component is the CANONICAL anchor (#416 §4.1), so two captures of
|
|
408
|
+
one physical window whose raw resets differ only by provider jitter tie here
|
|
409
|
+
and fall through to the deterministic physical position
|
|
410
|
+
(``source_path``, ``line_offset``) instead of being ordered by the jitter.
|
|
411
|
+
"""
|
|
225
412
|
return (
|
|
226
413
|
observation.captured_at,
|
|
227
|
-
observation.
|
|
414
|
+
observation.canonical_resets_at,
|
|
228
415
|
observation.source_path,
|
|
229
416
|
observation.line_offset,
|
|
230
417
|
)
|
|
@@ -242,7 +429,10 @@ def logical_value_tuple(observation: QuotaObservation) -> tuple[object, ...]:
|
|
|
242
429
|
identity.limit_id,
|
|
243
430
|
identity.limit_name,
|
|
244
431
|
observation.used_percent,
|
|
245
|
-
|
|
432
|
+
# CANONICAL, not raw (#416 §4.1): the reset is part of the interpreted
|
|
433
|
+
# point's value, so raw jitter would make one unchanged reading look
|
|
434
|
+
# like a run of distinct interpreted points and defeat the dedup.
|
|
435
|
+
observation.canonical_resets_at,
|
|
246
436
|
observation.plan_type,
|
|
247
437
|
observation.individual_limit_json,
|
|
248
438
|
observation.reached_type,
|
|
@@ -289,9 +479,13 @@ def _physical_window_key(observation: QuotaObservation) -> tuple[object, ...]:
|
|
|
289
479
|
"""The account-INDEPENDENT physical window key used by the continuity fold.
|
|
290
480
|
|
|
291
481
|
Two observations share a physical window iff they agree on root, limit key,
|
|
292
|
-
slot, window minutes, and the
|
|
293
|
-
|
|
294
|
-
|
|
482
|
+
slot, window minutes, and the reset boundary — the account is deliberately
|
|
483
|
+
EXCLUDED so unidentified observations can be adopted by a same-window
|
|
484
|
+
identified account (spec §2 window-account continuity).
|
|
485
|
+
|
|
486
|
+
The reset is the CANONICAL anchor (#416 §4.1). With the raw value, jitter
|
|
487
|
+
ALONE defeats continuity adoption: an unidentified observation a few seconds
|
|
488
|
+
off the identified one is a different physical window and is never adopted.
|
|
295
489
|
"""
|
|
296
490
|
identity = observation.identity
|
|
297
491
|
return (
|
|
@@ -300,7 +494,7 @@ def _physical_window_key(observation: QuotaObservation) -> tuple[object, ...]:
|
|
|
300
494
|
identity.logical_limit_key,
|
|
301
495
|
identity.observed_slot,
|
|
302
496
|
identity.window_minutes,
|
|
303
|
-
observation.
|
|
497
|
+
observation.canonical_resets_at,
|
|
304
498
|
)
|
|
305
499
|
|
|
306
500
|
|
|
@@ -376,11 +570,18 @@ def select_baseline(
|
|
|
376
570
|
|
|
377
571
|
|
|
378
572
|
def build_blocks(observations: Iterable[QuotaObservation]) -> tuple[QuotaBlock, ...]:
|
|
379
|
-
"""Segment deduplicated interpreted history at each native reset boundary.
|
|
573
|
+
"""Segment deduplicated interpreted history at each native reset boundary.
|
|
574
|
+
|
|
575
|
+
Blocks are keyed on the CANONICAL anchor (#416 §4.1), so one physical window
|
|
576
|
+
is one block however many raw reset spellings its observations carry — and
|
|
577
|
+
``QuotaBlock.resets_at``, which every renderer shows, is that same anchor.
|
|
578
|
+
"""
|
|
380
579
|
by_block: dict[tuple[QuotaWindowIdentity, dt.datetime], list[QuotaObservation]] = {}
|
|
381
580
|
for history in build_history(observations):
|
|
382
581
|
for observation in history.observations:
|
|
383
|
-
by_block.setdefault(
|
|
582
|
+
by_block.setdefault(
|
|
583
|
+
(history.identity, observation.canonical_resets_at), []
|
|
584
|
+
).append(observation)
|
|
384
585
|
|
|
385
586
|
blocks: list[QuotaBlock] = []
|
|
386
587
|
for (identity, resets_at), points in sorted(
|
|
@@ -495,9 +696,13 @@ def forecast_quota(
|
|
|
495
696
|
if baseline is None:
|
|
496
697
|
return _null_forecast("future" if freshness.state == "future" else "unavailable")
|
|
497
698
|
|
|
699
|
+
# Same-cycle selection on the CANONICAL anchor (#416 §4.1). With the raw
|
|
700
|
+
# value, a jittered cycle's evidence splits and the forecast silently runs
|
|
701
|
+
# on a fraction of the samples it should have.
|
|
498
702
|
points = tuple(
|
|
499
703
|
observation for observation in history.observations
|
|
500
|
-
if observation.
|
|
704
|
+
if observation.canonical_resets_at == baseline.canonical_resets_at
|
|
705
|
+
and observation.captured_at <= as_of
|
|
501
706
|
)
|
|
502
707
|
points = tuple(sorted(points, key=physical_order_key))
|
|
503
708
|
sample_count = 0
|
|
@@ -507,9 +712,13 @@ def forecast_quota(
|
|
|
507
712
|
if elapsed_seconds > 0 and delta_percent > 0:
|
|
508
713
|
sample_count += 1
|
|
509
714
|
|
|
510
|
-
|
|
715
|
+
# The cycle geometry rides the same anchor, so the countdown, the elapsed
|
|
716
|
+
# fraction, and the reported reset all describe ONE window rather than
|
|
717
|
+
# whichever jittered spelling the baseline observation happened to carry.
|
|
718
|
+
anchor = baseline.canonical_resets_at
|
|
719
|
+
remaining_seconds = max(0, int((anchor - as_of).total_seconds()))
|
|
511
720
|
native_window_seconds = baseline.identity.window_minutes * 60
|
|
512
|
-
cycle_start =
|
|
721
|
+
cycle_start = anchor - dt.timedelta(minutes=baseline.identity.window_minutes)
|
|
513
722
|
cycle_elapsed_seconds = min(
|
|
514
723
|
float(native_window_seconds),
|
|
515
724
|
max(0.0, (as_of - cycle_start).total_seconds()),
|
|
@@ -544,7 +753,7 @@ def forecast_quota(
|
|
|
544
753
|
current_percent=baseline.used_percent,
|
|
545
754
|
rate_percent_per_hour=rate,
|
|
546
755
|
projected_percent=projected,
|
|
547
|
-
resets_at=
|
|
756
|
+
resets_at=anchor,
|
|
548
757
|
remaining_seconds=remaining_seconds,
|
|
549
758
|
sample_count=sample_count,
|
|
550
759
|
sample_span_seconds=int(cycle_elapsed_seconds) if sample_count > 0 else 0,
|
package/bin/_lib_rederive.py
CHANGED
|
@@ -67,6 +67,12 @@ _EVT_CLASSIFICATIONS = {
|
|
|
67
67
|
"historical project budgets are not journaled; stale latches retire"),
|
|
68
68
|
"quota_alert_arming": KindClassification(
|
|
69
69
|
"retained", "Codex quota lifecycle state is outside claude-usage"),
|
|
70
|
+
# #416 spec §7.2: terminal Codex quota alert evidence. `retained`, exactly
|
|
71
|
+
# like its `quota_alert_arming` sibling — it is provider state OUTSIDE the
|
|
72
|
+
# claude-usage family, replayed by its own fold applier rather than
|
|
73
|
+
# re-derived into the claude-usage scratch index.
|
|
74
|
+
"quota_threshold_event": KindClassification(
|
|
75
|
+
"retained", "Codex terminal quota alert evidence is outside claude-usage"),
|
|
70
76
|
}
|
|
71
77
|
|
|
72
78
|
_OP_CLASSIFICATIONS = {
|
|
@@ -81,6 +87,12 @@ _OP_CLASSIFICATIONS = {
|
|
|
81
87
|
"legacy Claude ownership normalizes unstamped journal history"),
|
|
82
88
|
"sync_week": KindClassification(
|
|
83
89
|
"retained_input", "operator cost-sync request is re-executed in scratch"),
|
|
90
|
+
# #416: the durable Codex file/range attribution decision. `retained` (not
|
|
91
|
+
# `retained_input`) mirrors the `quota_alert_arming` precedent above — it is
|
|
92
|
+
# provider state OUTSIDE the claude-usage family, replayed by the Codex cache
|
|
93
|
+
# leg rather than into the claude-usage scratch index.
|
|
94
|
+
"codex_file_account": KindClassification(
|
|
95
|
+
"retained", "Codex file attribution decision is outside claude-usage"),
|
|
84
96
|
}
|
|
85
97
|
|
|
86
98
|
|
|
@@ -66,6 +66,19 @@ class QualifiedCodexEntry:
|
|
|
66
66
|
# source-analytics renderers never expose either raw identity.
|
|
67
67
|
session_id: str = ""
|
|
68
68
|
source_path: str = ""
|
|
69
|
+
# #416 spec §5.2 (review F9): the ingest-stamped account, carried so the
|
|
70
|
+
# dashboard can partition ALREADY-LOADED rows by account instead of running
|
|
71
|
+
# a second query shape per account.
|
|
72
|
+
#
|
|
73
|
+
# CARRIED, never grouped. It is a trailing defaulted field and it is
|
|
74
|
+
# deliberately absent from `project_key`, from the `(root, project)` project
|
|
75
|
+
# grouping, from the `(source_root_key, source_path)` session grouping, and
|
|
76
|
+
# from every aggregator key — the merged parent stays byte-identical BY
|
|
77
|
+
# CONSTRUCTION rather than by careful re-derivation. Adding it to a grouping
|
|
78
|
+
# key would silently split every merged bucket a file that switches accounts
|
|
79
|
+
# contributes to. The default is the reserved `unattributed` sentinel, so
|
|
80
|
+
# every existing constructor (and the whole CLI path) is unchanged.
|
|
81
|
+
account_key: str = "unattributed"
|
|
69
82
|
|
|
70
83
|
|
|
71
84
|
def emitted_project_label(entry: QualifiedCodexEntry) -> str:
|