cctally 1.83.0 → 1.84.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +2 -4
- package/bin/_cctally_account.py +144 -11
- package/bin/_cctally_alerts.py +3 -1
- package/bin/_cctally_cache.py +1396 -33
- package/bin/_cctally_dashboard.py +46 -2
- package/bin/_cctally_dashboard_conversation.py +16 -7
- package/bin/_cctally_dashboard_envelope.py +12 -0
- package/bin/_cctally_dashboard_share.py +58 -5
- package/bin/_cctally_dashboard_sources.py +641 -38
- package/bin/_cctally_db.py +173 -7
- package/bin/_cctally_doctor.py +112 -27
- package/bin/_cctally_journal.py +761 -74
- package/bin/_cctally_milestone_history.py +73 -16
- package/bin/_cctally_quota.py +313 -15
- package/bin/_cctally_source_analytics.py +4 -1
- package/bin/_cctally_tui.py +7 -3
- package/bin/_lib_doctor.py +67 -2
- package/bin/_lib_journal.py +48 -0
- package/bin/_lib_jsonl.py +119 -0
- package/bin/_lib_quota.py +222 -13
- package/bin/_lib_rederive.py +12 -0
- package/bin/_lib_source_analytics.py +13 -0
- package/bin/_lib_source_identity.py +24 -0
- package/dashboard/static/assets/index-DlVVJeS4.js +92 -0
- package/dashboard/static/assets/index-OYBkyglj.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +1 -1
- package/dashboard/static/assets/index-3bgCMVHb.js +0 -92
- package/dashboard/static/assets/index-D27EIHEI.css +0 -1
package/bin/_cctally_quota.py
CHANGED
|
@@ -21,6 +21,7 @@ import _cctally_core
|
|
|
21
21
|
import _lib_accounts
|
|
22
22
|
from _cctally_core import _command_as_of, eprint
|
|
23
23
|
from _lib_quota import (
|
|
24
|
+
CODEX_RESET_ANCHOR_TOLERANCE_SECONDS,
|
|
24
25
|
QuotaBlock,
|
|
25
26
|
QuotaForecast,
|
|
26
27
|
QuotaFreshness,
|
|
@@ -43,7 +44,13 @@ from _lib_quota import (
|
|
|
43
44
|
source_path_key,
|
|
44
45
|
)
|
|
45
46
|
from _lib_json_envelope import stamp_schema_version
|
|
46
|
-
from _lib_jsonl import
|
|
47
|
+
from _lib_jsonl import (
|
|
48
|
+
_codex_logical_limit_key,
|
|
49
|
+
codex_snap_equivalent_limit_keys,
|
|
50
|
+
codex_snap_equivalent_window_minutes,
|
|
51
|
+
snap_codex_window_minutes,
|
|
52
|
+
snap_window_minutes,
|
|
53
|
+
)
|
|
47
54
|
from _lib_codex_pools import (
|
|
48
55
|
codex_history_is_model_scoped,
|
|
49
56
|
codex_model_scoped_quota_pool,
|
|
@@ -298,6 +305,17 @@ def load_codex_quota_observations(
|
|
|
298
305
|
account_expr = (
|
|
299
306
|
"account_key" if has_account else "NULL AS account_key"
|
|
300
307
|
)
|
|
308
|
+
# canonical_resets_at_utc (#416 §4.2): resolved at ingest over the
|
|
309
|
+
# complete population. A cache that has not yet gained the column (or a
|
|
310
|
+
# row the 032 backfill has not reached) reads NULL, and
|
|
311
|
+
# `QuotaObservation` then falls back to the raw reset — exactly today's
|
|
312
|
+
# behaviour, never a failure.
|
|
313
|
+
has_anchor = has_columns(
|
|
314
|
+
"quota_window_snapshots", {"canonical_resets_at_utc"})
|
|
315
|
+
anchor_expr = (
|
|
316
|
+
"canonical_resets_at_utc" if has_anchor
|
|
317
|
+
else "NULL AS canonical_resets_at_utc"
|
|
318
|
+
)
|
|
301
319
|
entry_lookup = (
|
|
302
320
|
"(SELECT entries.model FROM codex_session_entries AS entries "
|
|
303
321
|
"WHERE entries.source_path=quota_window_snapshots.source_path "
|
|
@@ -323,10 +341,11 @@ def load_codex_quota_observations(
|
|
|
323
341
|
captured_at_utc, observed_slot, logical_limit_key, limit_id,
|
|
324
342
|
limit_name, window_minutes, used_percent, resets_at_utc,
|
|
325
343
|
plan_type, individual_limit_json, reached_type,
|
|
326
|
-
{model_expr}, {account_expr}
|
|
344
|
+
{model_expr}, {account_expr}, {anchor_expr}
|
|
327
345
|
FROM quota_window_snapshots
|
|
328
346
|
WHERE source='codex' AND source_root_key IS NOT NULL
|
|
329
|
-
""".format(model_expr=model_expr, account_expr=account_expr
|
|
347
|
+
""".format(model_expr=model_expr, account_expr=account_expr,
|
|
348
|
+
anchor_expr=anchor_expr)
|
|
330
349
|
params: list[object] = []
|
|
331
350
|
if requested is not None:
|
|
332
351
|
if not requested:
|
|
@@ -374,11 +393,28 @@ def load_codex_quota_observations(
|
|
|
374
393
|
if any(row[name] is None or not str(row[name]).strip() for name in required_text):
|
|
375
394
|
continue
|
|
376
395
|
try:
|
|
377
|
-
|
|
396
|
+
# #416 spec §4.3: snap a jittered `window_minutes` (the stray
|
|
397
|
+
# `10081`) onto its native length, in BOTH the column and the
|
|
398
|
+
# key member — an identity carries both, so snapping one alone
|
|
399
|
+
# would still leave two identities for one physical window.
|
|
400
|
+
#
|
|
401
|
+
# This runs on the READ path deliberately. It is a PURE PER-ROW
|
|
402
|
+
# function with no population dependence, so §4.1's argument
|
|
403
|
+
# against read-time canonicalization (a bounded read picks a
|
|
404
|
+
# different first member of a jitter cluster, so the dashboard
|
|
405
|
+
# and CLI disagree) does not apply to it. Snapping at ingest
|
|
406
|
+
# instead would change the journal quota natural key AND the
|
|
407
|
+
# cache UNIQUE key, so `--rebuild` would re-append every
|
|
408
|
+
# already-journalled observation under a new key and materialize
|
|
409
|
+
# BOTH forms — reintroducing the fragmentation being removed.
|
|
410
|
+
window_minutes = snap_codex_window_minutes(
|
|
411
|
+
int(row["window_minutes"]))
|
|
412
|
+
logical_limit_key = snap_window_minutes(
|
|
413
|
+
str(row["logical_limit_key"]))
|
|
378
414
|
if codex_model_scoped_quota_pool(row["observed_model"]) is not None:
|
|
379
415
|
logical_limit_key = _codex_logical_limit_key(
|
|
380
416
|
str(row["source_root_key"]), row["limit_id"],
|
|
381
|
-
str(row["observed_slot"]),
|
|
417
|
+
str(row["observed_slot"]), window_minutes,
|
|
382
418
|
str(row["observed_model"]),
|
|
383
419
|
)
|
|
384
420
|
raw_account = row["account_key"]
|
|
@@ -392,7 +428,7 @@ def load_codex_quota_observations(
|
|
|
392
428
|
account_key=account_key,
|
|
393
429
|
logical_limit_key=logical_limit_key,
|
|
394
430
|
observed_slot=str(row["observed_slot"]),
|
|
395
|
-
window_minutes=
|
|
431
|
+
window_minutes=window_minutes,
|
|
396
432
|
limit_id=row["limit_id"],
|
|
397
433
|
limit_name=row["limit_name"],
|
|
398
434
|
)
|
|
@@ -406,6 +442,11 @@ def load_codex_quota_observations(
|
|
|
406
442
|
plan_type=row["plan_type"],
|
|
407
443
|
individual_limit_json=row["individual_limit_json"],
|
|
408
444
|
reached_type=row["reached_type"],
|
|
445
|
+
canonical_resets_at=(
|
|
446
|
+
None if row["canonical_resets_at_utc"] in (None, "")
|
|
447
|
+
else _parse_utc(str(row["canonical_resets_at_utc"]),
|
|
448
|
+
"canonical_resets_at_utc")
|
|
449
|
+
),
|
|
409
450
|
)
|
|
410
451
|
except (TypeError, ValueError, OverflowError):
|
|
411
452
|
# Physical retention is intentionally more permissive than the
|
|
@@ -696,8 +737,22 @@ def _insert_quota_terminal_event(
|
|
|
696
737
|
*, identity: QuotaWindowIdentity, resets_at: dt.datetime,
|
|
697
738
|
threshold: int, kind: str, qualifying_percent: float | None,
|
|
698
739
|
projected_percent: float | None, disposition: str, now_iso: str,
|
|
740
|
+
journal_emit=None,
|
|
699
741
|
) -> bool:
|
|
700
|
-
"""Claim one durable threshold lifecycle row; unique-key races converge.
|
|
742
|
+
"""Claim one durable threshold lifecycle row; unique-key races converge.
|
|
743
|
+
|
|
744
|
+
#416 spec §7.2 (review F13): a claimed row is TERMINAL alert evidence and
|
|
745
|
+
must survive a stats.db rebuild, so a genuine claim is journaled through
|
|
746
|
+
``journal_emit``. Without it a rebuild could not recreate an ``alerted`` row
|
|
747
|
+
at all — `rematerialize_quota_projection_for_rebuild` runs with no
|
|
748
|
+
alert-eligible roots — and the crossing would be free to fire again.
|
|
749
|
+
|
|
750
|
+
``journal_emit`` is set only on the LIVE ingest-cycle path and is ``None``
|
|
751
|
+
for the rebuild re-materialization, which must never append. It fires only
|
|
752
|
+
when ``rowcount == 1``, i.e. on a genuinely NEW claim: re-emitting on a
|
|
753
|
+
converged race would append a duplicate record for a fact already journaled.
|
|
754
|
+
Same shape as ``_activate_quota_rule``'s arming emitter.
|
|
755
|
+
"""
|
|
701
756
|
alerted_at = now_iso if disposition == "alerted" else None
|
|
702
757
|
suppressed_at = now_iso if disposition == "suppressed_backfill" else None
|
|
703
758
|
cur = conn.execute(
|
|
@@ -715,7 +770,27 @@ def _insert_quota_terminal_event(
|
|
|
715
770
|
alerted_at, suppressed_at, identity.account_key,
|
|
716
771
|
),
|
|
717
772
|
)
|
|
718
|
-
|
|
773
|
+
claimed = cur.rowcount == 1
|
|
774
|
+
if claimed and journal_emit is not None:
|
|
775
|
+
journal_emit({
|
|
776
|
+
"source": identity.source,
|
|
777
|
+
"source_root_key": identity.source_root_key,
|
|
778
|
+
"account_key": identity.account_key,
|
|
779
|
+
"logical_limit_key": identity.logical_limit_key,
|
|
780
|
+
"observed_slot": identity.observed_slot,
|
|
781
|
+
"window_minutes": identity.window_minutes,
|
|
782
|
+
"resets_at_utc": _utc_iso(resets_at),
|
|
783
|
+
"threshold": threshold,
|
|
784
|
+
"qualifying_kind": kind,
|
|
785
|
+
"qualifying_percent": qualifying_percent,
|
|
786
|
+
"projected_percent": projected_percent,
|
|
787
|
+
"severity": _cctally().severity_for(threshold),
|
|
788
|
+
"created_at_utc": now_iso,
|
|
789
|
+
"disposition": disposition,
|
|
790
|
+
"alerted_at": alerted_at,
|
|
791
|
+
"suppressed_at": suppressed_at,
|
|
792
|
+
})
|
|
793
|
+
return claimed
|
|
719
794
|
|
|
720
795
|
|
|
721
796
|
def _block_observations_at_or_before(
|
|
@@ -754,6 +829,7 @@ def _evaluate_quota_alerts(
|
|
|
754
829
|
conn: sqlite3.Connection,
|
|
755
830
|
*, observations: tuple[QuotaObservation, ...], alert_eligible_roots: set[str],
|
|
756
831
|
now: dt.datetime, now_iso: str, journal_emit=None, journal_disarm=None,
|
|
832
|
+
journal_terminal=None,
|
|
757
833
|
) -> list[dict]:
|
|
758
834
|
"""Arm or claim quota alerts within the caller's stats transaction.
|
|
759
835
|
|
|
@@ -848,6 +924,7 @@ def _evaluate_quota_alerts(
|
|
|
848
924
|
projected_percent if decision.kind == "projected" else None
|
|
849
925
|
),
|
|
850
926
|
disposition="suppressed_backfill", now_iso=now_iso,
|
|
927
|
+
journal_emit=journal_terminal,
|
|
851
928
|
)
|
|
852
929
|
continue
|
|
853
930
|
later = tuple(point for point in present if point.captured_at > activated_at)
|
|
@@ -858,7 +935,10 @@ def _evaluate_quota_alerts(
|
|
|
858
935
|
baseline = select_baseline(history.observations, now)
|
|
859
936
|
if (
|
|
860
937
|
freshness.state != "stale" and baseline is not None
|
|
861
|
-
|
|
938
|
+
# CANONICAL on both sides (#416 §4.1): `block.resets_at` is
|
|
939
|
+
# now the anchor, so comparing it to the baseline's RAW reset
|
|
940
|
+
# would never match for a jittered window.
|
|
941
|
+
and baseline.canonical_resets_at == block.resets_at
|
|
862
942
|
and baseline.captured_at > activated_at
|
|
863
943
|
):
|
|
864
944
|
projected_percent = _quota_projection_for_block(history, block, now)
|
|
@@ -875,6 +955,7 @@ def _evaluate_quota_alerts(
|
|
|
875
955
|
threshold=decision.threshold, kind=decision.kind,
|
|
876
956
|
qualifying_percent=qualifying, projected_percent=projected,
|
|
877
957
|
disposition="alerted", now_iso=now_iso,
|
|
958
|
+
journal_emit=journal_terminal,
|
|
878
959
|
):
|
|
879
960
|
queued.append(_quota_alert_payload(
|
|
880
961
|
identity=identity, resets_at=block.resets_at,
|
|
@@ -885,10 +966,89 @@ def _evaluate_quota_alerts(
|
|
|
885
966
|
return queued
|
|
886
967
|
|
|
887
968
|
|
|
969
|
+
def _reanchor_terminal_events_sql(key_slots: int, minute_slots: int) -> str:
|
|
970
|
+
# `UPDATE OR IGNORE`, not a plain UPDATE: if this identity already carries an
|
|
971
|
+
# anchored row at the same threshold, moving the jittered twin onto it would
|
|
972
|
+
# violate the UNIQUE key. OR IGNORE SKIPS that move (it does not delete the
|
|
973
|
+
# twin), which is the right trade: the anchored row survives with its
|
|
974
|
+
# evidence intact and is the one every future evaluation keys against, so
|
|
975
|
+
# the re-fire is prevented either way — while deleting historical alert
|
|
976
|
+
# evidence to tidy the display would be irreversible and is not this pass's
|
|
977
|
+
# mandate. A plain UPDATE would raise and abort the whole projection
|
|
978
|
+
# transaction.
|
|
979
|
+
keys = ",".join(f":key{i}" for i in range(key_slots))
|
|
980
|
+
minutes = ",".join(f":min{i}" for i in range(minute_slots))
|
|
981
|
+
return (
|
|
982
|
+
"UPDATE OR IGNORE quota_threshold_events "
|
|
983
|
+
" SET resets_at_utc = :anchor, "
|
|
984
|
+
" logical_limit_key = :limit_key, "
|
|
985
|
+
" window_minutes = :minutes "
|
|
986
|
+
" WHERE source = :source AND source_root_key = :root "
|
|
987
|
+
" AND account_key = :account "
|
|
988
|
+
f" AND logical_limit_key IN ({keys}) "
|
|
989
|
+
f" AND observed_slot = :slot AND window_minutes IN ({minutes}) "
|
|
990
|
+
" AND (resets_at_utc <> :anchor OR logical_limit_key <> :limit_key "
|
|
991
|
+
" OR window_minutes <> :minutes) "
|
|
992
|
+
" AND abs(unixepoch(resets_at_utc) - unixepoch(:anchor)) <= :tolerance"
|
|
993
|
+
)
|
|
994
|
+
|
|
995
|
+
|
|
996
|
+
def _reanchor_terminal_events(conn: sqlite3.Connection, block) -> None:
|
|
997
|
+
"""Move terminal alert evidence for one window onto its canonical identity.
|
|
998
|
+
|
|
999
|
+
#416 spec §4.1 made `QuotaBlock.resets_at` the tolerance-anchored reset, but
|
|
1000
|
+
`quota_threshold_events.resets_at_utc` is part of that table's UNIQUE key and
|
|
1001
|
+
existing rows were written under whichever RAW spelling the block carried at
|
|
1002
|
+
the time. Without this, the very next reconcile after the canonicalization
|
|
1003
|
+
ships would look up an already-alerted threshold under the anchor, find
|
|
1004
|
+
nothing, claim it again, and DISPATCH A DUPLICATE ALERT for a crossing the
|
|
1005
|
+
user was already told about. The window would also appear twice in the
|
|
1006
|
+
dashboard's alert list.
|
|
1007
|
+
|
|
1008
|
+
The reset is not the only axis §4.3 moved. `window_minutes` is snapped too,
|
|
1009
|
+
and it lives in BOTH the logical limit key and a column of its own — so a
|
|
1010
|
+
terminal row written before the snap under the stray `10081` spelling is not
|
|
1011
|
+
reachable by an identity match on the snapped value at all. Matching only the
|
|
1012
|
+
snapped spelling would leave exactly the rows the canonicalization merged
|
|
1013
|
+
stranded under their old identity, which is the same duplicate-alert hazard
|
|
1014
|
+
one axis over. The match therefore enumerates the RAW spellings that snap
|
|
1015
|
+
onto this identity (`codex_snap_equivalent_limit_keys` /
|
|
1016
|
+
`codex_snap_equivalent_window_minutes`) and the UPDATE re-keys them, not just
|
|
1017
|
+
re-anchors them.
|
|
1018
|
+
|
|
1019
|
+
Runs inside the caller's transaction, on both the live leg and the rebuild
|
|
1020
|
+
re-materialization (they share this body), and is idempotent: a row already
|
|
1021
|
+
on the canonical identity is excluded by the three-way `<>` guard.
|
|
1022
|
+
|
|
1023
|
+
Bounded on both axes by the tolerances that produced the canonical identity —
|
|
1024
|
+
600s on the reset, ±1 minute on the length — so it can only ever collapse
|
|
1025
|
+
rows the canonicalization itself merged. Two genuinely different cycles are
|
|
1026
|
+
five hours or seven days apart, and a `10200` window is a different window,
|
|
1027
|
+
not jitter.
|
|
1028
|
+
"""
|
|
1029
|
+
identity = block.identity
|
|
1030
|
+
keys = codex_snap_equivalent_limit_keys(identity.logical_limit_key)
|
|
1031
|
+
minutes = codex_snap_equivalent_window_minutes(identity.window_minutes)
|
|
1032
|
+
params: dict[str, object] = {
|
|
1033
|
+
"anchor": _utc_iso(block.resets_at),
|
|
1034
|
+
"source": identity.source,
|
|
1035
|
+
"root": identity.source_root_key,
|
|
1036
|
+
"account": identity.account_key,
|
|
1037
|
+
"limit_key": identity.logical_limit_key,
|
|
1038
|
+
"slot": identity.observed_slot,
|
|
1039
|
+
"minutes": identity.window_minutes,
|
|
1040
|
+
"tolerance": CODEX_RESET_ANCHOR_TOLERANCE_SECONDS,
|
|
1041
|
+
}
|
|
1042
|
+
params.update({f"key{i}": value for i, value in enumerate(keys)})
|
|
1043
|
+
params.update({f"min{i}": value for i, value in enumerate(minutes)})
|
|
1044
|
+
conn.execute(
|
|
1045
|
+
_reanchor_terminal_events_sql(len(keys), len(minutes)), params)
|
|
1046
|
+
|
|
1047
|
+
|
|
888
1048
|
def _apply_quota_projection_rows(
|
|
889
1049
|
conn, *, observations, active_roots, now, now_iso,
|
|
890
1050
|
sink, alert_eligible_roots, journal_emit=None, journal_disarm=None,
|
|
891
|
-
holder=None,
|
|
1051
|
+
journal_terminal=None, holder=None,
|
|
892
1052
|
):
|
|
893
1053
|
"""Transaction-neutral quota projection apply (spec §5.3 "projection").
|
|
894
1054
|
|
|
@@ -908,6 +1068,7 @@ def _apply_quota_projection_rows(
|
|
|
908
1068
|
generation = secrets.token_hex(16)
|
|
909
1069
|
blocks = build_blocks(observations)
|
|
910
1070
|
for block in blocks:
|
|
1071
|
+
_reanchor_terminal_events(conn, block)
|
|
911
1072
|
conn.execute(_BLOCK_UPSERT, _block_params(block, generation))
|
|
912
1073
|
for milestone in percent_milestones(block):
|
|
913
1074
|
conn.execute(
|
|
@@ -921,7 +1082,7 @@ def _apply_quota_projection_rows(
|
|
|
921
1082
|
conn, observations=observations,
|
|
922
1083
|
alert_eligible_roots=alert_eligible_roots & active_roots,
|
|
923
1084
|
now=now, now_iso=now_iso, journal_emit=journal_emit,
|
|
924
|
-
journal_disarm=journal_disarm,
|
|
1085
|
+
journal_disarm=journal_disarm, journal_terminal=journal_terminal,
|
|
925
1086
|
)
|
|
926
1087
|
# The completion stamp is intentionally the final DML in the stats
|
|
927
1088
|
# transaction. A pre-commit failure rolls all projection updates back;
|
|
@@ -1103,6 +1264,7 @@ def reconcile_codex_quota_projection(
|
|
|
1103
1264
|
|
|
1104
1265
|
def _apply_projection(
|
|
1105
1266
|
conn, sink, *, journal_emit=None, journal_disarm=None,
|
|
1267
|
+
journal_terminal=None,
|
|
1106
1268
|
):
|
|
1107
1269
|
# No configured roots and no existing interpreted history means there is
|
|
1108
1270
|
# no stats work. This preserves the existing empty-Codex fast path.
|
|
@@ -1113,6 +1275,7 @@ def reconcile_codex_quota_projection(
|
|
|
1113
1275
|
now=now, now_iso=now_iso, sink=sink,
|
|
1114
1276
|
alert_eligible_roots=alert_eligible_roots,
|
|
1115
1277
|
journal_emit=journal_emit, journal_disarm=journal_disarm,
|
|
1278
|
+
journal_terminal=journal_terminal,
|
|
1116
1279
|
holder=holder,
|
|
1117
1280
|
)
|
|
1118
1281
|
|
|
@@ -1172,11 +1335,30 @@ def reconcile_codex_quota_projection(
|
|
|
1172
1335
|
))
|
|
1173
1336
|
ctx.events_emitted += 1
|
|
1174
1337
|
|
|
1338
|
+
def _emit_terminal_event(payload):
|
|
1339
|
+
# #416 spec §7.2: journal one TERMINAL threshold fact. The `qte:` id
|
|
1340
|
+
# mirrors the table's UNIQUE key, so one crossing is one event
|
|
1341
|
+
# forever and a replay converges instead of duplicating. This order
|
|
1342
|
+
# MUST match the cutover export's `natural_key_id`.
|
|
1343
|
+
eid = _jl.evt_id(
|
|
1344
|
+
"qte", payload["source"], payload["source_root_key"],
|
|
1345
|
+
payload["account_key"], payload["logical_limit_key"],
|
|
1346
|
+
payload["observed_slot"], payload["window_minutes"],
|
|
1347
|
+
payload["resets_at_utc"], payload["threshold"],
|
|
1348
|
+
)
|
|
1349
|
+
_jr.append_record(_jl.make_evt(
|
|
1350
|
+
kind="quota_threshold_event", id=eid,
|
|
1351
|
+
at=payload["created_at_utc"],
|
|
1352
|
+
payload={**payload, "journal_identity_version": 2},
|
|
1353
|
+
))
|
|
1354
|
+
ctx.events_emitted += 1
|
|
1355
|
+
|
|
1175
1356
|
_apply_projection(
|
|
1176
1357
|
ctx.conn,
|
|
1177
1358
|
ctx.pending_alerts,
|
|
1178
1359
|
journal_emit=_emit_arming,
|
|
1179
1360
|
journal_disarm=_emit_disarm,
|
|
1361
|
+
journal_terminal=_emit_terminal_event,
|
|
1180
1362
|
)
|
|
1181
1363
|
# `_before_stats_commit` fires INSIDE the cycle txn, before COMMIT — a
|
|
1182
1364
|
# raise rolls the whole cycle back, so the projection updates undo
|
|
@@ -1227,9 +1409,63 @@ def _load_active_milestones(
|
|
|
1227
1409
|
stats.close()
|
|
1228
1410
|
|
|
1229
1411
|
|
|
1412
|
+
def _codex_cache_account_predicate(
|
|
1413
|
+
account_key: str | None, *, admit_unattributed: bool = False,
|
|
1414
|
+
) -> tuple[str, tuple]:
|
|
1415
|
+
"""SQL fragment scoping a CACHE table to one account (#416 B2).
|
|
1416
|
+
|
|
1417
|
+
The cache columns are nullable ``TEXT``, so ``NULL ≡ unattributed`` — the
|
|
1418
|
+
established cache-read rule (``load_cached_rooted_codex_accounting_entries``
|
|
1419
|
+
uses the identical pair). ``None`` yields an empty fragment, i.e. today's
|
|
1420
|
+
merged read.
|
|
1421
|
+
|
|
1422
|
+
``admit_unattributed`` selects the ONE-DIRECTIONAL WIDENING flavour — the
|
|
1423
|
+
SQL twin of ``_codex_account_admits``: a REAL account admits its own rows
|
|
1424
|
+
PLUS the unattributed sentinel, while an ``unattributed`` scope still admits
|
|
1425
|
+
only unattributed, so no REAL account's rows ever reach another's read.
|
|
1426
|
+
|
|
1427
|
+
Which flavour a read wants is decided by the STAMPING MECHANISM behind its
|
|
1428
|
+
scope key, never by taste, and it is settled per read INSIDE this module —
|
|
1429
|
+
no caller elects it (#416 closeout F1/F3). The rule, in full:
|
|
1430
|
+
|
|
1431
|
+
* **Widen** iff a row genuinely belonging to the focused account can still
|
|
1432
|
+
carry the ``unattributed`` sentinel IN THE TABLE BEING FILTERED — i.e. the
|
|
1433
|
+
scope key and the rows were stamped by DIFFERENT mechanisms, or by the
|
|
1434
|
+
same mechanism over a different population/window-group.
|
|
1435
|
+
* **Strict** iff the scope key was derived from the very column being
|
|
1436
|
+
filtered, over the same population: the read is then a partition of the
|
|
1437
|
+
rows the key came from, and the children must sum to the parent.
|
|
1438
|
+
* Corollary: **selection/boundary reads widen; cost- and percentage-
|
|
1439
|
+
adoption reads stay strict.** A boundary read answers "where did this
|
|
1440
|
+
block open" — widening it is not attribution. A cost read answers "whose
|
|
1441
|
+
dollars are these" — widening it IS attribution, which D1 forbids, and it
|
|
1442
|
+
puts one row in two scopes.
|
|
1443
|
+
|
|
1444
|
+
Three stamping mechanisms exist and must never be conflated: the
|
|
1445
|
+
quota-observation fold (``adopt_unidentified_observations``, per physical-
|
|
1446
|
+
window group, landing post-fold in ``quota_window_blocks`` /
|
|
1447
|
+
``quota_percent_milestones`` and NEVER written back to
|
|
1448
|
+
``quota_window_snapshots``); per-file-range attribution
|
|
1449
|
+
(``codex_file_accounts`` -> ``codex_session_entries.account_key``,
|
|
1450
|
+
``stably_absent`` -> NULL); and the stats ``accounts`` registry.
|
|
1451
|
+
"""
|
|
1452
|
+
if account_key is None:
|
|
1453
|
+
return "", ()
|
|
1454
|
+
if account_key == _lib_accounts.UNATTRIBUTED:
|
|
1455
|
+
return "AND (account_key IS NULL OR account_key = ?)", (
|
|
1456
|
+
_lib_accounts.UNATTRIBUTED,)
|
|
1457
|
+
if admit_unattributed:
|
|
1458
|
+
return (
|
|
1459
|
+
"AND (account_key = ? OR account_key IS NULL OR account_key = ?)",
|
|
1460
|
+
(account_key, _lib_accounts.UNATTRIBUTED),
|
|
1461
|
+
)
|
|
1462
|
+
return "AND account_key = ?", (account_key,)
|
|
1463
|
+
|
|
1464
|
+
|
|
1230
1465
|
def _first_block_physical_tuple(
|
|
1231
1466
|
identity: QuotaWindowIdentity, resets_at: dt.datetime,
|
|
1232
1467
|
*, cache_conn: sqlite3.Connection | None = None,
|
|
1468
|
+
account_key: str | None = None,
|
|
1233
1469
|
) -> tuple[dt.datetime, str, int] | None:
|
|
1234
1470
|
"""Read the first physical tuple for one exact projected block.
|
|
1235
1471
|
|
|
@@ -1237,6 +1473,31 @@ def _first_block_physical_tuple(
|
|
|
1237
1473
|
for the root in Python just to discover this boundary. Keep the same
|
|
1238
1474
|
physical ordering while letting SQLite filter the exact identity/reset.
|
|
1239
1475
|
``unixepoch`` deliberately accepts retained ``Z`` and ``+00:00`` spellings.
|
|
1476
|
+
|
|
1477
|
+
``account_key`` (#416 Slice 3A review B2) scopes the boundary to one
|
|
1478
|
+
account: two accounts sharing one root and one canonical reset otherwise
|
|
1479
|
+
hand the focused account the OTHER account's earlier start, so its first
|
|
1480
|
+
milestone segment absorbs spend from before its own block opened.
|
|
1481
|
+
``NULL ≡ unattributed`` on this cache column; ``None`` keeps the merged
|
|
1482
|
+
read, which is byte-stable.
|
|
1483
|
+
|
|
1484
|
+
A non-``None`` scope ALWAYS takes the widening flavour here, unconditionally
|
|
1485
|
+
and with no caller say in it (#416 closeout F1). Every scope key that can
|
|
1486
|
+
reach this read is stamped by a different mechanism than
|
|
1487
|
+
``quota_window_snapshots.account_key``: the durable projection and the
|
|
1488
|
+
in-memory observation partition are both POST-fold
|
|
1489
|
+
(``load_codex_quota_observations`` runs ``adopt_unidentified_observations``
|
|
1490
|
+
before returning), while these snapshot rows are the PRE-fold raw cache the
|
|
1491
|
+
fold never writes back to. And this is a BOUNDARY read — "where did the
|
|
1492
|
+
block open" — so widening it attributes nothing.
|
|
1493
|
+
|
|
1494
|
+
Getting it wrong here is maximally destructive: ``codex_quota_breakdown``
|
|
1495
|
+
returns ``()`` outright when this read finds no row, so a post-fold key read
|
|
1496
|
+
strictly against pre-fold snapshots does not shrink the ladder, it DELETES
|
|
1497
|
+
it, while the cycle INDEX (reading the post-fold
|
|
1498
|
+
``quota_percent_milestones``) still counts the crossings — the #373
|
|
1499
|
+
root-cause-3 symptom exactly. The widening stays one-directional, so an
|
|
1500
|
+
``unattributed`` scope never adopts a REAL account's boundary.
|
|
1240
1501
|
"""
|
|
1241
1502
|
owns_conn = cache_conn is None
|
|
1242
1503
|
if cache_conn is None:
|
|
@@ -1247,6 +1508,8 @@ def _first_block_physical_tuple(
|
|
|
1247
1508
|
else:
|
|
1248
1509
|
cache = cache_conn
|
|
1249
1510
|
try:
|
|
1511
|
+
account_predicate, account_params = _codex_cache_account_predicate(
|
|
1512
|
+
account_key, admit_unattributed=True)
|
|
1250
1513
|
row = cache.execute(
|
|
1251
1514
|
"""SELECT captured_at_utc, source_path, line_offset
|
|
1252
1515
|
FROM quota_window_snapshots
|
|
@@ -1254,13 +1517,14 @@ def _first_block_physical_tuple(
|
|
|
1254
1517
|
AND logical_limit_key=? AND observed_slot=?
|
|
1255
1518
|
AND window_minutes=?
|
|
1256
1519
|
AND unixepoch(resets_at_utc)=unixepoch(?)
|
|
1520
|
+
""" + account_predicate + """
|
|
1257
1521
|
ORDER BY unixepoch(captured_at_utc), unixepoch(resets_at_utc),
|
|
1258
1522
|
source_path, line_offset
|
|
1259
1523
|
LIMIT 1""",
|
|
1260
1524
|
(
|
|
1261
1525
|
identity.source_root_key, identity.logical_limit_key,
|
|
1262
1526
|
identity.observed_slot, identity.window_minutes,
|
|
1263
|
-
_utc_iso(resets_at),
|
|
1527
|
+
_utc_iso(resets_at), *account_params,
|
|
1264
1528
|
),
|
|
1265
1529
|
).fetchone()
|
|
1266
1530
|
finally:
|
|
@@ -1279,6 +1543,7 @@ def codex_quota_breakdown(
|
|
|
1279
1543
|
resets_at: str | dt.datetime,
|
|
1280
1544
|
*, speed: str = "auto", cache_conn: sqlite3.Connection | None = None,
|
|
1281
1545
|
stats_conn: sqlite3.Connection | None = None,
|
|
1546
|
+
account_key: str | None = None,
|
|
1282
1547
|
) -> tuple[CodexQuotaBreakdownRow, ...]:
|
|
1283
1548
|
"""Correlate durable milestone boundaries with live-priced cache accounting.
|
|
1284
1549
|
|
|
@@ -1286,6 +1551,32 @@ def codex_quota_breakdown(
|
|
|
1286
1551
|
same-timestamp records stay deterministic. Pricing is deliberately read
|
|
1287
1552
|
now rather than materialized in stats.db, keeping a pricing refresh
|
|
1288
1553
|
immediately effective for historical quota breakdowns.
|
|
1554
|
+
|
|
1555
|
+
``account_key`` (#416 Slice 3A review B2) scopes BOTH cache reads below —
|
|
1556
|
+
the block-start boundary and the accounting rows — to one account. The
|
|
1557
|
+
milestone read is already account-scoped through ``identity.account_key``,
|
|
1558
|
+
so without this the durable ladder mixed one account's crossings with every
|
|
1559
|
+
account's spend on that root. ``None`` is today's merged read and is
|
|
1560
|
+
byte-stable, which is what every CLI caller keeps.
|
|
1561
|
+
|
|
1562
|
+
The two reads take DIFFERENT flavours, and neither is a caller's choice
|
|
1563
|
+
(#416 closeout F1/F3) — the single flag they used to share conflated two
|
|
1564
|
+
reads with opposite correctness requirements:
|
|
1565
|
+
|
|
1566
|
+
* the boundary (``_first_block_physical_tuple``, pre-fold
|
|
1567
|
+
``quota_window_snapshots``) always WIDENS. It is a selection read whose
|
|
1568
|
+
scope key comes from another mechanism entirely, and strict equality
|
|
1569
|
+
there blanks the whole ladder rather than trimming it.
|
|
1570
|
+
* the accounting read below (``codex_session_entries``) always stays
|
|
1571
|
+
STRICT. It is a COST read: widening it would file one unattributed row
|
|
1572
|
+
under a real account AND under the ``unattributed`` scope, which is
|
|
1573
|
+
inference D1 forbids and double-counting the children-sum-to-parent
|
|
1574
|
+
invariant forbids. A crossing whose spend is unattributed therefore
|
|
1575
|
+
renders an honest ``$0.00``; the dollars stay visible in the
|
|
1576
|
+
``unattributed`` scope, which owns them.
|
|
1577
|
+
|
|
1578
|
+
The full rule, and the three stamping mechanisms it turns on, are in
|
|
1579
|
+
``_codex_cache_account_predicate``.
|
|
1289
1580
|
"""
|
|
1290
1581
|
reset = _parse_utc(resets_at, "resets_at") if isinstance(resets_at, str) else resets_at
|
|
1291
1582
|
if reset.tzinfo is None or reset.utcoffset() is None:
|
|
@@ -1302,7 +1593,8 @@ def codex_quota_breakdown(
|
|
|
1302
1593
|
return ()
|
|
1303
1594
|
else:
|
|
1304
1595
|
cache = cache_conn
|
|
1305
|
-
start = _first_block_physical_tuple(
|
|
1596
|
+
start = _first_block_physical_tuple(
|
|
1597
|
+
identity, reset, cache_conn=cache, account_key=account_key)
|
|
1306
1598
|
if start is None:
|
|
1307
1599
|
if owns_cache:
|
|
1308
1600
|
cache.close()
|
|
@@ -1321,17 +1613,22 @@ def codex_quota_breakdown(
|
|
|
1321
1613
|
).astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
|
1322
1614
|
try:
|
|
1323
1615
|
entries = []
|
|
1616
|
+
# Cost read: strict, always. See the docstring above.
|
|
1617
|
+
entry_predicate, entry_params = _codex_cache_account_predicate(
|
|
1618
|
+
account_key)
|
|
1324
1619
|
for row in cache.execute(
|
|
1325
1620
|
"""SELECT timestamp_utc, source_path, line_offset, model,
|
|
1326
1621
|
input_tokens, cached_input_tokens, output_tokens,
|
|
1327
1622
|
reasoning_output_tokens, total_tokens
|
|
1328
1623
|
FROM codex_session_entries
|
|
1329
1624
|
WHERE source_root_key=?
|
|
1330
|
-
AND timestamp_utc>=? AND timestamp_utc<=?
|
|
1625
|
+
AND timestamp_utc>=? AND timestamp_utc<=?
|
|
1626
|
+
""" + entry_predicate,
|
|
1331
1627
|
(
|
|
1332
1628
|
identity.source_root_key,
|
|
1333
1629
|
query_start,
|
|
1334
1630
|
query_end,
|
|
1631
|
+
*entry_params,
|
|
1335
1632
|
),
|
|
1336
1633
|
):
|
|
1337
1634
|
try:
|
|
@@ -2016,7 +2313,8 @@ def cmd_codex_percent_breakdown(args) -> int:
|
|
|
2016
2313
|
).state == "fresh"
|
|
2017
2314
|
)
|
|
2018
2315
|
for block in build_blocks(history.physical_observations)
|
|
2019
|
-
|
|
2316
|
+
# CANONICAL on both sides (#416 §4.1) — see above.
|
|
2317
|
+
if block.resets_at == baseline.canonical_resets_at
|
|
2020
2318
|
)
|
|
2021
2319
|
if len(matching) != 1:
|
|
2022
2320
|
raise QuotaCLIError(
|
|
@@ -94,7 +94,7 @@ _OPAQUE_PROJECT_KEY_RE = re.compile(r"^project:[0-9a-f]{24}$")
|
|
|
94
94
|
_QUALIFIED_CODEX_ENTRIES_SQL = """
|
|
95
95
|
SELECT entries.timestamp_utc, entries.session_id, entries.source_path,
|
|
96
96
|
entries.source_root_key,
|
|
97
|
-
entries.conversation_key, entries.model,
|
|
97
|
+
entries.conversation_key, entries.model, entries.account_key,
|
|
98
98
|
entries.input_tokens, entries.cached_input_tokens,
|
|
99
99
|
entries.output_tokens, entries.reasoning_output_tokens,
|
|
100
100
|
entries.total_tokens, threads.cwd, threads.git_json,
|
|
@@ -577,6 +577,9 @@ def load_qualified_codex_entries(
|
|
|
577
577
|
reasoning_output_tokens=int(row["reasoning_output_tokens"]),
|
|
578
578
|
total_tokens=int(row["total_tokens"]),
|
|
579
579
|
cost_usd=cost_usd,
|
|
580
|
+
# NULL ≡ unattributed — the cache-read rule (#416 §5.2). Carried,
|
|
581
|
+
# never grouped; see the field's comment on `QualifiedCodexEntry`.
|
|
582
|
+
account_key=str(row["account_key"] or _lib_accounts.UNATTRIBUTED),
|
|
580
583
|
))
|
|
581
584
|
return tuple(result)
|
|
582
585
|
|
package/bin/_cctally_tui.py
CHANGED
|
@@ -1061,14 +1061,18 @@ def _tui_attribute_corruption(
|
|
|
1061
1061
|
"""Classify corruption against the actual database at the catch site.
|
|
1062
1062
|
|
|
1063
1063
|
Mixed stats/cache builders can surface the same SQLite message from either
|
|
1064
|
-
family.
|
|
1064
|
+
family. Only after a corruption-shaped exception do we run the expensive
|
|
1065
1065
|
stats ``quick_check``: a failed/non-ok result positively attributes stats;
|
|
1066
|
-
an intact stats family leaves the failure attributed to cache.
|
|
1067
|
-
|
|
1066
|
+
an intact stats family leaves the failure attributed to cache. Explicit
|
|
1067
|
+
third-store ownership, including ``conversations``, is preserved without a
|
|
1068
|
+
stats probe. No path or exception-text parsing is used for database
|
|
1069
|
+
identity.
|
|
1068
1070
|
"""
|
|
1069
1071
|
|
|
1070
1072
|
corruption = bool(_cctally()._is_sqlite_corruption_error(exc))
|
|
1071
1073
|
attributed = database
|
|
1074
|
+
if database == "conversations":
|
|
1075
|
+
return database, corruption
|
|
1072
1076
|
if corruption and database == "stats_or_cache":
|
|
1073
1077
|
try:
|
|
1074
1078
|
rows = conn.execute("PRAGMA quick_check").fetchall()
|