cctally 1.111.0 → 1.113.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +6 -6
- package/bin/_cctally_cache.py +14 -7
- package/bin/_cctally_core.py +9 -1
- package/bin/_cctally_dashboard_envelope.py +7 -6
- package/bin/_cctally_dashboard_sources.py +90 -55
- package/bin/_cctally_doctor.py +73 -27
- package/bin/_cctally_forecast.py +2 -1
- package/bin/_cctally_journal.py +147 -6
- package/bin/_cctally_milestones.py +29 -3
- package/bin/_cctally_quota.py +10 -1
- package/bin/_cctally_record.py +136 -4
- package/bin/_cctally_rederive.py +311 -8
- package/bin/_cctally_source_analytics.py +20 -8
- package/bin/_cctally_tui.py +546 -45
- package/bin/_lib_codex_conversation_query.py +40 -4
- package/bin/_lib_codex_metadata.py +57 -11
- package/bin/_lib_cost_provenance.py +303 -0
- package/bin/_lib_dashboard_sources.py +263 -28
- package/bin/_lib_doctor.py +2 -2
- package/bin/_lib_ingest_frontier.py +137 -38
- package/bin/_lib_pricing.py +29 -1
- package/bin/_lib_rederive.py +704 -36
- package/bin/_lib_snapshot_cache.py +332 -9
- package/dashboard/static/assets/{ConversationsView-BB9vUiL6.js → ConversationsView-CEZ-BUyi.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-DFY2iPDf.js → DoctorModal-CE1oH_Zy.js} +1 -1
- package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-DAv39uSo.js → ProjectsDrillPanel-B3DZoCeD.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-DSAWo2Fp.js → SourceDetailModal-CHp9EgLP.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-R_9dOVnh.js → UpdateModal-5iKST6jo.js} +1 -1
- package/dashboard/static/assets/{index-DdU62sov.js → index-Cm0VTxJC.js} +2 -2
- package/dashboard/static/assets/index-Dt4tqf43.css +1 -0
- package/dashboard/static/assets/{outlineNavigation-BOCLcRa6.js → outlineNavigation-DkXGL8J6.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +2 -1
- package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +0 -1
- package/dashboard/static/assets/index-COhxe3vU.css +0 -1
package/bin/_cctally_journal.py
CHANGED
|
@@ -1409,12 +1409,19 @@ class IngestContext:
|
|
|
1409
1409
|
# (reset INSERT OR IGNORE rowcount == 1), so a crash-replayed reset never
|
|
1410
1410
|
# re-suppresses with a divergent list.
|
|
1411
1411
|
suppression_map: dict = field(default_factory=dict)
|
|
1412
|
+
# Indexed by (event kind, local row id) until harvest persists the marker.
|
|
1413
|
+
pricing_provenance: dict = field(default_factory=dict)
|
|
1412
1414
|
# Task B rederive seam. Normal ingest leaves both defaults unchanged.
|
|
1413
1415
|
# A scratch planner supplies an in-memory sink so derived events are captured
|
|
1414
1416
|
# instead of appended to the durable journal, and disables projection-file
|
|
1415
1417
|
# writes while still exercising the same SQLite derivation/fold code.
|
|
1416
1418
|
event_sink: "list | None" = None
|
|
1417
1419
|
projection_writes: bool = True
|
|
1420
|
+
# #869 `db rederive` scratch replay only. Its keys name the (account_key,
|
|
1421
|
+
# window key) blocks the planner must prove; each value becomes the
|
|
1422
|
+
# retained entries behind the totals that block last applied. Live ingest
|
|
1423
|
+
# leaves it None and keeps no entries.
|
|
1424
|
+
block_source_evidence: "dict | None" = None
|
|
1418
1425
|
# Operator-reviewed scratch replay only. Normal ingest leaves this empty.
|
|
1419
1426
|
held_weekly_observation_ids: frozenset[str] = field(default_factory=frozenset)
|
|
1420
1427
|
reviewed_weekly_basis_by_account: dict = field(default_factory=dict)
|
|
@@ -4127,7 +4134,10 @@ def _apply_generic_evt(conn, evt):
|
|
|
4127
4134
|
return None
|
|
4128
4135
|
cols = {"journal_id": evt["id"]}
|
|
4129
4136
|
for key, value in payload.items():
|
|
4130
|
-
|
|
4137
|
+
# `journal_identity_version` names how the event id was minted (#869
|
|
4138
|
+
# alert-latch re-fire incarnations: bm2/pjm2/pbm2); it is journal
|
|
4139
|
+
# metadata, never a column.
|
|
4140
|
+
if key in ("kind", "_pricing", "journal_identity_version"):
|
|
4131
4141
|
continue
|
|
4132
4142
|
if key in spec.fk_refs:
|
|
4133
4143
|
column, ref_table = spec.fk_refs[key]
|
|
@@ -4493,7 +4503,7 @@ def _apply_block_close(conn, evt):
|
|
|
4493
4503
|
parent = {"journal_id": evt["id"]}
|
|
4494
4504
|
children = {}
|
|
4495
4505
|
for key, value in payload.items():
|
|
4496
|
-
if key
|
|
4506
|
+
if key in ("kind", "_pricing"):
|
|
4497
4507
|
continue
|
|
4498
4508
|
if key in _BLOCK_CHILD_KEYS:
|
|
4499
4509
|
children[key] = value or []
|
|
@@ -4552,7 +4562,7 @@ def _apply_reset_with_suppression(conn, evt):
|
|
|
4552
4562
|
return None
|
|
4553
4563
|
cols = {"journal_id": evt["id"]}
|
|
4554
4564
|
for key, value in payload.items():
|
|
4555
|
-
if key in ("kind", "suppression", "suppression_table"):
|
|
4565
|
+
if key in ("kind", "suppression", "suppression_table", "_pricing"):
|
|
4556
4566
|
continue
|
|
4557
4567
|
if key in spec.fk_refs:
|
|
4558
4568
|
column, ref_table = spec.fk_refs[key]
|
|
@@ -4834,6 +4844,59 @@ def _build_harvest_evt(ctx, spec, row):
|
|
|
4834
4844
|
{k: cr[k] for k in cr.keys() if k not in ("id", "block_id")}
|
|
4835
4845
|
for cr in child_rows
|
|
4836
4846
|
]
|
|
4847
|
+
if spec.kind == "five_hour_block_close":
|
|
4848
|
+
raw_pricing = row["pricing_provenance_json"]
|
|
4849
|
+
if raw_pricing is not None:
|
|
4850
|
+
try:
|
|
4851
|
+
pricing = json.loads(raw_pricing)
|
|
4852
|
+
except (TypeError, ValueError) as exc:
|
|
4853
|
+
raise JournalError(
|
|
4854
|
+
"five_hour_block_close has malformed computation-time "
|
|
4855
|
+
"pricing provenance"
|
|
4856
|
+
) from exc
|
|
4857
|
+
if (
|
|
4858
|
+
not isinstance(pricing, dict)
|
|
4859
|
+
or pricing.get("version") != 1
|
|
4860
|
+
or not isinstance(pricing.get("pricingDate"), str)
|
|
4861
|
+
or not pricing["pricingDate"]
|
|
4862
|
+
or not isinstance(pricing.get("unpricedModels"), list)
|
|
4863
|
+
or any(not isinstance(model, str) or not model
|
|
4864
|
+
for model in pricing["unpricedModels"])
|
|
4865
|
+
or pricing["unpricedModels"] != sorted(set(
|
|
4866
|
+
pricing["unpricedModels"]
|
|
4867
|
+
))
|
|
4868
|
+
or (
|
|
4869
|
+
"uncertainModels" in pricing
|
|
4870
|
+
and (
|
|
4871
|
+
not isinstance(pricing["uncertainModels"], list)
|
|
4872
|
+
or any(not isinstance(model, str) or not model
|
|
4873
|
+
for model in pricing["uncertainModels"])
|
|
4874
|
+
or pricing["uncertainModels"] != sorted(set(
|
|
4875
|
+
pricing["uncertainModels"]
|
|
4876
|
+
))
|
|
4877
|
+
)
|
|
4878
|
+
)
|
|
4879
|
+
):
|
|
4880
|
+
raise JournalError(
|
|
4881
|
+
"five_hour_block_close has incomplete computation-time "
|
|
4882
|
+
"pricing provenance"
|
|
4883
|
+
)
|
|
4884
|
+
positive_models = {
|
|
4885
|
+
child["model"] for child in payload.get("_models", [])
|
|
4886
|
+
if sum(int(child.get(key) or 0) for key in (
|
|
4887
|
+
"input_tokens", "output_tokens", "cache_create_tokens",
|
|
4888
|
+
"cache_read_tokens",
|
|
4889
|
+
)) > 0
|
|
4890
|
+
}
|
|
4891
|
+
if not (
|
|
4892
|
+
set(pricing["unpricedModels"])
|
|
4893
|
+
| set(pricing.get("uncertainModels") or [])
|
|
4894
|
+
) <= positive_models:
|
|
4895
|
+
raise JournalError(
|
|
4896
|
+
"five_hour_block_close pricing provenance does not match "
|
|
4897
|
+
"its computed model population"
|
|
4898
|
+
)
|
|
4899
|
+
payload["_pricing"] = pricing
|
|
4837
4900
|
# #750 S3 §1.1: ONE producer for the id parts and for the suppression-map
|
|
4838
4901
|
# key. A family with an `id_parts_fn` (today only `week_reset_events`,
|
|
4839
4902
|
# whose identity is dual-shaped) supplies both from that callable; every
|
|
@@ -4851,11 +4914,85 @@ def _build_harvest_evt(ctx, spec, row):
|
|
|
4851
4914
|
supp = ctx.suppression_map.get(supp_key)
|
|
4852
4915
|
if supp:
|
|
4853
4916
|
payload["suppression"] = list(supp)
|
|
4917
|
+
provenance = ctx.pricing_provenance.get((spec.kind, int(row["id"])))
|
|
4918
|
+
if provenance is not None:
|
|
4919
|
+
payload["_pricing"] = provenance
|
|
4854
4920
|
eid = _lib_journal.evt_id(spec.id_prefix, *parts)
|
|
4921
|
+
refire_prefix = _LATCH_REFIRE_ID_PREFIXES.get(spec.kind)
|
|
4922
|
+
if (
|
|
4923
|
+
refire_prefix is not None
|
|
4924
|
+
and ctx.event_sink is None
|
|
4925
|
+
and _latch_event_was_retired(conn, eid)
|
|
4926
|
+
):
|
|
4927
|
+
# #869 F6/V1: a correction retired this natural key's earlier crossing
|
|
4928
|
+
# with a higher-revision tombstone, so a revision-0 event under the
|
|
4929
|
+
# same id can never become effective. A later crossing is a new
|
|
4930
|
+
# incarnation: its id adds the crossing instant, while the table's
|
|
4931
|
+
# natural key still deduplicates it. Every other crossing, and every
|
|
4932
|
+
# already-journaled id, keeps the legacy spelling.
|
|
4933
|
+
eid = _lib_journal.evt_id(
|
|
4934
|
+
refire_prefix, *parts, row["crossed_at_utc"])
|
|
4935
|
+
payload["journal_identity_version"] = 2
|
|
4855
4936
|
at = row[spec.at_column] if spec.at_column else _now_iso()
|
|
4856
4937
|
return _lib_journal.make_evt(kind=spec.kind, id=eid, at=at, payload=payload)
|
|
4857
4938
|
|
|
4858
4939
|
|
|
4940
|
+
#: #869 F6/V1. The id prefix, per alert-latch kind, of a crossing that re-fires
|
|
4941
|
+
#: after a correction tombstoned the natural key's earlier crossing (the legacy
|
|
4942
|
+
#: prefix + identity version 2). `db rederive` retires these latches, so each
|
|
4943
|
+
#: one needs a spelling a later live crossing can journal under.
|
|
4944
|
+
#:
|
|
4945
|
+
#: Coupling: the live writers (`_claude_latch_recreates_retired_crossing` in
|
|
4946
|
+
#: `_cctally_record`) stamp a recreated CLAUDE latch without notifying, on the
|
|
4947
|
+
#: premise that every Claude-latch tombstone is the claude-usage family's
|
|
4948
|
+
#: stale-config retirement, never a verdict that the crossing was false. A
|
|
4949
|
+
#: future rule that retires a Claude latch as a false crossing must record
|
|
4950
|
+
#: that distinction durably, or the genuine later crossing is silently
|
|
4951
|
+
#: suppressed.
|
|
4952
|
+
_LATCH_REFIRE_ID_PREFIXES = {
|
|
4953
|
+
"budget": "bm2",
|
|
4954
|
+
"projected": "pjm2",
|
|
4955
|
+
"project_budget": "pbm2",
|
|
4956
|
+
}
|
|
4957
|
+
|
|
4958
|
+
|
|
4959
|
+
def _latch_event_was_retired(conn, legacy_event_id) -> bool:
|
|
4960
|
+
"""True iff the effective selection for this natural-key id is a tombstone.
|
|
4961
|
+
|
|
4962
|
+
Read from the live effective metadata the rebuild published, never
|
|
4963
|
+
recomputed. A missing row or an active event keeps the legacy id.
|
|
4964
|
+
"""
|
|
4965
|
+
prior = _metadata_row(conn, legacy_event_id)
|
|
4966
|
+
return prior is not None and str(prior[1]) == "tombstone"
|
|
4967
|
+
|
|
4968
|
+
|
|
4969
|
+
def latch_crossing_was_retired(conn, kind, rowid) -> bool:
|
|
4970
|
+
"""True iff alert-latch row ``rowid`` re-creates a retired crossing (#869).
|
|
4971
|
+
|
|
4972
|
+
``kind`` is a latch family with a re-fire incarnation (``budget``,
|
|
4973
|
+
``projected``, ``project_budget``). The row's legacy natural-key event id
|
|
4974
|
+
is minted exactly as harvest mints it, and the answer is the live effective
|
|
4975
|
+
metadata's: a tombstone means a completed correction batch retired an
|
|
4976
|
+
earlier crossing of this natural key, so harvest will journal this row
|
|
4977
|
+
under the kind's version-2 prefix. A live writer calls this in the same
|
|
4978
|
+
transaction as its insert, before harvest runs, so both decisions read one
|
|
4979
|
+
durable fact.
|
|
4980
|
+
"""
|
|
4981
|
+
if kind not in _LATCH_REFIRE_ID_PREFIXES:
|
|
4982
|
+
raise ValueError(f"{kind!r} has no re-fire incarnation")
|
|
4983
|
+
[spec] = [item for item in _HARVEST_SPECS if item.kind == kind]
|
|
4984
|
+
if spec.fk_refs or spec.id_parts_fn is not None:
|
|
4985
|
+
raise ValueError(f"{kind!r} ids are not plain natural-key columns")
|
|
4986
|
+
row = conn.execute(
|
|
4987
|
+
f"SELECT {', '.join(spec.id_parts)} FROM {spec.table} WHERE id = ?",
|
|
4988
|
+
(int(rowid),),
|
|
4989
|
+
).fetchone()
|
|
4990
|
+
if row is None:
|
|
4991
|
+
return False
|
|
4992
|
+
return _latch_event_was_retired(
|
|
4993
|
+
conn, _lib_journal.evt_id(spec.id_prefix, *tuple(row)))
|
|
4994
|
+
|
|
4995
|
+
|
|
4859
4996
|
def _emit_harvest_row(ctx, spec, row):
|
|
4860
4997
|
"""Journal and stamp one already-selected natural-keyed row."""
|
|
4861
4998
|
conn = ctx.conn
|
|
@@ -5334,7 +5471,10 @@ def _run_config_reconcile(ctx, reconcile_config) -> None:
|
|
|
5334
5471
|
if fn is None:
|
|
5335
5472
|
continue
|
|
5336
5473
|
try:
|
|
5337
|
-
|
|
5474
|
+
if axis == "codex_budget":
|
|
5475
|
+
fn(validated_budget, conn=ctx.conn, journal_ctx=ctx)
|
|
5476
|
+
else:
|
|
5477
|
+
fn(validated_budget, conn=ctx.conn)
|
|
5338
5478
|
except Exception as exc: # best-effort; never break the cycle over a reconcile
|
|
5339
5479
|
print(f"[budget-reconcile] {name} failed: {exc}", file=sys.stderr)
|
|
5340
5480
|
# Per-project reconcile takes `touched_projects` as its 2nd positional
|
|
@@ -6086,7 +6226,8 @@ def _effective_event_for_convergence(conn, event_id) -> dict:
|
|
|
6086
6226
|
|
|
6087
6227
|
# Effect keys that ride an evt payload but are NOT target-table columns.
|
|
6088
6228
|
_EVT_EFFECT_KEYS = frozenset(
|
|
6089
|
-
{"kind", "suppression", "suppression_table", "floor_suppression",
|
|
6229
|
+
{"kind", "suppression", "suppression_table", "floor_suppression",
|
|
6230
|
+
"hwm_floor", "_pricing", "journal_identity_version"}
|
|
6090
6231
|
)
|
|
6091
6232
|
|
|
6092
6233
|
|
|
@@ -8020,7 +8161,7 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
|
|
|
8020
8161
|
# omitted column, constraint, partial predicate, or index definition. An epoch
|
|
8021
8162
|
# schema change must update this contract alongside STATS_INDEX_EPOCH.
|
|
8022
8163
|
_REBUILD_SCHEMA_FINGERPRINT = (
|
|
8023
|
-
"
|
|
8164
|
+
"e0f2225f00843952b4887fa8afb046edacd3da0642d0f402ebaa5219a6e30697"
|
|
8024
8165
|
)
|
|
8025
8166
|
|
|
8026
8167
|
|
|
@@ -721,7 +721,8 @@ def _resolve_budget_window(conn, *, vendor, now_utc, period, config, tz,
|
|
|
721
721
|
|
|
722
722
|
|
|
723
723
|
def _budget_spend_for_vendor(conn, *, vendor, start_at, now_utc,
|
|
724
|
-
account_key: str = "*"
|
|
724
|
+
account_key: str = "*",
|
|
725
|
+
pricing_provenance: "dict | None" = None) -> float:
|
|
725
726
|
"""Spend over ``[start_at, now]`` for ``vendor`` (#143) — the COSTLY leg,
|
|
726
727
|
called only after the pre-probe finds pending thresholds (spec §4.2). claude
|
|
727
728
|
routes through the Claude cost SUM (``mode="auto"``); codex through the Codex
|
|
@@ -742,12 +743,15 @@ def _budget_spend_for_vendor(conn, *, vendor, start_at, now_utc,
|
|
|
742
743
|
extra = {} if scope is None else {"account_key": scope}
|
|
743
744
|
if vendor == "claude":
|
|
744
745
|
return c._sum_cost_for_range(start_at, now_utc, mode="auto", **extra)
|
|
746
|
+
if pricing_provenance is not None:
|
|
747
|
+
extra["pricing_provenance"] = pricing_provenance
|
|
745
748
|
return c._sum_codex_cost_for_range(start_at, now_utc, **extra)
|
|
746
749
|
|
|
747
750
|
|
|
748
751
|
def _reconcile_budget_milestones_on_set(
|
|
749
752
|
conn, *, vendor, target, thresholds, now_utc, period, config=None, tz=None,
|
|
750
753
|
as_of=None, commit=True, account_key: str = "*",
|
|
754
|
+
journal_ctx=None,
|
|
751
755
|
):
|
|
752
756
|
"""Forward-only-from-set reconcile for the budget axis (both vendors, #143):
|
|
753
757
|
on `budget set`, every threshold ALREADY crossed for the current
|
|
@@ -773,16 +777,22 @@ def _reconcile_budget_milestones_on_set(
|
|
|
773
777
|
if start_at is None:
|
|
774
778
|
return
|
|
775
779
|
period_key = start_at.isoformat(timespec="seconds")
|
|
780
|
+
provenance = {}
|
|
781
|
+
spend_options = (
|
|
782
|
+
{"pricing_provenance": provenance}
|
|
783
|
+
if vendor == "codex" and journal_ctx is not None else {}
|
|
784
|
+
)
|
|
776
785
|
spent = _budget_spend_for_vendor(
|
|
777
786
|
conn, vendor=vendor, start_at=start_at, now_utc=now_utc,
|
|
778
787
|
account_key=account_key,
|
|
788
|
+
**spend_options,
|
|
779
789
|
)
|
|
780
790
|
# target > 0 guaranteed by the caller (validated weekly_usd / amount_usd);
|
|
781
791
|
# the else is belt-and-suspenders.
|
|
782
792
|
consumption_pct = (spent / target * 100.0) if target > 0 else 0.0
|
|
783
793
|
for t in sorted(thresholds):
|
|
784
794
|
if consumption_pct + 1e-9 >= t:
|
|
785
|
-
insert_budget_milestone(
|
|
795
|
+
inserted = insert_budget_milestone(
|
|
786
796
|
conn,
|
|
787
797
|
vendor=vendor,
|
|
788
798
|
period_start_at=period_key,
|
|
@@ -795,6 +805,17 @@ def _reconcile_budget_milestones_on_set(
|
|
|
795
805
|
as_of=as_of,
|
|
796
806
|
account_key=account_key,
|
|
797
807
|
)
|
|
808
|
+
if inserted == 1 and journal_ctx is not None and provenance:
|
|
809
|
+
row = conn.execute(
|
|
810
|
+
"SELECT id FROM budget_milestones WHERE vendor=? "
|
|
811
|
+
"AND account_key=? AND period_start_at=? AND period=? "
|
|
812
|
+
"AND threshold=?",
|
|
813
|
+
(vendor, account_key, period_key, period, t),
|
|
814
|
+
).fetchone()
|
|
815
|
+
if row is not None:
|
|
816
|
+
journal_ctx.pricing_provenance[("budget", int(row[0]))] = dict(
|
|
817
|
+
provenance
|
|
818
|
+
)
|
|
798
819
|
# alerted_at UPDATE keys on the CONCRETE (vendor, account_key, period)
|
|
799
820
|
# (not the wildcard): only the row we just inserted is stamped, never
|
|
800
821
|
# a pre-011 NULL-period sibling (#137), another vendor's row (#143),
|
|
@@ -878,7 +899,9 @@ def _budget_crossings(
|
|
|
878
899
|
return fired
|
|
879
900
|
|
|
880
901
|
|
|
881
|
-
def _reconcile_codex_budget_on_config_write(
|
|
902
|
+
def _reconcile_codex_budget_on_config_write(
|
|
903
|
+
validated_budget, *, conn=None, as_of=None, journal_ctx=None,
|
|
904
|
+
):
|
|
882
905
|
"""Forward-only reconcile shared by the Codex-budget config write paths
|
|
883
906
|
(`budget set --vendor codex`, `config set budget.codex`). Gated +
|
|
884
907
|
best-effort: a Codex budget with alerts off or no thresholds records
|
|
@@ -907,6 +930,8 @@ def _reconcile_codex_budget_on_config_write(validated_budget, *, conn=None, as_o
|
|
|
907
930
|
config = c.load_config()
|
|
908
931
|
tz = c.resolve_display_tz(argparse.Namespace(tz=None), config)
|
|
909
932
|
now_utc = _as_of_or_command(as_of)
|
|
933
|
+
if as_of is None:
|
|
934
|
+
as_of = now_utc.isoformat().replace("+00:00", "Z")
|
|
910
935
|
# #386 R1-RESIDUAL: an own_conn write is an ADMINISTRATIVE stats
|
|
911
936
|
# writer — outside the ingest cycle and, before this, outside every
|
|
912
937
|
# lock. It now takes the maintenance lock + the sanctioned scope.
|
|
@@ -933,6 +958,7 @@ def _reconcile_codex_budget_on_config_write(validated_budget, *, conn=None, as_o
|
|
|
933
958
|
as_of=as_of,
|
|
934
959
|
commit=False,
|
|
935
960
|
account_key=acct_key,
|
|
961
|
+
journal_ctx=journal_ctx,
|
|
936
962
|
)
|
|
937
963
|
if own_conn:
|
|
938
964
|
conn.commit()
|
package/bin/_cctally_quota.py
CHANGED
|
@@ -356,7 +356,16 @@ PROJECTION_DYNAMIC_READ_SITES: "dict[str, int]" = {
|
|
|
356
356
|
"_cctally_five_hour.py": 1,
|
|
357
357
|
# Sixteen, not eighteen: the two `event {} rev {} conflicts with effective
|
|
358
358
|
# rev {} from {}` refusals are prose and no longer count.
|
|
359
|
-
|
|
359
|
+
#
|
|
360
|
+
# Seventeen since #869: `latch_crossing_was_retired` reads a latch row's
|
|
361
|
+
# natural-key columns with `SELECT … FROM {spec.table} WHERE id = ?` so a
|
|
362
|
+
# live writer can ask whether harvest will journal it as a re-fire. It
|
|
363
|
+
# accepts only the `budget`, `projected` and `project_budget` harvest
|
|
364
|
+
# specs (`_LATCH_REFIRE_ID_PREFIXES`), so `{spec.table}` is always
|
|
365
|
+
# `budget_milestones`, `projected_milestones` or
|
|
366
|
+
# `project_budget_milestones`. None is a projection table, so it needs no
|
|
367
|
+
# entry below.
|
|
368
|
+
"_cctally_journal.py": 17,
|
|
360
369
|
"_cctally_pricing_check.py": 1,
|
|
361
370
|
# One, in `_read_stats_component`: a single `SELECT {column} FROM
|
|
362
371
|
# {table}` shared by `week_reset_events` and `weekly_credit_floors`,
|
package/bin/_cctally_record.py
CHANGED
|
@@ -1041,10 +1041,32 @@ def maybe_record_milestone(
|
|
|
1041
1041
|
conn.close()
|
|
1042
1042
|
|
|
1043
1043
|
|
|
1044
|
+
def _claude_latch_recreates_retired_crossing(conn, kind, rowid) -> bool:
|
|
1045
|
+
"""True iff a just-inserted Claude alert latch re-creates a crossing that a
|
|
1046
|
+
correction retired, so it is stamped but never notified again (#869 V2).
|
|
1047
|
+
|
|
1048
|
+
`db rederive` cannot see historical alert configuration, so it retires
|
|
1049
|
+
every Claude latch (``budget``, ``projected``, ``project_budget``) as a
|
|
1050
|
+
re-materialized projection, never because the crossing was found false.
|
|
1051
|
+
The next live tick of the same period re-inserts the still-crossed
|
|
1052
|
+
threshold. What tells that recreation apart from a first crossing is a
|
|
1053
|
+
durable fact, not timing: the correction's tombstone on the natural key's
|
|
1054
|
+
legacy event, read from the live effective metadata in the writer's own
|
|
1055
|
+
transaction (`latch_crossing_was_retired`). Harvest reads the same fact to
|
|
1056
|
+
journal the row under its version-2 incarnation id. A Codex budget
|
|
1057
|
+
crossing is retired only when its own recorded pricing evidence puts it
|
|
1058
|
+
below the threshold, so its later crossing is genuine and still notifies.
|
|
1059
|
+
"""
|
|
1060
|
+
import _cctally_journal as _jr
|
|
1061
|
+
|
|
1062
|
+
return _jr.latch_crossing_was_retired(conn, kind, rowid)
|
|
1063
|
+
|
|
1064
|
+
|
|
1044
1065
|
def _record_budget_milestone_for_vendor(
|
|
1045
1066
|
*, vendor, target, thresholds, period, config, tz, build_payload,
|
|
1046
1067
|
raise_errors: bool = False, conn=None, as_of=None, alert_sink=None,
|
|
1047
1068
|
account_key: str = "*", window_account_key=None,
|
|
1069
|
+
journal_ctx=None,
|
|
1048
1070
|
) -> int:
|
|
1049
1071
|
"""Shared budget-milestone firing core for both vendors (#143).
|
|
1050
1072
|
|
|
@@ -1077,6 +1099,11 @@ def _record_budget_milestone_for_vendor(
|
|
|
1077
1099
|
"""
|
|
1078
1100
|
import _cctally_cache # for the fail-closed AccountAttributionUnavailable skip (#341)
|
|
1079
1101
|
now_utc = _as_of_or_command(as_of)
|
|
1102
|
+
if vendor == "codex" and as_of is None:
|
|
1103
|
+
# The source scan and durable crossing must share an exact cutoff.
|
|
1104
|
+
# now_utc_iso() reads a second clock and truncates subsecond precision;
|
|
1105
|
+
# preserving this instant also retains a crossing in its final second.
|
|
1106
|
+
as_of = now_utc.isoformat().replace("+00:00", "Z")
|
|
1080
1107
|
pending_alerts: list[dict[str, Any]] = []
|
|
1081
1108
|
own_conn = conn is None
|
|
1082
1109
|
if own_conn:
|
|
@@ -1102,9 +1129,15 @@ def _record_budget_milestone_for_vendor(
|
|
|
1102
1129
|
if not pending:
|
|
1103
1130
|
return 0 # nothing left this window → skip the cost SUM
|
|
1104
1131
|
|
|
1132
|
+
provenance = {}
|
|
1133
|
+
spend_options = (
|
|
1134
|
+
{"pricing_provenance": provenance}
|
|
1135
|
+
if vendor == "codex" and journal_ctx is not None else {}
|
|
1136
|
+
)
|
|
1105
1137
|
spent = _budget_spend_for_vendor(
|
|
1106
1138
|
conn, vendor=vendor, start_at=start_at, now_utc=now_utc,
|
|
1107
1139
|
account_key=account_key,
|
|
1140
|
+
**spend_options,
|
|
1108
1141
|
)
|
|
1109
1142
|
# Shared INSERT-and-arm core (set-then-dispatch, fire-once via rowcount);
|
|
1110
1143
|
# commit=False inside, so this conn owns the single durable commit below.
|
|
@@ -1120,6 +1153,25 @@ def _record_budget_milestone_for_vendor(
|
|
|
1120
1153
|
as_of=as_of,
|
|
1121
1154
|
account_key=account_key,
|
|
1122
1155
|
):
|
|
1156
|
+
row = None
|
|
1157
|
+
if (journal_ctx is not None and provenance) or vendor == "claude":
|
|
1158
|
+
row = conn.execute(
|
|
1159
|
+
"SELECT id FROM budget_milestones WHERE vendor=? "
|
|
1160
|
+
"AND account_key=? AND period_start_at=? AND period=? "
|
|
1161
|
+
"AND threshold=?",
|
|
1162
|
+
(vendor, account_key, period_key, period, t),
|
|
1163
|
+
).fetchone()
|
|
1164
|
+
if journal_ctx is not None and provenance and row is not None:
|
|
1165
|
+
journal_ctx.pricing_provenance[("budget", int(row[0]))] = dict(
|
|
1166
|
+
provenance
|
|
1167
|
+
)
|
|
1168
|
+
if (
|
|
1169
|
+
vendor == "claude" and row is not None
|
|
1170
|
+
and _claude_latch_recreates_retired_crossing(
|
|
1171
|
+
conn, "budget", int(row[0]))
|
|
1172
|
+
):
|
|
1173
|
+
# #869 V2: stamped by `_budget_crossings`, never re-sent.
|
|
1174
|
+
continue
|
|
1123
1175
|
pending_alerts.append(build_payload(
|
|
1124
1176
|
threshold=t,
|
|
1125
1177
|
crossed_at_utc=crossed_at,
|
|
@@ -1423,6 +1475,7 @@ def maybe_record_project_budget_milestone(
|
|
|
1423
1475
|
# racing record-usage instance OR an already-recorded pair gets
|
|
1424
1476
|
# rowcount==0 and skips.
|
|
1425
1477
|
if inserted == 1:
|
|
1478
|
+
rowid = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
|
|
1426
1479
|
crossed_at = as_of or now_utc_iso()
|
|
1427
1480
|
# set-then-dispatch: alerted_at lands on the row BEFORE the
|
|
1428
1481
|
# Popen, sharing this transaction with the INSERT (commit=False).
|
|
@@ -1433,6 +1486,10 @@ def maybe_record_project_budget_milestone(
|
|
|
1433
1486
|
" AND threshold = ? AND alerted_at IS NULL",
|
|
1434
1487
|
(crossed_at, week_key, project_key, t),
|
|
1435
1488
|
)
|
|
1489
|
+
if _claude_latch_recreates_retired_crossing(
|
|
1490
|
+
conn, "project_budget", rowid,
|
|
1491
|
+
):
|
|
1492
|
+
continue # #869 V2: stamped above, never re-sent
|
|
1436
1493
|
# Collision-aware label (shared primitive, #130); resolved once
|
|
1437
1494
|
# on the first dispatch and reused for the rest of this tick.
|
|
1438
1495
|
# Kept defensive fallback (F4).
|
|
@@ -1485,6 +1542,7 @@ def maybe_record_project_budget_milestone(
|
|
|
1485
1542
|
def maybe_record_codex_budget_milestone(
|
|
1486
1543
|
saved: dict[str, Any], *, raise_errors: bool = False, conn=None, as_of=None,
|
|
1487
1544
|
alert_sink: "list | None" = None,
|
|
1545
|
+
journal_ctx=None,
|
|
1488
1546
|
) -> int:
|
|
1489
1547
|
"""Fire Codex budget alerts on ACTUAL-Codex-spend threshold crossings (axis
|
|
1490
1548
|
``codex_budget``, calendar-period-codex-budgets spec §6 — the gap the Codex
|
|
@@ -1560,6 +1618,7 @@ def maybe_record_codex_budget_milestone(
|
|
|
1560
1618
|
build_payload=_codex_budget_payload, account_key="*",
|
|
1561
1619
|
raise_errors=raise_errors, conn=conn, as_of=as_of,
|
|
1562
1620
|
alert_sink=alert_sink,
|
|
1621
|
+
journal_ctx=journal_ctx,
|
|
1563
1622
|
) or 0
|
|
1564
1623
|
for acct_key, acct_usd in accounts.items():
|
|
1565
1624
|
fired += _record_budget_milestone_for_vendor(
|
|
@@ -1568,6 +1627,7 @@ def maybe_record_codex_budget_milestone(
|
|
|
1568
1627
|
build_payload=_codex_budget_payload, account_key=acct_key,
|
|
1569
1628
|
raise_errors=raise_errors, conn=conn, as_of=as_of,
|
|
1570
1629
|
alert_sink=alert_sink,
|
|
1630
|
+
journal_ctx=journal_ctx,
|
|
1571
1631
|
) or 0
|
|
1572
1632
|
return fired
|
|
1573
1633
|
|
|
@@ -1998,6 +2058,7 @@ def maybe_record_projected_alert(
|
|
|
1998
2058
|
# a racing record-usage instance gets rowcount==0 and skips. The
|
|
1999
2059
|
# alerted_at UPDATE keys on the CONCRETE `period` (#137).
|
|
2000
2060
|
if inserted == 1:
|
|
2061
|
+
rowid = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
|
|
2001
2062
|
conn.execute(
|
|
2002
2063
|
"UPDATE projected_milestones SET alerted_at = ? "
|
|
2003
2064
|
"WHERE week_start_at = ? AND period = ? AND metric = ? "
|
|
@@ -2005,6 +2066,12 @@ def maybe_record_projected_alert(
|
|
|
2005
2066
|
(as_of or now_utc_iso(), p["week_start_at"], p["period"],
|
|
2006
2067
|
p["metric"], p["threshold"]),
|
|
2007
2068
|
)
|
|
2069
|
+
if (
|
|
2070
|
+
not str(p["metric"]).startswith("codex_")
|
|
2071
|
+
and _claude_latch_recreates_retired_crossing(
|
|
2072
|
+
conn, "projected", rowid)
|
|
2073
|
+
):
|
|
2074
|
+
continue # #869 V2: stamped above, never re-sent
|
|
2008
2075
|
fired.append(p)
|
|
2009
2076
|
# Single commit: every INSERT + its alerted_at marker durable together.
|
|
2010
2077
|
# On the passed-conn (ingest) path the caller owns commit + dispatch.
|
|
@@ -2232,6 +2299,8 @@ def _compute_block_totals(
|
|
|
2232
2299
|
owner_key: Any,
|
|
2233
2300
|
windows: "Iterable[Any]",
|
|
2234
2301
|
skip_sync: bool = False,
|
|
2302
|
+
pricing_evidence: "dict | None" = None,
|
|
2303
|
+
owned_entries: "list | None" = None,
|
|
2235
2304
|
) -> dict[str, Any]:
|
|
2236
2305
|
"""Sum tokens + cost over the entries the target block OWNS within
|
|
2237
2306
|
[block_start_at, range_end], plus per-model and per-project breakdowns
|
|
@@ -2275,6 +2344,9 @@ def _compute_block_totals(
|
|
|
2275
2344
|
f"the competing-window context"
|
|
2276
2345
|
)
|
|
2277
2346
|
|
|
2347
|
+
missing_without_raw = set()
|
|
2348
|
+
observed_raw = set()
|
|
2349
|
+
|
|
2278
2350
|
def _priced():
|
|
2279
2351
|
loaded = get_claude_session_entries(
|
|
2280
2352
|
block_start_at, range_end, skip_sync=skip_sync,
|
|
@@ -2283,6 +2355,20 @@ def _compute_block_totals(
|
|
|
2283
2355
|
loaded, ownership_windows,
|
|
2284
2356
|
)
|
|
2285
2357
|
for entry in owned[owner_key]:
|
|
2358
|
+
if owned_entries is not None:
|
|
2359
|
+
owned_entries.append(entry)
|
|
2360
|
+
if pricing_evidence is not None and (
|
|
2361
|
+
entry.input_tokens or entry.output_tokens
|
|
2362
|
+
or entry.cache_creation_tokens or entry.cache_read_tokens
|
|
2363
|
+
):
|
|
2364
|
+
if entry.cost_usd is not None:
|
|
2365
|
+
observed_raw.add(entry.model)
|
|
2366
|
+
else:
|
|
2367
|
+
import _lib_pricing
|
|
2368
|
+
if _lib_pricing._resolve_model_pricing(
|
|
2369
|
+
entry.model, warn=False,
|
|
2370
|
+
) is None:
|
|
2371
|
+
missing_without_raw.add(entry.model)
|
|
2286
2372
|
usage = claude_usage_dict( # #195 chokepoint
|
|
2287
2373
|
input_tokens=entry.input_tokens,
|
|
2288
2374
|
output_tokens=entry.output_tokens,
|
|
@@ -2304,7 +2390,15 @@ def _compute_block_totals(
|
|
|
2304
2390
|
cost_usd=cost,
|
|
2305
2391
|
)
|
|
2306
2392
|
|
|
2307
|
-
|
|
2393
|
+
result = fold_block_totals(_priced()).as_legacy_dict()
|
|
2394
|
+
if pricing_evidence is not None:
|
|
2395
|
+
pricing_evidence["unpricedModels"] = sorted(
|
|
2396
|
+
missing_without_raw - observed_raw
|
|
2397
|
+
)
|
|
2398
|
+
ambiguous = sorted(missing_without_raw & observed_raw)
|
|
2399
|
+
if ambiguous:
|
|
2400
|
+
pricing_evidence["uncertainModels"] = ambiguous
|
|
2401
|
+
return result
|
|
2308
2402
|
|
|
2309
2403
|
|
|
2310
2404
|
def maybe_update_five_hour_block(
|
|
@@ -2455,10 +2549,41 @@ def maybe_update_five_hour_block(
|
|
|
2455
2549
|
load_start=block_start_dt,
|
|
2456
2550
|
load_end=captured_at_dt,
|
|
2457
2551
|
)
|
|
2552
|
+
pricing_evidence = {}
|
|
2553
|
+
# `db rederive`'s scratch replay alone supplies this sink (#869), keyed
|
|
2554
|
+
# by the blocks whose historical close it must prove. It keeps the
|
|
2555
|
+
# exact entries behind the totals this tick applies, so the planner
|
|
2556
|
+
# proves a missing card from the very population that produced the
|
|
2557
|
+
# replayed children. A frozen close's upsert applies nothing, so its
|
|
2558
|
+
# later recomputations are not kept.
|
|
2559
|
+
source_sink = getattr(journal_ctx, "block_source_evidence", None)
|
|
2560
|
+
source_key = (account_key, int(five_hour_window_key))
|
|
2561
|
+
owned_entries = (
|
|
2562
|
+
[] if source_sink is not None and source_key in source_sink
|
|
2563
|
+
and not current_is_frozen else None
|
|
2564
|
+
)
|
|
2458
2565
|
totals = _compute_block_totals(
|
|
2459
2566
|
block_start_dt, captured_at_dt,
|
|
2460
2567
|
owner_key=int(five_hour_window_key),
|
|
2461
2568
|
windows=ownership_windows,
|
|
2569
|
+
pricing_evidence=pricing_evidence,
|
|
2570
|
+
owned_entries=owned_entries,
|
|
2571
|
+
)
|
|
2572
|
+
if owned_entries is not None:
|
|
2573
|
+
source_sink[source_key] = tuple(owned_entries)
|
|
2574
|
+
# This describes the pricing used for THESE computed totals. A later
|
|
2575
|
+
# successor closes this block without recomputing it, so looking up
|
|
2576
|
+
# cards while harvesting the close would describe the wrong binary.
|
|
2577
|
+
import _lib_pricing
|
|
2578
|
+
pricing_marker = {
|
|
2579
|
+
"version": 1,
|
|
2580
|
+
"pricingDate": _lib_pricing.current_pricing_snapshot().snapshot_date,
|
|
2581
|
+
"unpricedModels": pricing_evidence["unpricedModels"],
|
|
2582
|
+
}
|
|
2583
|
+
if pricing_evidence.get("uncertainModels"):
|
|
2584
|
+
pricing_marker["uncertainModels"] = pricing_evidence["uncertainModels"]
|
|
2585
|
+
pricing_provenance_json = json.dumps(
|
|
2586
|
+
pricing_marker, sort_keys=True, separators=(",", ":"),
|
|
2462
2587
|
)
|
|
2463
2588
|
|
|
2464
2589
|
# Hoist alerts config above BEGIN (M1 + M2): single read serves
|
|
@@ -2607,12 +2732,13 @@ def maybe_update_five_hour_block(
|
|
|
2607
2732
|
total_cache_create_tokens,
|
|
2608
2733
|
total_cache_read_tokens,
|
|
2609
2734
|
total_cost_usd,
|
|
2735
|
+
pricing_provenance_json,
|
|
2610
2736
|
is_closed,
|
|
2611
2737
|
created_at_utc,
|
|
2612
2738
|
last_updated_at_utc,
|
|
2613
2739
|
account_key
|
|
2614
2740
|
)
|
|
2615
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, 0, ?, ?, ?)
|
|
2741
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?)
|
|
2616
2742
|
ON CONFLICT(account_key, five_hour_window_key) DO UPDATE SET
|
|
2617
2743
|
last_observed_at_utc = excluded.last_observed_at_utc,
|
|
2618
2744
|
final_five_hour_percent = excluded.final_five_hour_percent,
|
|
@@ -2622,6 +2748,7 @@ def maybe_update_five_hour_block(
|
|
|
2622
2748
|
total_cache_create_tokens = excluded.total_cache_create_tokens,
|
|
2623
2749
|
total_cache_read_tokens = excluded.total_cache_read_tokens,
|
|
2624
2750
|
total_cost_usd = excluded.total_cost_usd,
|
|
2751
|
+
pricing_provenance_json = excluded.pricing_provenance_json,
|
|
2625
2752
|
last_updated_at_utc = excluded.last_updated_at_utc
|
|
2626
2753
|
WHERE five_hour_blocks.is_closed = 0
|
|
2627
2754
|
OR five_hour_blocks.journal_id IS NULL
|
|
@@ -2641,6 +2768,7 @@ def maybe_update_five_hour_block(
|
|
|
2641
2768
|
totals["cache_create_tokens"],
|
|
2642
2769
|
totals["cache_read_tokens"],
|
|
2643
2770
|
totals["cost_usd"],
|
|
2771
|
+
pricing_provenance_json,
|
|
2644
2772
|
now_iso,
|
|
2645
2773
|
now_iso,
|
|
2646
2774
|
account_key,
|
|
@@ -6369,6 +6497,7 @@ def _cmd_hook_tick_codex(
|
|
|
6369
6497
|
def _codex_budget_leg(ictx):
|
|
6370
6498
|
_budget_holder["n"] = int(c.maybe_record_codex_budget_milestone(
|
|
6371
6499
|
{}, conn=ictx.conn, alert_sink=ictx.pending_alerts,
|
|
6500
|
+
journal_ctx=ictx,
|
|
6372
6501
|
raise_errors=True) or 0)
|
|
6373
6502
|
|
|
6374
6503
|
budget_ingest = _jr_codex.run_stats_ingest(
|
|
@@ -7134,7 +7263,8 @@ def _derive_5h_window_key(conn, five_hour_resets_at_epoch):
|
|
|
7134
7263
|
)
|
|
7135
7264
|
|
|
7136
7265
|
|
|
7137
|
-
def _run_dollar_axes(saved, *, conn, as_of, alert_sink, enabled=True
|
|
7266
|
+
def _run_dollar_axes(saved, *, conn, as_of, alert_sink, enabled=True,
|
|
7267
|
+
journal_ctx=None):
|
|
7138
7268
|
"""The four dollar-decoupled alert axes in cmd_record_usage's legacy order
|
|
7139
7269
|
(budget → project-budget → codex-budget → projected). Runs on BOTH the accept
|
|
7140
7270
|
path AND every dedup-skip tick (spec §4.5: USD spend can cross a $ threshold
|
|
@@ -7149,7 +7279,8 @@ def _run_dollar_axes(saved, *, conn, as_of, alert_sink, enabled=True):
|
|
|
7149
7279
|
c.maybe_record_project_budget_milestone(
|
|
7150
7280
|
saved, conn=conn, as_of=as_of, alert_sink=alert_sink)
|
|
7151
7281
|
c.maybe_record_codex_budget_milestone(
|
|
7152
|
-
saved, conn=conn, as_of=as_of, alert_sink=alert_sink
|
|
7282
|
+
saved, conn=conn, as_of=as_of, alert_sink=alert_sink,
|
|
7283
|
+
journal_ctx=journal_ctx)
|
|
7153
7284
|
c.maybe_record_projected_alert(
|
|
7154
7285
|
saved, conn=conn, as_of=as_of, alert_sink=alert_sink)
|
|
7155
7286
|
|
|
@@ -7561,6 +7692,7 @@ def _pipeline_claude_usage(ctx, rec):
|
|
|
7561
7692
|
conn=conn,
|
|
7562
7693
|
as_of=as_of,
|
|
7563
7694
|
alert_sink=ctx.pending_alerts,
|
|
7695
|
+
journal_ctx=ctx,
|
|
7564
7696
|
# Budget/projected rows depend on historical config that was not retained
|
|
7565
7697
|
# in the journal. Task B classifies them as re-materialized projections:
|
|
7566
7698
|
# the planner retires stale latches and never fabricates old config.
|