cctally 1.111.0 → 1.113.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/README.md +6 -6
  3. package/bin/_cctally_cache.py +14 -7
  4. package/bin/_cctally_core.py +9 -1
  5. package/bin/_cctally_dashboard_envelope.py +7 -6
  6. package/bin/_cctally_dashboard_sources.py +90 -55
  7. package/bin/_cctally_doctor.py +73 -27
  8. package/bin/_cctally_forecast.py +2 -1
  9. package/bin/_cctally_journal.py +147 -6
  10. package/bin/_cctally_milestones.py +29 -3
  11. package/bin/_cctally_quota.py +10 -1
  12. package/bin/_cctally_record.py +136 -4
  13. package/bin/_cctally_rederive.py +311 -8
  14. package/bin/_cctally_source_analytics.py +20 -8
  15. package/bin/_cctally_tui.py +546 -45
  16. package/bin/_lib_codex_conversation_query.py +40 -4
  17. package/bin/_lib_codex_metadata.py +57 -11
  18. package/bin/_lib_cost_provenance.py +303 -0
  19. package/bin/_lib_dashboard_sources.py +263 -28
  20. package/bin/_lib_doctor.py +2 -2
  21. package/bin/_lib_ingest_frontier.py +137 -38
  22. package/bin/_lib_pricing.py +29 -1
  23. package/bin/_lib_rederive.py +704 -36
  24. package/bin/_lib_snapshot_cache.py +332 -9
  25. package/dashboard/static/assets/{ConversationsView-BB9vUiL6.js → ConversationsView-CEZ-BUyi.js} +1 -1
  26. package/dashboard/static/assets/{DoctorModal-DFY2iPDf.js → DoctorModal-CE1oH_Zy.js} +1 -1
  27. package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +1 -0
  28. package/dashboard/static/assets/{ProjectsDrillPanel-DAv39uSo.js → ProjectsDrillPanel-B3DZoCeD.js} +1 -1
  29. package/dashboard/static/assets/{SourceDetailModal-DSAWo2Fp.js → SourceDetailModal-CHp9EgLP.js} +1 -1
  30. package/dashboard/static/assets/{UpdateModal-R_9dOVnh.js → UpdateModal-5iKST6jo.js} +1 -1
  31. package/dashboard/static/assets/{index-DdU62sov.js → index-Cm0VTxJC.js} +2 -2
  32. package/dashboard/static/assets/index-Dt4tqf43.css +1 -0
  33. package/dashboard/static/assets/{outlineNavigation-BOCLcRa6.js → outlineNavigation-DkXGL8J6.js} +1 -1
  34. package/dashboard/static/dashboard.html +2 -2
  35. package/package.json +2 -1
  36. package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +0 -1
  37. package/dashboard/static/assets/index-COhxe3vU.css +0 -1
@@ -1409,12 +1409,19 @@ class IngestContext:
1409
1409
  # (reset INSERT OR IGNORE rowcount == 1), so a crash-replayed reset never
1410
1410
  # re-suppresses with a divergent list.
1411
1411
  suppression_map: dict = field(default_factory=dict)
1412
+ # Indexed by (event kind, local row id) until harvest persists the marker.
1413
+ pricing_provenance: dict = field(default_factory=dict)
1412
1414
  # Task B rederive seam. Normal ingest leaves both defaults unchanged.
1413
1415
  # A scratch planner supplies an in-memory sink so derived events are captured
1414
1416
  # instead of appended to the durable journal, and disables projection-file
1415
1417
  # writes while still exercising the same SQLite derivation/fold code.
1416
1418
  event_sink: "list | None" = None
1417
1419
  projection_writes: bool = True
1420
+ # #869 `db rederive` scratch replay only. Its keys name the (account_key,
1421
+ # window key) blocks the planner must prove; each value becomes the
1422
+ # retained entries behind the totals that block last applied. Live ingest
1423
+ # leaves it None and keeps no entries.
1424
+ block_source_evidence: "dict | None" = None
1418
1425
  # Operator-reviewed scratch replay only. Normal ingest leaves this empty.
1419
1426
  held_weekly_observation_ids: frozenset[str] = field(default_factory=frozenset)
1420
1427
  reviewed_weekly_basis_by_account: dict = field(default_factory=dict)
@@ -4127,7 +4134,10 @@ def _apply_generic_evt(conn, evt):
4127
4134
  return None
4128
4135
  cols = {"journal_id": evt["id"]}
4129
4136
  for key, value in payload.items():
4130
- if key == "kind":
4137
+ # `journal_identity_version` names how the event id was minted (#869
4138
+ # alert-latch re-fire incarnations: bm2/pjm2/pbm2); it is journal
4139
+ # metadata, never a column.
4140
+ if key in ("kind", "_pricing", "journal_identity_version"):
4131
4141
  continue
4132
4142
  if key in spec.fk_refs:
4133
4143
  column, ref_table = spec.fk_refs[key]
@@ -4493,7 +4503,7 @@ def _apply_block_close(conn, evt):
4493
4503
  parent = {"journal_id": evt["id"]}
4494
4504
  children = {}
4495
4505
  for key, value in payload.items():
4496
- if key == "kind":
4506
+ if key in ("kind", "_pricing"):
4497
4507
  continue
4498
4508
  if key in _BLOCK_CHILD_KEYS:
4499
4509
  children[key] = value or []
@@ -4552,7 +4562,7 @@ def _apply_reset_with_suppression(conn, evt):
4552
4562
  return None
4553
4563
  cols = {"journal_id": evt["id"]}
4554
4564
  for key, value in payload.items():
4555
- if key in ("kind", "suppression", "suppression_table"):
4565
+ if key in ("kind", "suppression", "suppression_table", "_pricing"):
4556
4566
  continue
4557
4567
  if key in spec.fk_refs:
4558
4568
  column, ref_table = spec.fk_refs[key]
@@ -4834,6 +4844,59 @@ def _build_harvest_evt(ctx, spec, row):
4834
4844
  {k: cr[k] for k in cr.keys() if k not in ("id", "block_id")}
4835
4845
  for cr in child_rows
4836
4846
  ]
4847
+ if spec.kind == "five_hour_block_close":
4848
+ raw_pricing = row["pricing_provenance_json"]
4849
+ if raw_pricing is not None:
4850
+ try:
4851
+ pricing = json.loads(raw_pricing)
4852
+ except (TypeError, ValueError) as exc:
4853
+ raise JournalError(
4854
+ "five_hour_block_close has malformed computation-time "
4855
+ "pricing provenance"
4856
+ ) from exc
4857
+ if (
4858
+ not isinstance(pricing, dict)
4859
+ or pricing.get("version") != 1
4860
+ or not isinstance(pricing.get("pricingDate"), str)
4861
+ or not pricing["pricingDate"]
4862
+ or not isinstance(pricing.get("unpricedModels"), list)
4863
+ or any(not isinstance(model, str) or not model
4864
+ for model in pricing["unpricedModels"])
4865
+ or pricing["unpricedModels"] != sorted(set(
4866
+ pricing["unpricedModels"]
4867
+ ))
4868
+ or (
4869
+ "uncertainModels" in pricing
4870
+ and (
4871
+ not isinstance(pricing["uncertainModels"], list)
4872
+ or any(not isinstance(model, str) or not model
4873
+ for model in pricing["uncertainModels"])
4874
+ or pricing["uncertainModels"] != sorted(set(
4875
+ pricing["uncertainModels"]
4876
+ ))
4877
+ )
4878
+ )
4879
+ ):
4880
+ raise JournalError(
4881
+ "five_hour_block_close has incomplete computation-time "
4882
+ "pricing provenance"
4883
+ )
4884
+ positive_models = {
4885
+ child["model"] for child in payload.get("_models", [])
4886
+ if sum(int(child.get(key) or 0) for key in (
4887
+ "input_tokens", "output_tokens", "cache_create_tokens",
4888
+ "cache_read_tokens",
4889
+ )) > 0
4890
+ }
4891
+ if not (
4892
+ set(pricing["unpricedModels"])
4893
+ | set(pricing.get("uncertainModels") or [])
4894
+ ) <= positive_models:
4895
+ raise JournalError(
4896
+ "five_hour_block_close pricing provenance does not match "
4897
+ "its computed model population"
4898
+ )
4899
+ payload["_pricing"] = pricing
4837
4900
  # #750 S3 §1.1: ONE producer for the id parts and for the suppression-map
4838
4901
  # key. A family with an `id_parts_fn` (today only `week_reset_events`,
4839
4902
  # whose identity is dual-shaped) supplies both from that callable; every
@@ -4851,11 +4914,85 @@ def _build_harvest_evt(ctx, spec, row):
4851
4914
  supp = ctx.suppression_map.get(supp_key)
4852
4915
  if supp:
4853
4916
  payload["suppression"] = list(supp)
4917
+ provenance = ctx.pricing_provenance.get((spec.kind, int(row["id"])))
4918
+ if provenance is not None:
4919
+ payload["_pricing"] = provenance
4854
4920
  eid = _lib_journal.evt_id(spec.id_prefix, *parts)
4921
+ refire_prefix = _LATCH_REFIRE_ID_PREFIXES.get(spec.kind)
4922
+ if (
4923
+ refire_prefix is not None
4924
+ and ctx.event_sink is None
4925
+ and _latch_event_was_retired(conn, eid)
4926
+ ):
4927
+ # #869 F6/V1: a correction retired this natural key's earlier crossing
4928
+ # with a higher-revision tombstone, so a revision-0 event under the
4929
+ # same id can never become effective. A later crossing is a new
4930
+ # incarnation: its id adds the crossing instant, while the table's
4931
+ # natural key still deduplicates it. Every other crossing, and every
4932
+ # already-journaled id, keeps the legacy spelling.
4933
+ eid = _lib_journal.evt_id(
4934
+ refire_prefix, *parts, row["crossed_at_utc"])
4935
+ payload["journal_identity_version"] = 2
4855
4936
  at = row[spec.at_column] if spec.at_column else _now_iso()
4856
4937
  return _lib_journal.make_evt(kind=spec.kind, id=eid, at=at, payload=payload)
4857
4938
 
4858
4939
 
4940
+ #: #869 F6/V1. The id prefix, per alert-latch kind, of a crossing that re-fires
4941
+ #: after a correction tombstoned the natural key's earlier crossing (the legacy
4942
+ #: prefix + identity version 2). `db rederive` retires these latches, so each
4943
+ #: one needs a spelling a later live crossing can journal under.
4944
+ #:
4945
+ #: Coupling: the live writers (`_claude_latch_recreates_retired_crossing` in
4946
+ #: `_cctally_record`) stamp a recreated CLAUDE latch without notifying, on the
4947
+ #: premise that every Claude-latch tombstone is the claude-usage family's
4948
+ #: stale-config retirement, never a verdict that the crossing was false. A
4949
+ #: future rule that retires a Claude latch as a false crossing must record
4950
+ #: that distinction durably, or the genuine later crossing is silently
4951
+ #: suppressed.
4952
+ _LATCH_REFIRE_ID_PREFIXES = {
4953
+ "budget": "bm2",
4954
+ "projected": "pjm2",
4955
+ "project_budget": "pbm2",
4956
+ }
4957
+
4958
+
4959
+ def _latch_event_was_retired(conn, legacy_event_id) -> bool:
4960
+ """True iff the effective selection for this natural-key id is a tombstone.
4961
+
4962
+ Read from the live effective metadata the rebuild published, never
4963
+ recomputed. A missing row or an active event keeps the legacy id.
4964
+ """
4965
+ prior = _metadata_row(conn, legacy_event_id)
4966
+ return prior is not None and str(prior[1]) == "tombstone"
4967
+
4968
+
4969
+ def latch_crossing_was_retired(conn, kind, rowid) -> bool:
4970
+ """True iff alert-latch row ``rowid`` re-creates a retired crossing (#869).
4971
+
4972
+ ``kind`` is a latch family with a re-fire incarnation (``budget``,
4973
+ ``projected``, ``project_budget``). The row's legacy natural-key event id
4974
+ is minted exactly as harvest mints it, and the answer is the live effective
4975
+ metadata's: a tombstone means a completed correction batch retired an
4976
+ earlier crossing of this natural key, so harvest will journal this row
4977
+ under the kind's version-2 prefix. A live writer calls this in the same
4978
+ transaction as its insert, before harvest runs, so both decisions read one
4979
+ durable fact.
4980
+ """
4981
+ if kind not in _LATCH_REFIRE_ID_PREFIXES:
4982
+ raise ValueError(f"{kind!r} has no re-fire incarnation")
4983
+ [spec] = [item for item in _HARVEST_SPECS if item.kind == kind]
4984
+ if spec.fk_refs or spec.id_parts_fn is not None:
4985
+ raise ValueError(f"{kind!r} ids are not plain natural-key columns")
4986
+ row = conn.execute(
4987
+ f"SELECT {', '.join(spec.id_parts)} FROM {spec.table} WHERE id = ?",
4988
+ (int(rowid),),
4989
+ ).fetchone()
4990
+ if row is None:
4991
+ return False
4992
+ return _latch_event_was_retired(
4993
+ conn, _lib_journal.evt_id(spec.id_prefix, *tuple(row)))
4994
+
4995
+
4859
4996
  def _emit_harvest_row(ctx, spec, row):
4860
4997
  """Journal and stamp one already-selected natural-keyed row."""
4861
4998
  conn = ctx.conn
@@ -5334,7 +5471,10 @@ def _run_config_reconcile(ctx, reconcile_config) -> None:
5334
5471
  if fn is None:
5335
5472
  continue
5336
5473
  try:
5337
- fn(validated_budget, conn=ctx.conn)
5474
+ if axis == "codex_budget":
5475
+ fn(validated_budget, conn=ctx.conn, journal_ctx=ctx)
5476
+ else:
5477
+ fn(validated_budget, conn=ctx.conn)
5338
5478
  except Exception as exc: # best-effort; never break the cycle over a reconcile
5339
5479
  print(f"[budget-reconcile] {name} failed: {exc}", file=sys.stderr)
5340
5480
  # Per-project reconcile takes `touched_projects` as its 2nd positional
@@ -6086,7 +6226,8 @@ def _effective_event_for_convergence(conn, event_id) -> dict:
6086
6226
 
6087
6227
  # Effect keys that ride an evt payload but are NOT target-table columns.
6088
6228
  _EVT_EFFECT_KEYS = frozenset(
6089
- {"kind", "suppression", "suppression_table", "floor_suppression", "hwm_floor"}
6229
+ {"kind", "suppression", "suppression_table", "floor_suppression",
6230
+ "hwm_floor", "_pricing", "journal_identity_version"}
6090
6231
  )
6091
6232
 
6092
6233
 
@@ -8020,7 +8161,7 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
8020
8161
  # omitted column, constraint, partial predicate, or index definition. An epoch
8021
8162
  # schema change must update this contract alongside STATS_INDEX_EPOCH.
8022
8163
  _REBUILD_SCHEMA_FINGERPRINT = (
8023
- "b8f18315bce871bd9692ba1db73cf7e2bda716b360616fc65149ce209248906d"
8164
+ "e0f2225f00843952b4887fa8afb046edacd3da0642d0f402ebaa5219a6e30697"
8024
8165
  )
8025
8166
 
8026
8167
 
@@ -721,7 +721,8 @@ def _resolve_budget_window(conn, *, vendor, now_utc, period, config, tz,
721
721
 
722
722
 
723
723
  def _budget_spend_for_vendor(conn, *, vendor, start_at, now_utc,
724
- account_key: str = "*") -> float:
724
+ account_key: str = "*",
725
+ pricing_provenance: "dict | None" = None) -> float:
725
726
  """Spend over ``[start_at, now]`` for ``vendor`` (#143) — the COSTLY leg,
726
727
  called only after the pre-probe finds pending thresholds (spec §4.2). claude
727
728
  routes through the Claude cost SUM (``mode="auto"``); codex through the Codex
@@ -742,12 +743,15 @@ def _budget_spend_for_vendor(conn, *, vendor, start_at, now_utc,
742
743
  extra = {} if scope is None else {"account_key": scope}
743
744
  if vendor == "claude":
744
745
  return c._sum_cost_for_range(start_at, now_utc, mode="auto", **extra)
746
+ if pricing_provenance is not None:
747
+ extra["pricing_provenance"] = pricing_provenance
745
748
  return c._sum_codex_cost_for_range(start_at, now_utc, **extra)
746
749
 
747
750
 
748
751
  def _reconcile_budget_milestones_on_set(
749
752
  conn, *, vendor, target, thresholds, now_utc, period, config=None, tz=None,
750
753
  as_of=None, commit=True, account_key: str = "*",
754
+ journal_ctx=None,
751
755
  ):
752
756
  """Forward-only-from-set reconcile for the budget axis (both vendors, #143):
753
757
  on `budget set`, every threshold ALREADY crossed for the current
@@ -773,16 +777,22 @@ def _reconcile_budget_milestones_on_set(
773
777
  if start_at is None:
774
778
  return
775
779
  period_key = start_at.isoformat(timespec="seconds")
780
+ provenance = {}
781
+ spend_options = (
782
+ {"pricing_provenance": provenance}
783
+ if vendor == "codex" and journal_ctx is not None else {}
784
+ )
776
785
  spent = _budget_spend_for_vendor(
777
786
  conn, vendor=vendor, start_at=start_at, now_utc=now_utc,
778
787
  account_key=account_key,
788
+ **spend_options,
779
789
  )
780
790
  # target > 0 guaranteed by the caller (validated weekly_usd / amount_usd);
781
791
  # the else is belt-and-suspenders.
782
792
  consumption_pct = (spent / target * 100.0) if target > 0 else 0.0
783
793
  for t in sorted(thresholds):
784
794
  if consumption_pct + 1e-9 >= t:
785
- insert_budget_milestone(
795
+ inserted = insert_budget_milestone(
786
796
  conn,
787
797
  vendor=vendor,
788
798
  period_start_at=period_key,
@@ -795,6 +805,17 @@ def _reconcile_budget_milestones_on_set(
795
805
  as_of=as_of,
796
806
  account_key=account_key,
797
807
  )
808
+ if inserted == 1 and journal_ctx is not None and provenance:
809
+ row = conn.execute(
810
+ "SELECT id FROM budget_milestones WHERE vendor=? "
811
+ "AND account_key=? AND period_start_at=? AND period=? "
812
+ "AND threshold=?",
813
+ (vendor, account_key, period_key, period, t),
814
+ ).fetchone()
815
+ if row is not None:
816
+ journal_ctx.pricing_provenance[("budget", int(row[0]))] = dict(
817
+ provenance
818
+ )
798
819
  # alerted_at UPDATE keys on the CONCRETE (vendor, account_key, period)
799
820
  # (not the wildcard): only the row we just inserted is stamped, never
800
821
  # a pre-011 NULL-period sibling (#137), another vendor's row (#143),
@@ -878,7 +899,9 @@ def _budget_crossings(
878
899
  return fired
879
900
 
880
901
 
881
- def _reconcile_codex_budget_on_config_write(validated_budget, *, conn=None, as_of=None):
902
+ def _reconcile_codex_budget_on_config_write(
903
+ validated_budget, *, conn=None, as_of=None, journal_ctx=None,
904
+ ):
882
905
  """Forward-only reconcile shared by the Codex-budget config write paths
883
906
  (`budget set --vendor codex`, `config set budget.codex`). Gated +
884
907
  best-effort: a Codex budget with alerts off or no thresholds records
@@ -907,6 +930,8 @@ def _reconcile_codex_budget_on_config_write(validated_budget, *, conn=None, as_o
907
930
  config = c.load_config()
908
931
  tz = c.resolve_display_tz(argparse.Namespace(tz=None), config)
909
932
  now_utc = _as_of_or_command(as_of)
933
+ if as_of is None:
934
+ as_of = now_utc.isoformat().replace("+00:00", "Z")
910
935
  # #386 R1-RESIDUAL: an own_conn write is an ADMINISTRATIVE stats
911
936
  # writer — outside the ingest cycle and, before this, outside every
912
937
  # lock. It now takes the maintenance lock + the sanctioned scope.
@@ -933,6 +958,7 @@ def _reconcile_codex_budget_on_config_write(validated_budget, *, conn=None, as_o
933
958
  as_of=as_of,
934
959
  commit=False,
935
960
  account_key=acct_key,
961
+ journal_ctx=journal_ctx,
936
962
  )
937
963
  if own_conn:
938
964
  conn.commit()
@@ -356,7 +356,16 @@ PROJECTION_DYNAMIC_READ_SITES: "dict[str, int]" = {
356
356
  "_cctally_five_hour.py": 1,
357
357
  # Sixteen, not eighteen: the two `event {} rev {} conflicts with effective
358
358
  # rev {} from {}` refusals are prose and no longer count.
359
- "_cctally_journal.py": 16,
359
+ #
360
+ # Seventeen since #869: `latch_crossing_was_retired` reads a latch row's
361
+ # natural-key columns with `SELECT … FROM {spec.table} WHERE id = ?` so a
362
+ # live writer can ask whether harvest will journal it as a re-fire. It
363
+ # accepts only the `budget`, `projected` and `project_budget` harvest
364
+ # specs (`_LATCH_REFIRE_ID_PREFIXES`), so `{spec.table}` is always
365
+ # `budget_milestones`, `projected_milestones` or
366
+ # `project_budget_milestones`. None is a projection table, so it needs no
367
+ # entry below.
368
+ "_cctally_journal.py": 17,
360
369
  "_cctally_pricing_check.py": 1,
361
370
  # One, in `_read_stats_component`: a single `SELECT {column} FROM
362
371
  # {table}` shared by `week_reset_events` and `weekly_credit_floors`,
@@ -1041,10 +1041,32 @@ def maybe_record_milestone(
1041
1041
  conn.close()
1042
1042
 
1043
1043
 
1044
+ def _claude_latch_recreates_retired_crossing(conn, kind, rowid) -> bool:
1045
+ """True iff a just-inserted Claude alert latch re-creates a crossing that a
1046
+ correction retired, so it is stamped but never notified again (#869 V2).
1047
+
1048
+ `db rederive` cannot see historical alert configuration, so it retires
1049
+ every Claude latch (``budget``, ``projected``, ``project_budget``) as a
1050
+ re-materialized projection, never because the crossing was found false.
1051
+ The next live tick of the same period re-inserts the still-crossed
1052
+ threshold. What tells that recreation apart from a first crossing is a
1053
+ durable fact, not timing: the correction's tombstone on the natural key's
1054
+ legacy event, read from the live effective metadata in the writer's own
1055
+ transaction (`latch_crossing_was_retired`). Harvest reads the same fact to
1056
+ journal the row under its version-2 incarnation id. A Codex budget
1057
+ crossing is retired only when its own recorded pricing evidence puts it
1058
+ below the threshold, so its later crossing is genuine and still notifies.
1059
+ """
1060
+ import _cctally_journal as _jr
1061
+
1062
+ return _jr.latch_crossing_was_retired(conn, kind, rowid)
1063
+
1064
+
1044
1065
  def _record_budget_milestone_for_vendor(
1045
1066
  *, vendor, target, thresholds, period, config, tz, build_payload,
1046
1067
  raise_errors: bool = False, conn=None, as_of=None, alert_sink=None,
1047
1068
  account_key: str = "*", window_account_key=None,
1069
+ journal_ctx=None,
1048
1070
  ) -> int:
1049
1071
  """Shared budget-milestone firing core for both vendors (#143).
1050
1072
 
@@ -1077,6 +1099,11 @@ def _record_budget_milestone_for_vendor(
1077
1099
  """
1078
1100
  import _cctally_cache # for the fail-closed AccountAttributionUnavailable skip (#341)
1079
1101
  now_utc = _as_of_or_command(as_of)
1102
+ if vendor == "codex" and as_of is None:
1103
+ # The source scan and durable crossing must share an exact cutoff.
1104
+ # now_utc_iso() reads a second clock and truncates subsecond precision;
1105
+ # preserving this instant also retains a crossing in its final second.
1106
+ as_of = now_utc.isoformat().replace("+00:00", "Z")
1080
1107
  pending_alerts: list[dict[str, Any]] = []
1081
1108
  own_conn = conn is None
1082
1109
  if own_conn:
@@ -1102,9 +1129,15 @@ def _record_budget_milestone_for_vendor(
1102
1129
  if not pending:
1103
1130
  return 0 # nothing left this window → skip the cost SUM
1104
1131
 
1132
+ provenance = {}
1133
+ spend_options = (
1134
+ {"pricing_provenance": provenance}
1135
+ if vendor == "codex" and journal_ctx is not None else {}
1136
+ )
1105
1137
  spent = _budget_spend_for_vendor(
1106
1138
  conn, vendor=vendor, start_at=start_at, now_utc=now_utc,
1107
1139
  account_key=account_key,
1140
+ **spend_options,
1108
1141
  )
1109
1142
  # Shared INSERT-and-arm core (set-then-dispatch, fire-once via rowcount);
1110
1143
  # commit=False inside, so this conn owns the single durable commit below.
@@ -1120,6 +1153,25 @@ def _record_budget_milestone_for_vendor(
1120
1153
  as_of=as_of,
1121
1154
  account_key=account_key,
1122
1155
  ):
1156
+ row = None
1157
+ if (journal_ctx is not None and provenance) or vendor == "claude":
1158
+ row = conn.execute(
1159
+ "SELECT id FROM budget_milestones WHERE vendor=? "
1160
+ "AND account_key=? AND period_start_at=? AND period=? "
1161
+ "AND threshold=?",
1162
+ (vendor, account_key, period_key, period, t),
1163
+ ).fetchone()
1164
+ if journal_ctx is not None and provenance and row is not None:
1165
+ journal_ctx.pricing_provenance[("budget", int(row[0]))] = dict(
1166
+ provenance
1167
+ )
1168
+ if (
1169
+ vendor == "claude" and row is not None
1170
+ and _claude_latch_recreates_retired_crossing(
1171
+ conn, "budget", int(row[0]))
1172
+ ):
1173
+ # #869 V2: stamped by `_budget_crossings`, never re-sent.
1174
+ continue
1123
1175
  pending_alerts.append(build_payload(
1124
1176
  threshold=t,
1125
1177
  crossed_at_utc=crossed_at,
@@ -1423,6 +1475,7 @@ def maybe_record_project_budget_milestone(
1423
1475
  # racing record-usage instance OR an already-recorded pair gets
1424
1476
  # rowcount==0 and skips.
1425
1477
  if inserted == 1:
1478
+ rowid = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
1426
1479
  crossed_at = as_of or now_utc_iso()
1427
1480
  # set-then-dispatch: alerted_at lands on the row BEFORE the
1428
1481
  # Popen, sharing this transaction with the INSERT (commit=False).
@@ -1433,6 +1486,10 @@ def maybe_record_project_budget_milestone(
1433
1486
  " AND threshold = ? AND alerted_at IS NULL",
1434
1487
  (crossed_at, week_key, project_key, t),
1435
1488
  )
1489
+ if _claude_latch_recreates_retired_crossing(
1490
+ conn, "project_budget", rowid,
1491
+ ):
1492
+ continue # #869 V2: stamped above, never re-sent
1436
1493
  # Collision-aware label (shared primitive, #130); resolved once
1437
1494
  # on the first dispatch and reused for the rest of this tick.
1438
1495
  # Kept defensive fallback (F4).
@@ -1485,6 +1542,7 @@ def maybe_record_project_budget_milestone(
1485
1542
  def maybe_record_codex_budget_milestone(
1486
1543
  saved: dict[str, Any], *, raise_errors: bool = False, conn=None, as_of=None,
1487
1544
  alert_sink: "list | None" = None,
1545
+ journal_ctx=None,
1488
1546
  ) -> int:
1489
1547
  """Fire Codex budget alerts on ACTUAL-Codex-spend threshold crossings (axis
1490
1548
  ``codex_budget``, calendar-period-codex-budgets spec §6 — the gap the Codex
@@ -1560,6 +1618,7 @@ def maybe_record_codex_budget_milestone(
1560
1618
  build_payload=_codex_budget_payload, account_key="*",
1561
1619
  raise_errors=raise_errors, conn=conn, as_of=as_of,
1562
1620
  alert_sink=alert_sink,
1621
+ journal_ctx=journal_ctx,
1563
1622
  ) or 0
1564
1623
  for acct_key, acct_usd in accounts.items():
1565
1624
  fired += _record_budget_milestone_for_vendor(
@@ -1568,6 +1627,7 @@ def maybe_record_codex_budget_milestone(
1568
1627
  build_payload=_codex_budget_payload, account_key=acct_key,
1569
1628
  raise_errors=raise_errors, conn=conn, as_of=as_of,
1570
1629
  alert_sink=alert_sink,
1630
+ journal_ctx=journal_ctx,
1571
1631
  ) or 0
1572
1632
  return fired
1573
1633
 
@@ -1998,6 +2058,7 @@ def maybe_record_projected_alert(
1998
2058
  # a racing record-usage instance gets rowcount==0 and skips. The
1999
2059
  # alerted_at UPDATE keys on the CONCRETE `period` (#137).
2000
2060
  if inserted == 1:
2061
+ rowid = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
2001
2062
  conn.execute(
2002
2063
  "UPDATE projected_milestones SET alerted_at = ? "
2003
2064
  "WHERE week_start_at = ? AND period = ? AND metric = ? "
@@ -2005,6 +2066,12 @@ def maybe_record_projected_alert(
2005
2066
  (as_of or now_utc_iso(), p["week_start_at"], p["period"],
2006
2067
  p["metric"], p["threshold"]),
2007
2068
  )
2069
+ if (
2070
+ not str(p["metric"]).startswith("codex_")
2071
+ and _claude_latch_recreates_retired_crossing(
2072
+ conn, "projected", rowid)
2073
+ ):
2074
+ continue # #869 V2: stamped above, never re-sent
2008
2075
  fired.append(p)
2009
2076
  # Single commit: every INSERT + its alerted_at marker durable together.
2010
2077
  # On the passed-conn (ingest) path the caller owns commit + dispatch.
@@ -2232,6 +2299,8 @@ def _compute_block_totals(
2232
2299
  owner_key: Any,
2233
2300
  windows: "Iterable[Any]",
2234
2301
  skip_sync: bool = False,
2302
+ pricing_evidence: "dict | None" = None,
2303
+ owned_entries: "list | None" = None,
2235
2304
  ) -> dict[str, Any]:
2236
2305
  """Sum tokens + cost over the entries the target block OWNS within
2237
2306
  [block_start_at, range_end], plus per-model and per-project breakdowns
@@ -2275,6 +2344,9 @@ def _compute_block_totals(
2275
2344
  f"the competing-window context"
2276
2345
  )
2277
2346
 
2347
+ missing_without_raw = set()
2348
+ observed_raw = set()
2349
+
2278
2350
  def _priced():
2279
2351
  loaded = get_claude_session_entries(
2280
2352
  block_start_at, range_end, skip_sync=skip_sync,
@@ -2283,6 +2355,20 @@ def _compute_block_totals(
2283
2355
  loaded, ownership_windows,
2284
2356
  )
2285
2357
  for entry in owned[owner_key]:
2358
+ if owned_entries is not None:
2359
+ owned_entries.append(entry)
2360
+ if pricing_evidence is not None and (
2361
+ entry.input_tokens or entry.output_tokens
2362
+ or entry.cache_creation_tokens or entry.cache_read_tokens
2363
+ ):
2364
+ if entry.cost_usd is not None:
2365
+ observed_raw.add(entry.model)
2366
+ else:
2367
+ import _lib_pricing
2368
+ if _lib_pricing._resolve_model_pricing(
2369
+ entry.model, warn=False,
2370
+ ) is None:
2371
+ missing_without_raw.add(entry.model)
2286
2372
  usage = claude_usage_dict( # #195 chokepoint
2287
2373
  input_tokens=entry.input_tokens,
2288
2374
  output_tokens=entry.output_tokens,
@@ -2304,7 +2390,15 @@ def _compute_block_totals(
2304
2390
  cost_usd=cost,
2305
2391
  )
2306
2392
 
2307
- return fold_block_totals(_priced()).as_legacy_dict()
2393
+ result = fold_block_totals(_priced()).as_legacy_dict()
2394
+ if pricing_evidence is not None:
2395
+ pricing_evidence["unpricedModels"] = sorted(
2396
+ missing_without_raw - observed_raw
2397
+ )
2398
+ ambiguous = sorted(missing_without_raw & observed_raw)
2399
+ if ambiguous:
2400
+ pricing_evidence["uncertainModels"] = ambiguous
2401
+ return result
2308
2402
 
2309
2403
 
2310
2404
  def maybe_update_five_hour_block(
@@ -2455,10 +2549,41 @@ def maybe_update_five_hour_block(
2455
2549
  load_start=block_start_dt,
2456
2550
  load_end=captured_at_dt,
2457
2551
  )
2552
+ pricing_evidence = {}
2553
+ # `db rederive`'s scratch replay alone supplies this sink (#869), keyed
2554
+ # by the blocks whose historical close it must prove. It keeps the
2555
+ # exact entries behind the totals this tick applies, so the planner
2556
+ # proves a missing card from the very population that produced the
2557
+ # replayed children. A frozen close's upsert applies nothing, so its
2558
+ # later recomputations are not kept.
2559
+ source_sink = getattr(journal_ctx, "block_source_evidence", None)
2560
+ source_key = (account_key, int(five_hour_window_key))
2561
+ owned_entries = (
2562
+ [] if source_sink is not None and source_key in source_sink
2563
+ and not current_is_frozen else None
2564
+ )
2458
2565
  totals = _compute_block_totals(
2459
2566
  block_start_dt, captured_at_dt,
2460
2567
  owner_key=int(five_hour_window_key),
2461
2568
  windows=ownership_windows,
2569
+ pricing_evidence=pricing_evidence,
2570
+ owned_entries=owned_entries,
2571
+ )
2572
+ if owned_entries is not None:
2573
+ source_sink[source_key] = tuple(owned_entries)
2574
+ # This describes the pricing used for THESE computed totals. A later
2575
+ # successor closes this block without recomputing it, so looking up
2576
+ # cards while harvesting the close would describe the wrong binary.
2577
+ import _lib_pricing
2578
+ pricing_marker = {
2579
+ "version": 1,
2580
+ "pricingDate": _lib_pricing.current_pricing_snapshot().snapshot_date,
2581
+ "unpricedModels": pricing_evidence["unpricedModels"],
2582
+ }
2583
+ if pricing_evidence.get("uncertainModels"):
2584
+ pricing_marker["uncertainModels"] = pricing_evidence["uncertainModels"]
2585
+ pricing_provenance_json = json.dumps(
2586
+ pricing_marker, sort_keys=True, separators=(",", ":"),
2462
2587
  )
2463
2588
 
2464
2589
  # Hoist alerts config above BEGIN (M1 + M2): single read serves
@@ -2607,12 +2732,13 @@ def maybe_update_five_hour_block(
2607
2732
  total_cache_create_tokens,
2608
2733
  total_cache_read_tokens,
2609
2734
  total_cost_usd,
2735
+ pricing_provenance_json,
2610
2736
  is_closed,
2611
2737
  created_at_utc,
2612
2738
  last_updated_at_utc,
2613
2739
  account_key
2614
2740
  )
2615
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, 0, ?, ?, ?)
2741
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?)
2616
2742
  ON CONFLICT(account_key, five_hour_window_key) DO UPDATE SET
2617
2743
  last_observed_at_utc = excluded.last_observed_at_utc,
2618
2744
  final_five_hour_percent = excluded.final_five_hour_percent,
@@ -2622,6 +2748,7 @@ def maybe_update_five_hour_block(
2622
2748
  total_cache_create_tokens = excluded.total_cache_create_tokens,
2623
2749
  total_cache_read_tokens = excluded.total_cache_read_tokens,
2624
2750
  total_cost_usd = excluded.total_cost_usd,
2751
+ pricing_provenance_json = excluded.pricing_provenance_json,
2625
2752
  last_updated_at_utc = excluded.last_updated_at_utc
2626
2753
  WHERE five_hour_blocks.is_closed = 0
2627
2754
  OR five_hour_blocks.journal_id IS NULL
@@ -2641,6 +2768,7 @@ def maybe_update_five_hour_block(
2641
2768
  totals["cache_create_tokens"],
2642
2769
  totals["cache_read_tokens"],
2643
2770
  totals["cost_usd"],
2771
+ pricing_provenance_json,
2644
2772
  now_iso,
2645
2773
  now_iso,
2646
2774
  account_key,
@@ -6369,6 +6497,7 @@ def _cmd_hook_tick_codex(
6369
6497
  def _codex_budget_leg(ictx):
6370
6498
  _budget_holder["n"] = int(c.maybe_record_codex_budget_milestone(
6371
6499
  {}, conn=ictx.conn, alert_sink=ictx.pending_alerts,
6500
+ journal_ctx=ictx,
6372
6501
  raise_errors=True) or 0)
6373
6502
 
6374
6503
  budget_ingest = _jr_codex.run_stats_ingest(
@@ -7134,7 +7263,8 @@ def _derive_5h_window_key(conn, five_hour_resets_at_epoch):
7134
7263
  )
7135
7264
 
7136
7265
 
7137
- def _run_dollar_axes(saved, *, conn, as_of, alert_sink, enabled=True):
7266
+ def _run_dollar_axes(saved, *, conn, as_of, alert_sink, enabled=True,
7267
+ journal_ctx=None):
7138
7268
  """The four dollar-decoupled alert axes in cmd_record_usage's legacy order
7139
7269
  (budget → project-budget → codex-budget → projected). Runs on BOTH the accept
7140
7270
  path AND every dedup-skip tick (spec §4.5: USD spend can cross a $ threshold
@@ -7149,7 +7279,8 @@ def _run_dollar_axes(saved, *, conn, as_of, alert_sink, enabled=True):
7149
7279
  c.maybe_record_project_budget_milestone(
7150
7280
  saved, conn=conn, as_of=as_of, alert_sink=alert_sink)
7151
7281
  c.maybe_record_codex_budget_milestone(
7152
- saved, conn=conn, as_of=as_of, alert_sink=alert_sink)
7282
+ saved, conn=conn, as_of=as_of, alert_sink=alert_sink,
7283
+ journal_ctx=journal_ctx)
7153
7284
  c.maybe_record_projected_alert(
7154
7285
  saved, conn=conn, as_of=as_of, alert_sink=alert_sink)
7155
7286
 
@@ -7561,6 +7692,7 @@ def _pipeline_claude_usage(ctx, rec):
7561
7692
  conn=conn,
7562
7693
  as_of=as_of,
7563
7694
  alert_sink=ctx.pending_alerts,
7695
+ journal_ctx=ctx,
7564
7696
  # Budget/projected rows depend on historical config that was not retained
7565
7697
  # in the journal. Task B classifies them as re-materialized projections:
7566
7698
  # the planner retires stale latches and never fabricates old config.