cctally 1.104.0 → 1.105.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +38 -0
  2. package/README.md +6 -6
  3. package/bin/_cctally_alerts.py +46 -13
  4. package/bin/_cctally_cache.py +193 -71
  5. package/bin/_cctally_dashboard.py +683 -70
  6. package/bin/_cctally_dashboard_conversation.py +400 -6
  7. package/bin/_cctally_dashboard_envelope.py +17 -0
  8. package/bin/_cctally_dashboard_perf.py +93 -0
  9. package/bin/_cctally_dashboard_sources.py +988 -94
  10. package/bin/_cctally_db.py +30 -3
  11. package/bin/_cctally_diagnosis_sources.py +699 -186
  12. package/bin/_cctally_doctor.py +22 -8
  13. package/bin/_cctally_forecast.py +17 -9
  14. package/bin/_cctally_journal.py +298 -40
  15. package/bin/_cctally_quota.py +15 -0
  16. package/bin/_cctally_quota_model.py +584 -58
  17. package/bin/_cctally_record.py +133 -67
  18. package/bin/_cctally_tui.py +317 -70
  19. package/bin/_lib_codex_conversation_query.py +258 -12
  20. package/bin/_lib_codex_hooks.py +26 -0
  21. package/bin/_lib_conversation_query.py +145 -15
  22. package/bin/_lib_dashboard_json.py +105 -0
  23. package/bin/_lib_diagnosis.py +21 -2
  24. package/bin/_lib_doctor.py +64 -8
  25. package/bin/_lib_ingest_frontier.py +889 -0
  26. package/bin/_lib_meter_rate_change.py +95 -29
  27. package/bin/_lib_perf.py +22 -0
  28. package/bin/_lib_pricing.py +30 -3
  29. package/bin/_lib_quota_copy.py +30 -4
  30. package/bin/_lib_quota_model.py +215 -28
  31. package/bin/_lib_rate_change_delivery.py +119 -0
  32. package/bin/_lib_record.py +50 -0
  33. package/bin/_lib_retained_size.py +187 -0
  34. package/bin/_lib_snapshot_cache.py +504 -17
  35. package/bin/_lib_statusline.py +3 -29
  36. package/bin/_lib_tick_stats.py +28 -4
  37. package/dashboard/static/assets/ConversationsView-DA4j7Yso.js +72 -0
  38. package/dashboard/static/assets/DoctorModal-D4PbVnbE.js +1 -0
  39. package/dashboard/static/assets/ModalRoot-D4oSPxPp.js +1 -0
  40. package/dashboard/static/assets/ProjectsDrillPanel-DkPlB9JH.js +1 -0
  41. package/dashboard/static/assets/SourceDetailModal-BE7FONsS.js +1 -0
  42. package/dashboard/static/assets/UpdateModal-BJuI8glf.js +7 -0
  43. package/dashboard/static/assets/index-w_bINkJ8.js +13 -0
  44. package/dashboard/static/assets/outlineNavigation-zDVm6Hdd.js +9 -0
  45. package/dashboard/static/assets/useKeymap-CJ-Pi17D.js +1 -0
  46. package/dashboard/static/dashboard.html +2 -1
  47. package/package.json +4 -1
  48. package/dashboard/static/assets/index-D19TO7Mg.js +0 -97
@@ -1022,8 +1022,18 @@ def _load_codex_quota_observations_for_doctor(*, force_cold: bool = False):
1022
1022
 
1023
1023
 
1024
1024
  _QUOTA_NO_CHANGE: dict = {
1025
- "active": False, "effective_from": None,
1025
+ "active": False, "assessed": True, "effective_from": None,
1026
1026
  "previous_units_per_point": None, "new_units_per_point": None,
1027
+ "calibration_status": None,
1028
+ }
1029
+
1030
+ #: No regime pair was examined, so no transition claim is available either
1031
+ #: way (#688). Distinct from `_QUOTA_NO_CHANGE`, which is a real negative
1032
+ #: finding over regimes that exist.
1033
+ _QUOTA_NOT_ASSESSED: dict = {
1034
+ "active": False, "assessed": False, "effective_from": None,
1035
+ "previous_units_per_point": None, "new_units_per_point": None,
1036
+ "calibration_status": None,
1027
1037
  }
1028
1038
 
1029
1039
 
@@ -1036,14 +1046,18 @@ def _gather_quota_rate_change(c, rejection=None) -> "dict | None":
1036
1046
  gather goes to the stored regimes rather than through the validated
1037
1047
  reader's default, which refuses that mark.
1038
1048
 
1039
- An ABSENT calibration is "no change detected", not "not assessed" — a
1040
- store with no fitted regime has no transition to report, and that is the
1041
- state of every install that has never run `cctally quota`. `None` is
1042
- reserved for a calibration that exists and could not be read, where the
1043
- sibling `quota.calibration` check is already WARNing about it.
1049
+ An ABSENT calibration is "not assessed", NEVER "no change detected"
1050
+ (#688). It was the latter until a real metering change was reported to a
1051
+ user as a green all-clear: the fit was withheld, so no regime persisted,
1052
+ so the calibration was absent, and this gather turned that absence into a
1053
+ negative finding. A store with no fitted regime has no transition to
1054
+ report AND no evidence that none occurred, and `docs/commands/quota.md`
1055
+ forbids conflating the two. `None` is reserved for a calibration that
1056
+ exists and could not be read, where the sibling `quota.calibration` check
1057
+ is already WARNing about it; both render as not assessed.
1044
1058
  """
1045
1059
  if rejection == "calibration-absent":
1046
- return dict(_QUOTA_NO_CHANGE)
1060
+ return dict(_QUOTA_NOT_ASSESSED)
1047
1061
  mrc = c._load_sibling("_lib_meter_rate_change")
1048
1062
  glue = c._load_sibling("_cctally_quota_model")
1049
1063
  loaded = glue.read_stored_state_readonly()
@@ -1053,7 +1067,7 @@ def _gather_quota_rate_change(c, rejection=None) -> "dict | None":
1053
1067
  active = mrc.active_rate_change(regimes)
1054
1068
  if active is None:
1055
1069
  return dict(_QUOTA_NO_CHANGE)
1056
- return active
1070
+ return dict(active, assessed=True)
1057
1071
 
1058
1072
 
1059
1073
  def doctor_gather_state(
@@ -258,10 +258,14 @@ def _apply_midweek_reset_override(
258
258
  week_start_at: dt.datetime,
259
259
  week_end_at: dt.datetime,
260
260
  samples: list,
261
+ *,
262
+ now_utc: dt.datetime,
261
263
  ) -> tuple[dt.datetime, list]:
262
264
  """If the current week's end_at matches a recorded reset event's
263
- ``new_week_end_at``, shift ``week_start_at`` to the effective reset
264
- moment and drop pre-reset samples.
265
+ ``new_week_end_at`` and was detected by ``now_utc``, shift
266
+ ``week_start_at`` to the effective reset moment and drop pre-reset
267
+ samples. The detection bound keeps historical replay causal while leaving
268
+ the live path unchanged (live ``now_utc`` postdates retained events).
265
269
 
266
270
  Keeps callers (``_load_forecast_inputs``, ``_tui_build_current_week``)
267
271
  from reporting spent_usd summed across the pre-reset window.
@@ -276,8 +280,10 @@ def _apply_midweek_reset_override(
276
280
  ).isoformat(timespec="seconds")
277
281
  event_row = conn.execute(
278
282
  "SELECT effective_reset_at_utc FROM week_reset_events "
279
- "WHERE new_week_end_at = ?",
280
- (end_iso,),
283
+ "WHERE new_week_end_at = ? "
284
+ " AND datetime(detected_at_utc) <= datetime(?) "
285
+ "ORDER BY datetime(detected_at_utc) DESC LIMIT 1",
286
+ (end_iso, now_utc.isoformat()),
281
287
  ).fetchone()
282
288
  if event_row and event_row["effective_reset_at_utc"]:
283
289
  reset_dt = parse_iso_datetime(
@@ -314,7 +320,7 @@ def _resolve_current_budget_window(conn, now_utc, *, account_key=None):
314
320
  return None
315
321
  week_start_at, week_end_at, samples = fetched
316
322
  week_start_at, _samples = _apply_midweek_reset_override(
317
- conn, week_start_at, week_end_at, samples
323
+ conn, week_start_at, week_end_at, samples, now_utc=now_utc
318
324
  )
319
325
  return (week_start_at, week_end_at)
320
326
 
@@ -1158,8 +1164,10 @@ def _calibrated_week_detail(
1158
1164
  # A `CalibratedWeek`, never the two-value tuple this function returned
1159
1165
  # before it grew a dataclass. `_calibrated_projection` reads
1160
1166
  # `detail.projection_pct` off the result, so a tuple here raises
1161
- # `AttributeError` instead of withholding — the same defect class as
1162
- # `persist_and_detect`'s `None, None`.
1167
+ # `AttributeError` instead of withholding — the same defect class as a
1168
+ # `persist_and_detect` early return whose arity does not match the rest
1169
+ # of the function. That function now returns THREE values, which is why
1170
+ # its unreadable-state path returns `None, (), ()`.
1163
1171
  return CalibratedWeek(code="unavailable")
1164
1172
  read = qcg.read_calibration_file(account_key=account_key)
1165
1173
  if read.regime is None:
@@ -1295,7 +1303,7 @@ def _load_forecast_inputs(
1295
1303
  # reset moment and drop pre-reset samples so elapsed/remaining math
1296
1304
  # and spent_usd reflect the post-reset window only.
1297
1305
  week_start_at, samples = _apply_midweek_reset_override(
1298
- conn, week_start_at, week_end_at, samples
1306
+ conn, week_start_at, week_end_at, samples, now_utc=now_utc
1299
1307
  )
1300
1308
 
1301
1309
  if not samples:
@@ -2542,7 +2550,7 @@ def cmd_forecast(args: argparse.Namespace) -> int:
2542
2550
  if fetched is not None:
2543
2551
  _ws_at, _we_at, raw_samples = fetched
2544
2552
  _ws_at_shifted, samples = _apply_midweek_reset_override(
2545
- conn, _ws_at, _we_at, raw_samples
2553
+ conn, _ws_at, _we_at, raw_samples, now_utc=now_utc
2546
2554
  )
2547
2555
  tz_render = getattr(args, "_resolved_tz", None)
2548
2556
  for cap_at, pct, _five_hr in samples:
@@ -1295,6 +1295,32 @@ class IngestResult:
1295
1295
  # on a clean cycle. Authoritative callers never see this — their cycle
1296
1296
  # exception propagates.
1297
1297
  error: object = None
1298
+ # #695: this cycle's meter_rate_change outcome, or None when no descriptor
1299
+ # was passed. `deferred_alerts` alone is insufficient — with notifications
1300
+ # off there is no payload, yet the identity was still decided and the
1301
+ # caller must still mark the ledger.
1302
+ meter_rate_change_result: object = None
1303
+ deferred_alerts: list = field(default_factory=list)
1304
+
1305
+
1306
+ @dataclass(frozen=True)
1307
+ class RateChangeRecordResult:
1308
+ """What one `record_meter_rate_change` call did to one identity (#695).
1309
+
1310
+ THREE booleans, because the caller must tell three outcomes apart and a
1311
+ single one cannot. `row_created` drives nothing on its own.
1312
+ `notification_queued` says a payload is waiting in `deferred_alerts`.
1313
+ `notification_decided` is the one the ledger keys on: it is true when this
1314
+ call created the row, and when it took the owed branch and observed the
1315
+ durable row — INCLUDING when notifications are off and nothing was
1316
+ queued, because deciding not to notify is a decision. A duplicate or a
1317
+ conflict decides nothing, or it would win the compare-and-set away from
1318
+ the process that is actually going to dispatch.
1319
+ """
1320
+
1321
+ row_created: bool = False
1322
+ notification_queued: bool = False
1323
+ notification_decided: bool = False
1298
1324
 
1299
1325
 
1300
1326
  @dataclass
@@ -1315,6 +1341,12 @@ class IngestContext:
1315
1341
  batch: list # decoded obs/op records this cycle
1316
1342
  config: object = None
1317
1343
  pending_alerts: list = field(default_factory=list)
1344
+ # #695: the meter_rate_change family's post-commit payloads. Step 6 does
1345
+ # NOT drain this — `_run_cycle` returns it, and `cmd_quota` dispatches only
1346
+ # after winning the delivery ledger's compare-and-set. Invariant (iv) is
1347
+ # unaffected: these payloads still originate in the live sink and step-4a
1348
+ # replay still has no ctx to add to either list.
1349
+ deferred_alerts: list = field(default_factory=list)
1318
1350
  events_emitted: int = 0
1319
1351
  # Design B (DB journal redesign §5.3 event+effects): the per-cycle
1320
1352
  # suppression map a reset/credit pipeline hook populates BEFORE it runs its
@@ -2811,7 +2843,18 @@ def rehydrate_codex_journal_families(
2811
2843
  "DELETE FROM cache_meta WHERE key = ?",
2812
2844
  (_cctally_cache.CODEX_FILE_ACCOUNT_CURSOR_KEY,))
2813
2845
  if authoritative and want_window_attributions:
2846
+ before = cache_conn.total_changes
2814
2847
  cache_conn.execute("DELETE FROM codex_window_attributions")
2848
+ window_rows_cleared = cache_conn.total_changes > before
2849
+ if window_rows_cleared:
2850
+ # The cross-build quota memo keys the semantic attribution
2851
+ # revision, not the replay cursor. With no replacement
2852
+ # assertion, `_apply_window_attribution_records` has no chance
2853
+ # to advance that revision, yet the authoritative empty result
2854
+ # still changed the projection input. Keep the invalidation in
2855
+ # the same transaction as the deletion.
2856
+ _cctally_cache.bump_codex_window_attribution_revision(
2857
+ cache_conn)
2815
2858
  # #500 review finding F10: the cursor describes a table that no
2816
2859
  # longer has any rows, so leaving it would let the NEXT delta pass
2817
2860
  # skip journal bytes on the strength of a claim this branch just
@@ -2820,11 +2863,14 @@ def rehydrate_codex_journal_families(
2820
2863
  "DELETE FROM cache_meta WHERE key = ?",
2821
2864
  (_cctally_cache.CODEX_WINDOW_ATTRIBUTION_CURSOR_KEY,))
2822
2865
  return CodexJournalRehydration(0, None, 0, 0, 0)
2866
+ window_rows_cleared = False
2823
2867
  if authoritative:
2824
2868
  if want_file_accounts:
2825
2869
  cache_conn.execute("DELETE FROM codex_file_accounts")
2826
2870
  if want_window_attributions:
2871
+ before = cache_conn.total_changes
2827
2872
  cache_conn.execute("DELETE FROM codex_window_attributions")
2873
+ window_rows_cleared = cache_conn.total_changes > before
2828
2874
  # A clear-then-replay is only correct from the beginning of the journal.
2829
2875
  file_account_since = window_attribution_since = None
2830
2876
 
@@ -2858,6 +2904,7 @@ def rehydrate_codex_journal_families(
2858
2904
  segment_positions = {name: idx for idx, name in enumerate(segments)}
2859
2905
 
2860
2906
  applied = conflicts = attributed = skipped = 0
2907
+ window_records_changed = False
2861
2908
  # Streamed, never materialized, for the reason above: this runs on the FIRST
2862
2909
  # ordinary sync of every cache.db while both cache flocks are held. The
2863
2910
  # cheap byte prefilters skip the JSON decode for every uninteresting line;
@@ -2892,6 +2939,14 @@ def rehydrate_codex_journal_families(
2892
2939
  _apply_window_attribution_records(cache_conn, (rec,)))
2893
2940
  attributed += _asserted
2894
2941
  skipped += _skipped
2942
+ window_records_changed = window_records_changed or bool(
2943
+ _asserted or _retracted)
2944
+ if window_rows_cleared and not window_records_changed:
2945
+ # A replacement assertion/retraction advances the revision inside its
2946
+ # applier. The deletion-only authoritative outcome has no applier-side
2947
+ # mutation, so advance it here instead. This also covers a journal
2948
+ # containing only malformed/skipped attribution records.
2949
+ _cctally_cache.bump_codex_window_attribution_revision(cache_conn)
2895
2950
  if want_window_attributions and (
2896
2951
  authoritative or window_attribution_since != (str(hw[0]), int(hw[1]))):
2897
2952
  # Only when it actually moves. This runs on every ordinary Codex sync,
@@ -3307,18 +3362,29 @@ def _now_iso() -> str:
3307
3362
  )
3308
3363
 
3309
3364
 
3310
- def _usage_snapshot_fold_decision(conn, payload) -> tuple[bool, object]:
3365
+ def _usage_snapshot_fold_decision(conn, payload) -> tuple[bool, object, str]:
3311
3366
  """The apply-time dedup for a Claude rate-limit obs — the exact predicate
3312
3367
  ported from `cmd_record_usage`'s insert guard (bin/_cctally_record.py), now
3313
3368
  at fold time (spec §4.5 / §5.3).
3314
3369
 
3315
- Returns `(skip, adjusted_five_hour_percent)`:
3370
+ Returns `(skip, adjusted_five_hour_percent, reason)`:
3316
3371
  - reset-aware 7d HWM clamp (`_reset_aware_floor` + reset-aware MAX +
3317
- `hwm_clamp_applies`) → skip when a lower 7d % would be clamped;
3372
+ `hwm_clamp_applies`) → skip when a lower 7d % would be clamped, with
3373
+ reason `SNAPSHOT_SKIP_CLAMP`;
3318
3374
  - the 5h clamp adjusts `five_hour_percent` UP to the in-window MAX but
3319
3375
  never gates (mirrors the nested-else in the live code);
3320
- - dedup vs the latest snapshot in the week: both percents unchanged → skip.
3321
- """
3376
+ - dedup vs the latest snapshot in the week: both percents unchanged → skip,
3377
+ with reason `SNAPSHOT_SKIP_DEDUP`;
3378
+ - otherwise accept, with reason `SNAPSHOT_ACCEPT`.
3379
+
3380
+ `reason` exists because the two skips mean OPPOSITE things about the
3381
+ incoming observation and the stored row (see `_lib_record` Fragment 8): a
3382
+ dedup skip AGREES with the stored row, so re-running the derivation
3383
+ chokepoints against it heals a killed tick; a clamp skip CONTRADICTS it, so
3384
+ deriving a weekly milestone from the higher stored value fabricates a
3385
+ crossing. `_pipeline_claude_usage` gates the weekly milestone on it. The
3386
+ reason is derived from the same `conn` state and payload as `skip` itself,
3387
+ so it is exactly as replay-deterministic as the boolean it accompanies."""
3322
3388
  week_start_date = payload["week_start_date"]
3323
3389
  week_start_at = payload.get("week_start_at")
3324
3390
  week_end_at = payload.get("week_end_at")
@@ -3344,7 +3410,7 @@ def _usage_snapshot_fold_decision(conn, payload) -> tuple[bool, object]:
3344
3410
  ).fetchone()
3345
3411
  max_v = max_row[0] if max_row else None
3346
3412
  if _lib_record.hwm_clamp_applies(weekly_percent, max_v):
3347
- return True, five_hour_percent
3413
+ return True, five_hour_percent, _lib_record.SNAPSHOT_SKIP_CLAMP
3348
3414
 
3349
3415
  adjusted_5h = five_hour_percent
3350
3416
  if five_hour_percent is not None and five_hour_window_key is not None:
@@ -3374,8 +3440,8 @@ def _usage_snapshot_fold_decision(conn, payload) -> tuple[bool, object]:
3374
3440
  if adjusted_5h is None or (
3375
3441
  last_5h is not None and float(last_5h) == float(adjusted_5h)
3376
3442
  ):
3377
- return True, adjusted_5h
3378
- return False, adjusted_5h
3443
+ return True, adjusted_5h, _lib_record.SNAPSHOT_SKIP_DEDUP
3444
+ return False, adjusted_5h, _lib_record.SNAPSHOT_ACCEPT
3379
3445
 
3380
3446
 
3381
3447
  # NOTE (rev 3): the direct obs -> weekly_usage_snapshots fold is GONE. That
@@ -4297,8 +4363,11 @@ def emit_model_a(ctx, *, kind, evt_id, table, columns, refs=None, at=None):
4297
4363
  decision = _classify_live_effective_event(ctx.conn, evt)
4298
4364
  if decision == CLASSIFY_CONFLICT:
4299
4365
  # No append, no metadata mutation — converge the row instead.
4300
- _record_dropped_conflict(ctx, evt)
4301
- _converge_row_from_effective(ctx.conn, evt_id, table=table)
4366
+ selected = _record_dropped_conflict(ctx, evt)
4367
+ _converge_and_report(
4368
+ selected,
4369
+ lambda: _converge_row_from_effective(
4370
+ ctx.conn, evt_id, table=table))
4302
4371
  else:
4303
4372
  # `new` AND `duplicate` both still append: crash-replay convergence
4304
4373
  # and two-bootstrap idempotency are built on that.
@@ -4386,10 +4455,11 @@ def _emit_harvest_row(ctx, spec, row):
4386
4455
 
4387
4456
  decision = _classify_live_effective_event(conn, evt)
4388
4457
  if decision == CLASSIFY_CONFLICT:
4389
- _record_dropped_conflict(ctx, evt)
4390
- _converge_row_from_effective(
4391
- conn, evt["id"], table=spec.table, rowid=row["id"]
4392
- )
4458
+ selected = _record_dropped_conflict(ctx, evt)
4459
+ _converge_and_report(
4460
+ selected,
4461
+ lambda: _converge_row_from_effective(
4462
+ conn, evt["id"], table=spec.table, rowid=row["id"]))
4393
4463
  return evt
4394
4464
 
4395
4465
  append_record(evt)
@@ -4468,36 +4538,169 @@ PIPELINE.append(_pipeline_op_fold)
4468
4538
 
4469
4539
 
4470
4540
  def record_meter_rate_change(ctx, transition, *, notify: bool,
4471
- created_at: str) -> bool:
4541
+ created_at: str,
4542
+ notification_owed: bool = False):
4472
4543
  """Steps 4 and 5 of spec §6.5, run INSIDE the cycle transaction.
4473
4544
 
4474
4545
  Journal-first (invariant i): the evt line is appended and fsync'd through
4475
4546
  the leaf `journal.lock` before the transaction containing its row commits,
4476
- exactly as `emit_model_a` does. The payload is a pure function of the
4477
- transition, so a crash-replayed duplicate line is byte-identical and folds
4478
- to a clean no-op.
4479
-
4480
- Returns whether THIS call created the row. A notification is queued only
4481
- then, and only when `notify` is true — §6.2 splits recording from
4482
- notifying, so the row exists from the first upgrade while the push follows
4483
- the existing default-off toggle. The queue is `ctx.pending_alerts`, which
4484
- step 6 drains AFTER the commit: committed-before-notify, and a crash
4485
- between the two loses at most one dispatch rather than recording an alert
4486
- that never happened.
4547
+ exactly as `emit_model_a` does.
4548
+
4549
+ TWO guards, and they catch inverse conditions (#689). The natural-key
4550
+ recheck catches a committed physical latch whose effective metadata is not
4551
+ yet populated. `_classify_live_effective_event` catches the inverse —
4552
+ prior metadata with the row absent — and additionally tells an exact
4553
+ duplicate apart from divergent same-revision content. Recovery re-offers
4554
+ persisted pairs under a later command clock, and the payload includes that
4555
+ clock, so without the classifier a retry would append a byte-different
4556
+ line under an id the journal already holds. This was the only emit path in
4557
+ this file that appended without classifying.
4558
+
4559
+ NEITHER the duplicate nor the conflict branch may rely on replay to
4560
+ restore a missing row. `_preflight_live_events` omits a same-revision,
4561
+ same-hash event from `to_apply`, and step 4a applies only what
4562
+ `_record_live_effective_event` returns True for, which is NEW alone — so
4563
+ the cursor advances past a duplicate and the row would stay absent.
4564
+ `_converge_row_from_effective` cannot cover it either, because this
4565
+ family's `_EVT_SPECS` entry has `table=None` and it returns
4566
+ CONVERGE_DROPPED. So this function materializes its own row: from the
4567
+ identical event on a duplicate, and from the PRIOR journaled event on a
4568
+ conflict, never from the rejected candidate.
4569
+
4570
+ Returns a `RateChangeRecordResult`, not a boolean (#695). Three outcomes
4571
+ have to be told apart: whether this call created the row, whether a
4572
+ payload is waiting, and whether this call OWNS the dispatch decision for
4573
+ this identity. `notification_decided` is the last of those, and the
4574
+ caller's delivery ledger keys on it.
4575
+
4576
+ A notification is queued when this call created the row and `notify` is
4577
+ true — §6.2 splits recording from notifying, so the row exists from the
4578
+ first upgrade while the push follows the existing default-off toggle.
4579
+
4580
+ #695 CORRECTS this docstring's former claim that "existing effective
4581
+ metadata proves this identity was journaled once and so already had its
4582
+ notification". It does not. Step-4a replay of an orphan line — appended by
4583
+ a cycle that then rolled back — creates the row with no `IngestContext`,
4584
+ so invariant (iv) makes that replay structurally unable to queue anything.
4585
+ The row exists and nobody was told. So an existing row now takes the owed
4586
+ branch when, and only when, `notification_owed` says the live command's
4587
+ delivery ledger holds no decision for this identity; the payload is then
4588
+ rebuilt from the DURABLE ROW, which is what was actually recorded. An
4589
+ ordinary duplicate and every conflict branch stay silent, because
4590
+ re-notifying without that explicit intent would refire history.
4591
+
4592
+ The queue is `ctx.deferred_alerts`, which step 6 does NOT drain: this
4593
+ family's dispatch is returned to `cmd_quota`, which fires it only after
4594
+ winning the ledger's compare-and-set. Invariant (iv) is untouched — the
4595
+ payload still originates in the live sink, post-commit, and replay still
4596
+ has no ctx to add to either list.
4487
4597
  """
4488
4598
  mrc = _load_meter_rate_change()
4599
+ existing = ctx.conn.execute(
4600
+ "SELECT provider, account_key, effective_from,"
4601
+ " previous_units_per_point, new_units_per_point, severity "
4602
+ "FROM meter_rate_change_events "
4603
+ "WHERE provider = ? AND account_key = ? AND effective_from = ? "
4604
+ "LIMIT 1", transition.identity()).fetchone()
4605
+ if existing is not None:
4606
+ # #695. A row can reach this table with NO notification: step-4a
4607
+ # replay of an orphan line creates it, and replay has no ctx to queue
4608
+ # anything. So an existing row no longer proves a notification
4609
+ # happened, and `notification_owed` is the live command's assertion
4610
+ # that its delivery ledger holds no decision for this identity.
4611
+ # Without that flag this stays silent, because an ordinary duplicate
4612
+ # must not re-fire.
4613
+ if not notification_owed:
4614
+ return RateChangeRecordResult()
4615
+ if not notify:
4616
+ return RateChangeRecordResult(notification_decided=True)
4617
+ # Rebuilt from the ROW rather than from the caller's descriptor: the
4618
+ # sweep knows only the identity, so its descriptor carries placeholder
4619
+ # rates and severity. `withholding_status` stays at its `None`
4620
+ # default, because it describes the analysis that originally admitted
4621
+ # the transition and a later run's analysis is not that one.
4622
+ ctx.deferred_alerts.append(mrc.alert_payload(
4623
+ mrc.RateChangeTransition(
4624
+ provider=str(existing[0]),
4625
+ account_key=str(existing[1]),
4626
+ effective_from=str(existing[2]),
4627
+ previous_units_per_point=float(existing[3]),
4628
+ new_units_per_point=float(existing[4]),
4629
+ severity=str(existing[5]),
4630
+ detected_at=str(transition.detected_at))))
4631
+ return RateChangeRecordResult(notification_queued=True,
4632
+ notification_decided=True)
4633
+ if notification_owed:
4634
+ # #695. The owed sweep offers an IDENTITY whose durable row it read a
4635
+ # moment ago, so its descriptor carries placeholder zero rates and an
4636
+ # `info` severity that are never meant to be published. Reaching here
4637
+ # means the row is gone — there is nothing to re-notify, and the
4638
+ # placeholder must not be journaled as a real zero-rate change. This
4639
+ # is what ENFORCES the claim `_owed_rate_change_transitions` makes;
4640
+ # without it the prose asserts an invariant the code does not hold.
4641
+ return RateChangeRecordResult()
4489
4642
  payload = mrc.event_payload(transition, created_at=created_at)
4490
4643
  eid = _lib_journal.evt_id(
4491
4644
  mrc.EVT_ID_PREFIX, transition.provider, transition.account_key,
4492
4645
  transition.effective_from)
4493
4646
  evt = _lib_journal.make_evt(
4494
4647
  kind=mrc.EVT_KIND, id=eid, at=created_at, payload=payload)
4495
- append_record(evt)
4496
- ctx.events_emitted += 1
4497
- created = _insert_meter_rate_change(ctx.conn, evt)
4498
- if created and notify:
4499
- ctx.pending_alerts.append(mrc.alert_payload(transition))
4500
- return created
4648
+ decision = _classify_live_effective_event(ctx.conn, evt)
4649
+ if decision == CLASSIFY_NEW:
4650
+ append_record(evt)
4651
+ ctx.events_emitted += 1
4652
+ _record_new_effective_event(ctx.conn, evt)
4653
+ created = _insert_meter_rate_change(ctx.conn, evt)
4654
+ if created and notify:
4655
+ ctx.deferred_alerts.append(mrc.alert_payload(transition))
4656
+ return RateChangeRecordResult(
4657
+ row_created=created,
4658
+ notification_queued=bool(created and notify),
4659
+ notification_decided=created)
4660
+ if decision == CLASSIFY_CONFLICT:
4661
+ selected = _record_dropped_conflict(ctx, evt)
4662
+ _converge_and_report(
4663
+ selected, lambda: _converge_meter_rate_change_row(ctx.conn, eid))
4664
+ return RateChangeRecordResult()
4665
+ _insert_meter_rate_change(ctx.conn, evt)
4666
+ return RateChangeRecordResult()
4667
+
4668
+
4669
+ def _converge_meter_rate_change_row(conn, event_id) -> str:
4670
+ """Materialize the withheld row from the PRIOR journaled event (#689).
4671
+
4672
+ The outcome is the INSERT's own answer, never an assumption that calling
4673
+ it worked. `_insert_meter_rate_change` returns False for a payload
4674
+ `_meter_rate_change_row` cannot normalise — a missing `provider` or
4675
+ `effective_from`, or a non-numeric rate — and reporting
4676
+ `CONVERGE_APPLIED` there would print a line claiming a convergence that
4677
+ inserted nothing, which is exactly what acceptance 6 forbids. A False
4678
+ return here cannot mean "a row already existed": the natural-key recheck
4679
+ at the top of `record_meter_rate_change` ran inside this same transaction
4680
+ and found none.
4681
+ """
4682
+ prior = _prior_meter_rate_change_event(conn, event_id)
4683
+ if prior is None:
4684
+ return CONVERGE_DROPPED
4685
+ return (CONVERGE_APPLIED if _insert_meter_rate_change(conn, prior)
4686
+ else CONVERGE_DROPPED)
4687
+
4688
+
4689
+ def _prior_meter_rate_change_event(conn, event_id):
4690
+ """The prior effective event to materialize a withheld row from, or None.
4691
+
4692
+ A TOMBSTONED selection stores `event_json` as NULL and
4693
+ `_effective_event_for_convergence` fails closed and raises on it. Here a
4694
+ tombstone is a terminal negative latch rather than an error: a completed
4695
+ correction saying this identity should have no row must not be undone by
4696
+ materializing one.
4697
+ """
4698
+ row = conn.execute(
4699
+ "SELECT status FROM journal_effective_events WHERE event_id = ?",
4700
+ (event_id,)).fetchone()
4701
+ if row is None or str(row[0]) != "active":
4702
+ return None
4703
+ return _effective_event_for_convergence(conn, event_id)
4501
4704
 
4502
4705
 
4503
4706
  def _load_meter_rate_change():
@@ -5268,9 +5471,16 @@ def _record_live_effective_event(conn, evt) -> bool:
5268
5471
  return False
5269
5472
 
5270
5473
 
5271
- def _record_dropped_conflict(ctx, evt) -> None:
5272
- """Count + report one withheld emission (spec §8: a one-line stderr note per
5273
- dropped emission, and a count on the cycle summary)."""
5474
+ def _record_dropped_conflict(ctx, evt):
5475
+ """Count one withheld emission and RETURN its selection, so the caller can
5476
+ render the outcome once convergence has resolved (#689).
5477
+
5478
+ Spec §8 asks for a one-line stderr note per dropped emission and a count on
5479
+ the cycle summary. Recording is separated from rendering because the
5480
+ wording depends on whether a row was actually converged, and for a
5481
+ `table=None` family nothing is. Doing both after convergence would lose the
5482
+ `DroppedConflict` too when convergence raises, not merely the stderr line.
5483
+ """
5274
5484
  selected = _lib_journal.resolve_effective_events([evt]).by_id[evt["id"]]
5275
5485
  ctx.conflicts_dropped.append(
5276
5486
  DroppedConflict(
@@ -5279,13 +5489,42 @@ def _record_dropped_conflict(ctx, evt) -> None:
5279
5489
  rejected_hash=selected.content_hash,
5280
5490
  )
5281
5491
  )
5492
+ return selected
5493
+
5494
+
5495
+ def _report_dropped_conflict(selected, outcome) -> None:
5496
+ """Spec §8's one-line stderr note, worded from what convergence did.
5497
+
5498
+ An unrecognised outcome takes the DROPPED wording, because the direction
5499
+ of mistake that matters is claiming a convergence that did not happen.
5500
+ """
5501
+ tail = _CONVERGE_OUTCOME_WORDING.get(
5502
+ outcome, _CONVERGE_OUTCOME_WORDING[CONVERGE_DROPPED])
5282
5503
  print(
5283
5504
  f"[journal] withheld a divergent emission for {selected.event_id} "
5284
- f"rev {selected.rev}; converged the row from the journaled event",
5505
+ f"rev {selected.rev}; {tail}",
5285
5506
  file=sys.stderr,
5286
5507
  )
5287
5508
 
5288
5509
 
5510
+ def _converge_and_report(selected, converge):
5511
+ """Run one convergence and render §8's line WHATEVER it does (#689).
5512
+
5513
+ The line is in a `finally`, so a convergence that fails closed still
5514
+ leaves the diagnostic behind. Before the record/report split it did,
5515
+ because the whole call ran ahead of convergence; the split moved the
5516
+ rendering after the outcome was known and would otherwise have dropped
5517
+ the line on exactly the path that most needs one. The exception is not
5518
+ caught — it propagates through the `finally` unchanged.
5519
+ """
5520
+ outcome = CONVERGE_RAISED
5521
+ try:
5522
+ outcome = converge()
5523
+ finally:
5524
+ _report_dropped_conflict(selected, outcome)
5525
+ return outcome
5526
+
5527
+
5289
5528
  def _effective_event_for_convergence(conn, event_id) -> dict:
5290
5529
  """The ACTIVE, validated journal record the live row must converge to.
5291
5530
 
@@ -5355,6 +5594,18 @@ def _evt_target_columns(conn, evt, spec) -> tuple:
5355
5594
 
5356
5595
  CONVERGE_DROPPED = "dropped"
5357
5596
  CONVERGE_APPLIED = "converged"
5597
+ #: Convergence raised rather than returning an outcome. Only `_converge_and_
5598
+ #: report`'s `finally` produces it, and the exception still propagates.
5599
+ CONVERGE_RAISED = "raised"
5600
+
5601
+ #: §8's stderr tail per outcome. The APPLIED and DROPPED wordings are frozen —
5602
+ #: `tests/test_meter_rate_change_journal.py` asserts both verbatim.
5603
+ _CONVERGE_OUTCOME_WORDING: dict = {
5604
+ CONVERGE_APPLIED: "converged the row from the journaled event",
5605
+ CONVERGE_DROPPED: "the journaled event stands and there was no row to "
5606
+ "converge",
5607
+ CONVERGE_RAISED: "convergence failed and the row was left unchanged",
5608
+ }
5358
5609
 
5359
5610
 
5360
5611
  def _converge_row_from_effective(conn, event_id, *, table=None, rowid=None) -> str:
@@ -6282,6 +6533,9 @@ def _run_cycle(conn: sqlite3.Connection, *, reconcile_config=None,
6282
6533
  # the next cycle's step 4a.
6283
6534
  ctx = IngestContext(conn=conn, batch=batch,
6284
6535
  config=(_load_config_once() if batch else None))
6536
+ # #695: None unless step 4b''' runs, so a cycle with no `meter_rate_change`
6537
+ # descriptor reports exactly what it did before this change.
6538
+ rate_change_result = None
6285
6539
  conn.execute("BEGIN IMMEDIATE")
6286
6540
  try:
6287
6541
  # 4a. Replay journal evt lines (a prior cycle's emission that landed past
@@ -6323,14 +6577,16 @@ def _run_cycle(conn: sqlite3.Connection, *, reconcile_config=None,
6323
6577
  # already decided under the calibration file's leaf lock and released
6324
6578
  # before any stats lock was taken (step 2), is journaled and applied
6325
6579
  # here — inside this transaction, journal-first, with its notification
6326
- # queued to `ctx.pending_alerts` for the post-commit dispatch below.
6580
+ # queued to `ctx.deferred_alerts` for the CALLER to dispatch (#695).
6327
6581
  # It sits beside the Codex leg rather than in the pipeline because its
6328
6582
  # trigger is a persistence transition rather than a journal record.
6329
6583
  if meter_rate_change is not None:
6330
- record_meter_rate_change(
6584
+ rate_change_result = record_meter_rate_change(
6331
6585
  ctx, meter_rate_change["transition"],
6332
6586
  notify=bool(meter_rate_change.get("notify")),
6333
- created_at=str(meter_rate_change["created_at"]))
6587
+ created_at=str(meter_rate_change["created_at"]),
6588
+ notification_owed=bool(
6589
+ meter_rate_change.get("notification_owed")))
6334
6590
  # 4c. Journal + stamp the natural-keyed rows the pipeline inserted.
6335
6591
  # Early-out (Task 6 gate P2): the ONLY source of `journal_id IS NULL`
6336
6592
  # rows is a Task-5 chokepoint called from a step-4b pipeline hook —
@@ -6397,7 +6653,9 @@ def _run_cycle(conn: sqlite3.Connection, *, reconcile_config=None,
6397
6653
  return IngestResult(ran=True, consumed=len(records), malformed=malformed,
6398
6654
  events_emitted=ctx.events_emitted, alerts=alerts,
6399
6655
  conflicts_dropped=(len(ctx.conflicts_dropped)
6400
- + len(preflight_conflicts)))
6656
+ + len(preflight_conflicts)),
6657
+ meter_rate_change_result=rate_change_result,
6658
+ deferred_alerts=list(ctx.deferred_alerts))
6401
6659
 
6402
6660
 
6403
6661
  def _run_stats_ingest_once(
@@ -317,6 +317,11 @@ PROJECTION_DYNAMIC_READ_SITES: "dict[str, int]" = {
317
317
  "_cctally_dashboard.py": 3,
318
318
  "_cctally_dashboard_envelope.py": 5,
319
319
  "_cctally_db.py": 7,
320
+ # NOT a read. The worker-loader failure says `cannot load cctally worker
321
+ # from {path}`, which the case-insensitive `FROM\s+{` pattern cannot tell
322
+ # from a dynamic SQL target. The module's SQL table names remain literal,
323
+ # so no `PROJECTION_DYNAMIC_READ_ACTIONS` classification applies.
324
+ "_cctally_diagnosis_sources.py": 1,
320
325
  "_cctally_doctor.py": 1,
321
326
  "_cctally_five_hour.py": 1,
322
327
  "_cctally_journal.py": 18,
@@ -331,9 +336,19 @@ PROJECTION_DYNAMIC_READ_SITES: "dict[str, int]" = {
331
336
  "_cctally_release.py": 4,
332
337
  "_cctally_setup.py": 3,
333
338
  "_cctally_tui.py": 1,
339
+ # One, in `_codex_rollup_revision`: the table is selected from a hardcoded
340
+ # pair of Codex conversation rollups. Neither is a quota projection table,
341
+ # so no `PROJECTION_DYNAMIC_READ_ACTIONS` classification applies.
342
+ "_lib_codex_conversation_query.py": 1,
334
343
  "_lib_conversation_query.py": 1,
335
344
  "_lib_conversation_retention.py": 2,
336
345
  "_lib_doctor.py": 1,
346
+ # Three: `_target_has_cursor_gap` selects one of the two accounting source
347
+ # tables, while `_conversation_source_paths` and `_conversation_target_risk`
348
+ # select one of the two transcript source tables. Every choice is hardcoded;
349
+ # none is a quota projection table, so no
350
+ # `PROJECTION_DYNAMIC_READ_ACTIONS` classification applies.
351
+ "_lib_ingest_frontier.py": 3,
337
352
  "_lib_snapshot_cache.py": 1,
338
353
  "_lib_subscription_weeks.py": 1,
339
354
  # NOT a read. `bin/_lib_test_estate.py` opens no database and issues no SQL;