cctally 1.104.0 → 1.105.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/README.md +6 -6
- package/bin/_cctally_alerts.py +46 -13
- package/bin/_cctally_cache.py +193 -71
- package/bin/_cctally_dashboard.py +683 -70
- package/bin/_cctally_dashboard_conversation.py +400 -6
- package/bin/_cctally_dashboard_envelope.py +17 -0
- package/bin/_cctally_dashboard_perf.py +93 -0
- package/bin/_cctally_dashboard_sources.py +988 -94
- package/bin/_cctally_db.py +30 -3
- package/bin/_cctally_diagnosis_sources.py +699 -186
- package/bin/_cctally_doctor.py +22 -8
- package/bin/_cctally_forecast.py +17 -9
- package/bin/_cctally_journal.py +298 -40
- package/bin/_cctally_quota.py +15 -0
- package/bin/_cctally_quota_model.py +584 -58
- package/bin/_cctally_record.py +133 -67
- package/bin/_cctally_tui.py +317 -70
- package/bin/_lib_codex_conversation_query.py +258 -12
- package/bin/_lib_codex_hooks.py +26 -0
- package/bin/_lib_conversation_query.py +145 -15
- package/bin/_lib_dashboard_json.py +105 -0
- package/bin/_lib_diagnosis.py +21 -2
- package/bin/_lib_doctor.py +64 -8
- package/bin/_lib_ingest_frontier.py +889 -0
- package/bin/_lib_meter_rate_change.py +95 -29
- package/bin/_lib_perf.py +22 -0
- package/bin/_lib_pricing.py +30 -3
- package/bin/_lib_quota_copy.py +30 -4
- package/bin/_lib_quota_model.py +215 -28
- package/bin/_lib_rate_change_delivery.py +119 -0
- package/bin/_lib_record.py +50 -0
- package/bin/_lib_retained_size.py +187 -0
- package/bin/_lib_snapshot_cache.py +504 -17
- package/bin/_lib_statusline.py +3 -29
- package/bin/_lib_tick_stats.py +28 -4
- package/dashboard/static/assets/ConversationsView-DA4j7Yso.js +72 -0
- package/dashboard/static/assets/DoctorModal-D4PbVnbE.js +1 -0
- package/dashboard/static/assets/ModalRoot-D4oSPxPp.js +1 -0
- package/dashboard/static/assets/ProjectsDrillPanel-DkPlB9JH.js +1 -0
- package/dashboard/static/assets/SourceDetailModal-BE7FONsS.js +1 -0
- package/dashboard/static/assets/UpdateModal-BJuI8glf.js +7 -0
- package/dashboard/static/assets/index-w_bINkJ8.js +13 -0
- package/dashboard/static/assets/outlineNavigation-zDVm6Hdd.js +9 -0
- package/dashboard/static/assets/useKeymap-CJ-Pi17D.js +1 -0
- package/dashboard/static/dashboard.html +2 -1
- package/package.json +4 -1
- package/dashboard/static/assets/index-D19TO7Mg.js +0 -97
package/bin/_cctally_doctor.py
CHANGED
|
@@ -1022,8 +1022,18 @@ def _load_codex_quota_observations_for_doctor(*, force_cold: bool = False):
|
|
|
1022
1022
|
|
|
1023
1023
|
|
|
1024
1024
|
_QUOTA_NO_CHANGE: dict = {
|
|
1025
|
-
"active": False, "effective_from": None,
|
|
1025
|
+
"active": False, "assessed": True, "effective_from": None,
|
|
1026
1026
|
"previous_units_per_point": None, "new_units_per_point": None,
|
|
1027
|
+
"calibration_status": None,
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
#: No regime pair was examined, so no transition claim is available either
|
|
1031
|
+
#: way (#688). Distinct from `_QUOTA_NO_CHANGE`, which is a real negative
|
|
1032
|
+
#: finding over regimes that exist.
|
|
1033
|
+
_QUOTA_NOT_ASSESSED: dict = {
|
|
1034
|
+
"active": False, "assessed": False, "effective_from": None,
|
|
1035
|
+
"previous_units_per_point": None, "new_units_per_point": None,
|
|
1036
|
+
"calibration_status": None,
|
|
1027
1037
|
}
|
|
1028
1038
|
|
|
1029
1039
|
|
|
@@ -1036,14 +1046,18 @@ def _gather_quota_rate_change(c, rejection=None) -> "dict | None":
|
|
|
1036
1046
|
gather goes to the stored regimes rather than through the validated
|
|
1037
1047
|
reader's default, which refuses that mark.
|
|
1038
1048
|
|
|
1039
|
-
An ABSENT calibration is "
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1049
|
+
An ABSENT calibration is "not assessed", NEVER "no change detected"
|
|
1050
|
+
(#688). It was the latter until a real metering change was reported to a
|
|
1051
|
+
user as a green all-clear: the fit was withheld, so no regime persisted,
|
|
1052
|
+
so the calibration was absent, and this gather turned that absence into a
|
|
1053
|
+
negative finding. A store with no fitted regime has no transition to
|
|
1054
|
+
report AND no evidence that none occurred, and `docs/commands/quota.md`
|
|
1055
|
+
forbids conflating the two. `None` is reserved for a calibration that
|
|
1056
|
+
exists and could not be read, where the sibling `quota.calibration` check
|
|
1057
|
+
is already WARNing about it; both render as not assessed.
|
|
1044
1058
|
"""
|
|
1045
1059
|
if rejection == "calibration-absent":
|
|
1046
|
-
return dict(
|
|
1060
|
+
return dict(_QUOTA_NOT_ASSESSED)
|
|
1047
1061
|
mrc = c._load_sibling("_lib_meter_rate_change")
|
|
1048
1062
|
glue = c._load_sibling("_cctally_quota_model")
|
|
1049
1063
|
loaded = glue.read_stored_state_readonly()
|
|
@@ -1053,7 +1067,7 @@ def _gather_quota_rate_change(c, rejection=None) -> "dict | None":
|
|
|
1053
1067
|
active = mrc.active_rate_change(regimes)
|
|
1054
1068
|
if active is None:
|
|
1055
1069
|
return dict(_QUOTA_NO_CHANGE)
|
|
1056
|
-
return active
|
|
1070
|
+
return dict(active, assessed=True)
|
|
1057
1071
|
|
|
1058
1072
|
|
|
1059
1073
|
def doctor_gather_state(
|
package/bin/_cctally_forecast.py
CHANGED
|
@@ -258,10 +258,14 @@ def _apply_midweek_reset_override(
|
|
|
258
258
|
week_start_at: dt.datetime,
|
|
259
259
|
week_end_at: dt.datetime,
|
|
260
260
|
samples: list,
|
|
261
|
+
*,
|
|
262
|
+
now_utc: dt.datetime,
|
|
261
263
|
) -> tuple[dt.datetime, list]:
|
|
262
264
|
"""If the current week's end_at matches a recorded reset event's
|
|
263
|
-
``new_week_end_at
|
|
264
|
-
moment and drop pre-reset
|
|
265
|
+
``new_week_end_at`` and was detected by ``now_utc``, shift
|
|
266
|
+
``week_start_at`` to the effective reset moment and drop pre-reset
|
|
267
|
+
samples. The detection bound keeps historical replay causal while leaving
|
|
268
|
+
the live path unchanged (live ``now_utc`` postdates retained events).
|
|
265
269
|
|
|
266
270
|
Keeps callers (``_load_forecast_inputs``, ``_tui_build_current_week``)
|
|
267
271
|
from reporting spent_usd summed across the pre-reset window.
|
|
@@ -276,8 +280,10 @@ def _apply_midweek_reset_override(
|
|
|
276
280
|
).isoformat(timespec="seconds")
|
|
277
281
|
event_row = conn.execute(
|
|
278
282
|
"SELECT effective_reset_at_utc FROM week_reset_events "
|
|
279
|
-
"WHERE new_week_end_at = ?"
|
|
280
|
-
(
|
|
283
|
+
"WHERE new_week_end_at = ? "
|
|
284
|
+
" AND datetime(detected_at_utc) <= datetime(?) "
|
|
285
|
+
"ORDER BY datetime(detected_at_utc) DESC LIMIT 1",
|
|
286
|
+
(end_iso, now_utc.isoformat()),
|
|
281
287
|
).fetchone()
|
|
282
288
|
if event_row and event_row["effective_reset_at_utc"]:
|
|
283
289
|
reset_dt = parse_iso_datetime(
|
|
@@ -314,7 +320,7 @@ def _resolve_current_budget_window(conn, now_utc, *, account_key=None):
|
|
|
314
320
|
return None
|
|
315
321
|
week_start_at, week_end_at, samples = fetched
|
|
316
322
|
week_start_at, _samples = _apply_midweek_reset_override(
|
|
317
|
-
conn, week_start_at, week_end_at, samples
|
|
323
|
+
conn, week_start_at, week_end_at, samples, now_utc=now_utc
|
|
318
324
|
)
|
|
319
325
|
return (week_start_at, week_end_at)
|
|
320
326
|
|
|
@@ -1158,8 +1164,10 @@ def _calibrated_week_detail(
|
|
|
1158
1164
|
# A `CalibratedWeek`, never the two-value tuple this function returned
|
|
1159
1165
|
# before it grew a dataclass. `_calibrated_projection` reads
|
|
1160
1166
|
# `detail.projection_pct` off the result, so a tuple here raises
|
|
1161
|
-
# `AttributeError` instead of withholding — the same defect class as
|
|
1162
|
-
# `persist_and_detect`
|
|
1167
|
+
# `AttributeError` instead of withholding — the same defect class as a
|
|
1168
|
+
# `persist_and_detect` early return whose arity does not match the rest
|
|
1169
|
+
# of the function. That function now returns THREE values, which is why
|
|
1170
|
+
# its unreadable-state path returns `None, (), ()`.
|
|
1163
1171
|
return CalibratedWeek(code="unavailable")
|
|
1164
1172
|
read = qcg.read_calibration_file(account_key=account_key)
|
|
1165
1173
|
if read.regime is None:
|
|
@@ -1295,7 +1303,7 @@ def _load_forecast_inputs(
|
|
|
1295
1303
|
# reset moment and drop pre-reset samples so elapsed/remaining math
|
|
1296
1304
|
# and spent_usd reflect the post-reset window only.
|
|
1297
1305
|
week_start_at, samples = _apply_midweek_reset_override(
|
|
1298
|
-
conn, week_start_at, week_end_at, samples
|
|
1306
|
+
conn, week_start_at, week_end_at, samples, now_utc=now_utc
|
|
1299
1307
|
)
|
|
1300
1308
|
|
|
1301
1309
|
if not samples:
|
|
@@ -2542,7 +2550,7 @@ def cmd_forecast(args: argparse.Namespace) -> int:
|
|
|
2542
2550
|
if fetched is not None:
|
|
2543
2551
|
_ws_at, _we_at, raw_samples = fetched
|
|
2544
2552
|
_ws_at_shifted, samples = _apply_midweek_reset_override(
|
|
2545
|
-
conn, _ws_at, _we_at, raw_samples
|
|
2553
|
+
conn, _ws_at, _we_at, raw_samples, now_utc=now_utc
|
|
2546
2554
|
)
|
|
2547
2555
|
tz_render = getattr(args, "_resolved_tz", None)
|
|
2548
2556
|
for cap_at, pct, _five_hr in samples:
|
package/bin/_cctally_journal.py
CHANGED
|
@@ -1295,6 +1295,32 @@ class IngestResult:
|
|
|
1295
1295
|
# on a clean cycle. Authoritative callers never see this — their cycle
|
|
1296
1296
|
# exception propagates.
|
|
1297
1297
|
error: object = None
|
|
1298
|
+
# #695: this cycle's meter_rate_change outcome, or None when no descriptor
|
|
1299
|
+
# was passed. `deferred_alerts` alone is insufficient — with notifications
|
|
1300
|
+
# off there is no payload, yet the identity was still decided and the
|
|
1301
|
+
# caller must still mark the ledger.
|
|
1302
|
+
meter_rate_change_result: object = None
|
|
1303
|
+
deferred_alerts: list = field(default_factory=list)
|
|
1304
|
+
|
|
1305
|
+
|
|
1306
|
+
@dataclass(frozen=True)
|
|
1307
|
+
class RateChangeRecordResult:
|
|
1308
|
+
"""What one `record_meter_rate_change` call did to one identity (#695).
|
|
1309
|
+
|
|
1310
|
+
THREE booleans, because the caller must tell three outcomes apart and a
|
|
1311
|
+
single one cannot. `row_created` drives nothing on its own.
|
|
1312
|
+
`notification_queued` says a payload is waiting in `deferred_alerts`.
|
|
1313
|
+
`notification_decided` is the one the ledger keys on: it is true when this
|
|
1314
|
+
call created the row, and when it took the owed branch and observed the
|
|
1315
|
+
durable row — INCLUDING when notifications are off and nothing was
|
|
1316
|
+
queued, because deciding not to notify is a decision. A duplicate or a
|
|
1317
|
+
conflict decides nothing, or it would win the compare-and-set away from
|
|
1318
|
+
the process that is actually going to dispatch.
|
|
1319
|
+
"""
|
|
1320
|
+
|
|
1321
|
+
row_created: bool = False
|
|
1322
|
+
notification_queued: bool = False
|
|
1323
|
+
notification_decided: bool = False
|
|
1298
1324
|
|
|
1299
1325
|
|
|
1300
1326
|
@dataclass
|
|
@@ -1315,6 +1341,12 @@ class IngestContext:
|
|
|
1315
1341
|
batch: list # decoded obs/op records this cycle
|
|
1316
1342
|
config: object = None
|
|
1317
1343
|
pending_alerts: list = field(default_factory=list)
|
|
1344
|
+
# #695: the meter_rate_change family's post-commit payloads. Step 6 does
|
|
1345
|
+
# NOT drain this — `_run_cycle` returns it, and `cmd_quota` dispatches only
|
|
1346
|
+
# after winning the delivery ledger's compare-and-set. Invariant (iv) is
|
|
1347
|
+
# unaffected: these payloads still originate in the live sink and step-4a
|
|
1348
|
+
# replay still has no ctx to add to either list.
|
|
1349
|
+
deferred_alerts: list = field(default_factory=list)
|
|
1318
1350
|
events_emitted: int = 0
|
|
1319
1351
|
# Design B (DB journal redesign §5.3 event+effects): the per-cycle
|
|
1320
1352
|
# suppression map a reset/credit pipeline hook populates BEFORE it runs its
|
|
@@ -2811,7 +2843,18 @@ def rehydrate_codex_journal_families(
|
|
|
2811
2843
|
"DELETE FROM cache_meta WHERE key = ?",
|
|
2812
2844
|
(_cctally_cache.CODEX_FILE_ACCOUNT_CURSOR_KEY,))
|
|
2813
2845
|
if authoritative and want_window_attributions:
|
|
2846
|
+
before = cache_conn.total_changes
|
|
2814
2847
|
cache_conn.execute("DELETE FROM codex_window_attributions")
|
|
2848
|
+
window_rows_cleared = cache_conn.total_changes > before
|
|
2849
|
+
if window_rows_cleared:
|
|
2850
|
+
# The cross-build quota memo keys the semantic attribution
|
|
2851
|
+
# revision, not the replay cursor. With no replacement
|
|
2852
|
+
# assertion, `_apply_window_attribution_records` has no chance
|
|
2853
|
+
# to advance that revision, yet the authoritative empty result
|
|
2854
|
+
# still changed the projection input. Keep the invalidation in
|
|
2855
|
+
# the same transaction as the deletion.
|
|
2856
|
+
_cctally_cache.bump_codex_window_attribution_revision(
|
|
2857
|
+
cache_conn)
|
|
2815
2858
|
# #500 review finding F10: the cursor describes a table that no
|
|
2816
2859
|
# longer has any rows, so leaving it would let the NEXT delta pass
|
|
2817
2860
|
# skip journal bytes on the strength of a claim this branch just
|
|
@@ -2820,11 +2863,14 @@ def rehydrate_codex_journal_families(
|
|
|
2820
2863
|
"DELETE FROM cache_meta WHERE key = ?",
|
|
2821
2864
|
(_cctally_cache.CODEX_WINDOW_ATTRIBUTION_CURSOR_KEY,))
|
|
2822
2865
|
return CodexJournalRehydration(0, None, 0, 0, 0)
|
|
2866
|
+
window_rows_cleared = False
|
|
2823
2867
|
if authoritative:
|
|
2824
2868
|
if want_file_accounts:
|
|
2825
2869
|
cache_conn.execute("DELETE FROM codex_file_accounts")
|
|
2826
2870
|
if want_window_attributions:
|
|
2871
|
+
before = cache_conn.total_changes
|
|
2827
2872
|
cache_conn.execute("DELETE FROM codex_window_attributions")
|
|
2873
|
+
window_rows_cleared = cache_conn.total_changes > before
|
|
2828
2874
|
# A clear-then-replay is only correct from the beginning of the journal.
|
|
2829
2875
|
file_account_since = window_attribution_since = None
|
|
2830
2876
|
|
|
@@ -2858,6 +2904,7 @@ def rehydrate_codex_journal_families(
|
|
|
2858
2904
|
segment_positions = {name: idx for idx, name in enumerate(segments)}
|
|
2859
2905
|
|
|
2860
2906
|
applied = conflicts = attributed = skipped = 0
|
|
2907
|
+
window_records_changed = False
|
|
2861
2908
|
# Streamed, never materialized, for the reason above: this runs on the FIRST
|
|
2862
2909
|
# ordinary sync of every cache.db while both cache flocks are held. The
|
|
2863
2910
|
# cheap byte prefilters skip the JSON decode for every uninteresting line;
|
|
@@ -2892,6 +2939,14 @@ def rehydrate_codex_journal_families(
|
|
|
2892
2939
|
_apply_window_attribution_records(cache_conn, (rec,)))
|
|
2893
2940
|
attributed += _asserted
|
|
2894
2941
|
skipped += _skipped
|
|
2942
|
+
window_records_changed = window_records_changed or bool(
|
|
2943
|
+
_asserted or _retracted)
|
|
2944
|
+
if window_rows_cleared and not window_records_changed:
|
|
2945
|
+
# A replacement assertion/retraction advances the revision inside its
|
|
2946
|
+
# applier. The deletion-only authoritative outcome has no applier-side
|
|
2947
|
+
# mutation, so advance it here instead. This also covers a journal
|
|
2948
|
+
# containing only malformed/skipped attribution records.
|
|
2949
|
+
_cctally_cache.bump_codex_window_attribution_revision(cache_conn)
|
|
2895
2950
|
if want_window_attributions and (
|
|
2896
2951
|
authoritative or window_attribution_since != (str(hw[0]), int(hw[1]))):
|
|
2897
2952
|
# Only when it actually moves. This runs on every ordinary Codex sync,
|
|
@@ -3307,18 +3362,29 @@ def _now_iso() -> str:
|
|
|
3307
3362
|
)
|
|
3308
3363
|
|
|
3309
3364
|
|
|
3310
|
-
def _usage_snapshot_fold_decision(conn, payload) -> tuple[bool, object]:
|
|
3365
|
+
def _usage_snapshot_fold_decision(conn, payload) -> tuple[bool, object, str]:
|
|
3311
3366
|
"""The apply-time dedup for a Claude rate-limit obs — the exact predicate
|
|
3312
3367
|
ported from `cmd_record_usage`'s insert guard (bin/_cctally_record.py), now
|
|
3313
3368
|
at fold time (spec §4.5 / §5.3).
|
|
3314
3369
|
|
|
3315
|
-
Returns `(skip, adjusted_five_hour_percent)`:
|
|
3370
|
+
Returns `(skip, adjusted_five_hour_percent, reason)`:
|
|
3316
3371
|
- reset-aware 7d HWM clamp (`_reset_aware_floor` + reset-aware MAX +
|
|
3317
|
-
`hwm_clamp_applies`) → skip when a lower 7d % would be clamped
|
|
3372
|
+
`hwm_clamp_applies`) → skip when a lower 7d % would be clamped, with
|
|
3373
|
+
reason `SNAPSHOT_SKIP_CLAMP`;
|
|
3318
3374
|
- the 5h clamp adjusts `five_hour_percent` UP to the in-window MAX but
|
|
3319
3375
|
never gates (mirrors the nested-else in the live code);
|
|
3320
|
-
- dedup vs the latest snapshot in the week: both percents unchanged → skip
|
|
3321
|
-
|
|
3376
|
+
- dedup vs the latest snapshot in the week: both percents unchanged → skip,
|
|
3377
|
+
with reason `SNAPSHOT_SKIP_DEDUP`;
|
|
3378
|
+
- otherwise accept, with reason `SNAPSHOT_ACCEPT`.
|
|
3379
|
+
|
|
3380
|
+
`reason` exists because the two skips mean OPPOSITE things about the
|
|
3381
|
+
incoming observation and the stored row (see `_lib_record` Fragment 8): a
|
|
3382
|
+
dedup skip AGREES with the stored row, so re-running the derivation
|
|
3383
|
+
chokepoints against it heals a killed tick; a clamp skip CONTRADICTS it, so
|
|
3384
|
+
deriving a weekly milestone from the higher stored value fabricates a
|
|
3385
|
+
crossing. `_pipeline_claude_usage` gates the weekly milestone on it. The
|
|
3386
|
+
reason is derived from the same `conn` state and payload as `skip` itself,
|
|
3387
|
+
so it is exactly as replay-deterministic as the boolean it accompanies."""
|
|
3322
3388
|
week_start_date = payload["week_start_date"]
|
|
3323
3389
|
week_start_at = payload.get("week_start_at")
|
|
3324
3390
|
week_end_at = payload.get("week_end_at")
|
|
@@ -3344,7 +3410,7 @@ def _usage_snapshot_fold_decision(conn, payload) -> tuple[bool, object]:
|
|
|
3344
3410
|
).fetchone()
|
|
3345
3411
|
max_v = max_row[0] if max_row else None
|
|
3346
3412
|
if _lib_record.hwm_clamp_applies(weekly_percent, max_v):
|
|
3347
|
-
return True, five_hour_percent
|
|
3413
|
+
return True, five_hour_percent, _lib_record.SNAPSHOT_SKIP_CLAMP
|
|
3348
3414
|
|
|
3349
3415
|
adjusted_5h = five_hour_percent
|
|
3350
3416
|
if five_hour_percent is not None and five_hour_window_key is not None:
|
|
@@ -3374,8 +3440,8 @@ def _usage_snapshot_fold_decision(conn, payload) -> tuple[bool, object]:
|
|
|
3374
3440
|
if adjusted_5h is None or (
|
|
3375
3441
|
last_5h is not None and float(last_5h) == float(adjusted_5h)
|
|
3376
3442
|
):
|
|
3377
|
-
return True, adjusted_5h
|
|
3378
|
-
return False, adjusted_5h
|
|
3443
|
+
return True, adjusted_5h, _lib_record.SNAPSHOT_SKIP_DEDUP
|
|
3444
|
+
return False, adjusted_5h, _lib_record.SNAPSHOT_ACCEPT
|
|
3379
3445
|
|
|
3380
3446
|
|
|
3381
3447
|
# NOTE (rev 3): the direct obs -> weekly_usage_snapshots fold is GONE. That
|
|
@@ -4297,8 +4363,11 @@ def emit_model_a(ctx, *, kind, evt_id, table, columns, refs=None, at=None):
|
|
|
4297
4363
|
decision = _classify_live_effective_event(ctx.conn, evt)
|
|
4298
4364
|
if decision == CLASSIFY_CONFLICT:
|
|
4299
4365
|
# No append, no metadata mutation — converge the row instead.
|
|
4300
|
-
_record_dropped_conflict(ctx, evt)
|
|
4301
|
-
|
|
4366
|
+
selected = _record_dropped_conflict(ctx, evt)
|
|
4367
|
+
_converge_and_report(
|
|
4368
|
+
selected,
|
|
4369
|
+
lambda: _converge_row_from_effective(
|
|
4370
|
+
ctx.conn, evt_id, table=table))
|
|
4302
4371
|
else:
|
|
4303
4372
|
# `new` AND `duplicate` both still append: crash-replay convergence
|
|
4304
4373
|
# and two-bootstrap idempotency are built on that.
|
|
@@ -4386,10 +4455,11 @@ def _emit_harvest_row(ctx, spec, row):
|
|
|
4386
4455
|
|
|
4387
4456
|
decision = _classify_live_effective_event(conn, evt)
|
|
4388
4457
|
if decision == CLASSIFY_CONFLICT:
|
|
4389
|
-
_record_dropped_conflict(ctx, evt)
|
|
4390
|
-
|
|
4391
|
-
|
|
4392
|
-
|
|
4458
|
+
selected = _record_dropped_conflict(ctx, evt)
|
|
4459
|
+
_converge_and_report(
|
|
4460
|
+
selected,
|
|
4461
|
+
lambda: _converge_row_from_effective(
|
|
4462
|
+
conn, evt["id"], table=spec.table, rowid=row["id"]))
|
|
4393
4463
|
return evt
|
|
4394
4464
|
|
|
4395
4465
|
append_record(evt)
|
|
@@ -4468,36 +4538,169 @@ PIPELINE.append(_pipeline_op_fold)
|
|
|
4468
4538
|
|
|
4469
4539
|
|
|
4470
4540
|
def record_meter_rate_change(ctx, transition, *, notify: bool,
|
|
4471
|
-
created_at: str
|
|
4541
|
+
created_at: str,
|
|
4542
|
+
notification_owed: bool = False):
|
|
4472
4543
|
"""Steps 4 and 5 of spec §6.5, run INSIDE the cycle transaction.
|
|
4473
4544
|
|
|
4474
4545
|
Journal-first (invariant i): the evt line is appended and fsync'd through
|
|
4475
4546
|
the leaf `journal.lock` before the transaction containing its row commits,
|
|
4476
|
-
exactly as `emit_model_a` does.
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
|
|
4484
|
-
|
|
4485
|
-
|
|
4486
|
-
that
|
|
4547
|
+
exactly as `emit_model_a` does.
|
|
4548
|
+
|
|
4549
|
+
TWO guards, and they catch inverse conditions (#689). The natural-key
|
|
4550
|
+
recheck catches a committed physical latch whose effective metadata is not
|
|
4551
|
+
yet populated. `_classify_live_effective_event` catches the inverse —
|
|
4552
|
+
prior metadata with the row absent — and additionally tells an exact
|
|
4553
|
+
duplicate apart from divergent same-revision content. Recovery re-offers
|
|
4554
|
+
persisted pairs under a later command clock, and the payload includes that
|
|
4555
|
+
clock, so without the classifier a retry would append a byte-different
|
|
4556
|
+
line under an id the journal already holds. This was the only emit path in
|
|
4557
|
+
this file that appended without classifying.
|
|
4558
|
+
|
|
4559
|
+
NEITHER the duplicate nor the conflict branch may rely on replay to
|
|
4560
|
+
restore a missing row. `_preflight_live_events` omits a same-revision,
|
|
4561
|
+
same-hash event from `to_apply`, and step 4a applies only what
|
|
4562
|
+
`_record_live_effective_event` returns True for, which is NEW alone — so
|
|
4563
|
+
the cursor advances past a duplicate and the row would stay absent.
|
|
4564
|
+
`_converge_row_from_effective` cannot cover it either, because this
|
|
4565
|
+
family's `_EVT_SPECS` entry has `table=None` and it returns
|
|
4566
|
+
CONVERGE_DROPPED. So this function materializes its own row: from the
|
|
4567
|
+
identical event on a duplicate, and from the PRIOR journaled event on a
|
|
4568
|
+
conflict, never from the rejected candidate.
|
|
4569
|
+
|
|
4570
|
+
Returns a `RateChangeRecordResult`, not a boolean (#695). Three outcomes
|
|
4571
|
+
have to be told apart: whether this call created the row, whether a
|
|
4572
|
+
payload is waiting, and whether this call OWNS the dispatch decision for
|
|
4573
|
+
this identity. `notification_decided` is the last of those, and the
|
|
4574
|
+
caller's delivery ledger keys on it.
|
|
4575
|
+
|
|
4576
|
+
A notification is queued when this call created the row and `notify` is
|
|
4577
|
+
true — §6.2 splits recording from notifying, so the row exists from the
|
|
4578
|
+
first upgrade while the push follows the existing default-off toggle.
|
|
4579
|
+
|
|
4580
|
+
#695 CORRECTS this docstring's former claim that "existing effective
|
|
4581
|
+
metadata proves this identity was journaled once and so already had its
|
|
4582
|
+
notification". It does not. Step-4a replay of an orphan line — appended by
|
|
4583
|
+
a cycle that then rolled back — creates the row with no `IngestContext`,
|
|
4584
|
+
so invariant (iv) makes that replay structurally unable to queue anything.
|
|
4585
|
+
The row exists and nobody was told. So an existing row now takes the owed
|
|
4586
|
+
branch when, and only when, `notification_owed` says the live command's
|
|
4587
|
+
delivery ledger holds no decision for this identity; the payload is then
|
|
4588
|
+
rebuilt from the DURABLE ROW, which is what was actually recorded. An
|
|
4589
|
+
ordinary duplicate and every conflict branch stay silent, because
|
|
4590
|
+
re-notifying without that explicit intent would refire history.
|
|
4591
|
+
|
|
4592
|
+
The queue is `ctx.deferred_alerts`, which step 6 does NOT drain: this
|
|
4593
|
+
family's dispatch is returned to `cmd_quota`, which fires it only after
|
|
4594
|
+
winning the ledger's compare-and-set. Invariant (iv) is untouched — the
|
|
4595
|
+
payload still originates in the live sink, post-commit, and replay still
|
|
4596
|
+
has no ctx to add to either list.
|
|
4487
4597
|
"""
|
|
4488
4598
|
mrc = _load_meter_rate_change()
|
|
4599
|
+
existing = ctx.conn.execute(
|
|
4600
|
+
"SELECT provider, account_key, effective_from,"
|
|
4601
|
+
" previous_units_per_point, new_units_per_point, severity "
|
|
4602
|
+
"FROM meter_rate_change_events "
|
|
4603
|
+
"WHERE provider = ? AND account_key = ? AND effective_from = ? "
|
|
4604
|
+
"LIMIT 1", transition.identity()).fetchone()
|
|
4605
|
+
if existing is not None:
|
|
4606
|
+
# #695. A row can reach this table with NO notification: step-4a
|
|
4607
|
+
# replay of an orphan line creates it, and replay has no ctx to queue
|
|
4608
|
+
# anything. So an existing row no longer proves a notification
|
|
4609
|
+
# happened, and `notification_owed` is the live command's assertion
|
|
4610
|
+
# that its delivery ledger holds no decision for this identity.
|
|
4611
|
+
# Without that flag this stays silent, because an ordinary duplicate
|
|
4612
|
+
# must not re-fire.
|
|
4613
|
+
if not notification_owed:
|
|
4614
|
+
return RateChangeRecordResult()
|
|
4615
|
+
if not notify:
|
|
4616
|
+
return RateChangeRecordResult(notification_decided=True)
|
|
4617
|
+
# Rebuilt from the ROW rather than from the caller's descriptor: the
|
|
4618
|
+
# sweep knows only the identity, so its descriptor carries placeholder
|
|
4619
|
+
# rates and severity. `withholding_status` stays at its `None`
|
|
4620
|
+
# default, because it describes the analysis that originally admitted
|
|
4621
|
+
# the transition and a later run's analysis is not that one.
|
|
4622
|
+
ctx.deferred_alerts.append(mrc.alert_payload(
|
|
4623
|
+
mrc.RateChangeTransition(
|
|
4624
|
+
provider=str(existing[0]),
|
|
4625
|
+
account_key=str(existing[1]),
|
|
4626
|
+
effective_from=str(existing[2]),
|
|
4627
|
+
previous_units_per_point=float(existing[3]),
|
|
4628
|
+
new_units_per_point=float(existing[4]),
|
|
4629
|
+
severity=str(existing[5]),
|
|
4630
|
+
detected_at=str(transition.detected_at))))
|
|
4631
|
+
return RateChangeRecordResult(notification_queued=True,
|
|
4632
|
+
notification_decided=True)
|
|
4633
|
+
if notification_owed:
|
|
4634
|
+
# #695. The owed sweep offers an IDENTITY whose durable row it read a
|
|
4635
|
+
# moment ago, so its descriptor carries placeholder zero rates and an
|
|
4636
|
+
# `info` severity that are never meant to be published. Reaching here
|
|
4637
|
+
# means the row is gone — there is nothing to re-notify, and the
|
|
4638
|
+
# placeholder must not be journaled as a real zero-rate change. This
|
|
4639
|
+
# is what ENFORCES the claim `_owed_rate_change_transitions` makes;
|
|
4640
|
+
# without it the prose asserts an invariant the code does not hold.
|
|
4641
|
+
return RateChangeRecordResult()
|
|
4489
4642
|
payload = mrc.event_payload(transition, created_at=created_at)
|
|
4490
4643
|
eid = _lib_journal.evt_id(
|
|
4491
4644
|
mrc.EVT_ID_PREFIX, transition.provider, transition.account_key,
|
|
4492
4645
|
transition.effective_from)
|
|
4493
4646
|
evt = _lib_journal.make_evt(
|
|
4494
4647
|
kind=mrc.EVT_KIND, id=eid, at=created_at, payload=payload)
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
|
|
4498
|
-
|
|
4499
|
-
ctx.
|
|
4500
|
-
|
|
4648
|
+
decision = _classify_live_effective_event(ctx.conn, evt)
|
|
4649
|
+
if decision == CLASSIFY_NEW:
|
|
4650
|
+
append_record(evt)
|
|
4651
|
+
ctx.events_emitted += 1
|
|
4652
|
+
_record_new_effective_event(ctx.conn, evt)
|
|
4653
|
+
created = _insert_meter_rate_change(ctx.conn, evt)
|
|
4654
|
+
if created and notify:
|
|
4655
|
+
ctx.deferred_alerts.append(mrc.alert_payload(transition))
|
|
4656
|
+
return RateChangeRecordResult(
|
|
4657
|
+
row_created=created,
|
|
4658
|
+
notification_queued=bool(created and notify),
|
|
4659
|
+
notification_decided=created)
|
|
4660
|
+
if decision == CLASSIFY_CONFLICT:
|
|
4661
|
+
selected = _record_dropped_conflict(ctx, evt)
|
|
4662
|
+
_converge_and_report(
|
|
4663
|
+
selected, lambda: _converge_meter_rate_change_row(ctx.conn, eid))
|
|
4664
|
+
return RateChangeRecordResult()
|
|
4665
|
+
_insert_meter_rate_change(ctx.conn, evt)
|
|
4666
|
+
return RateChangeRecordResult()
|
|
4667
|
+
|
|
4668
|
+
|
|
4669
|
+
def _converge_meter_rate_change_row(conn, event_id) -> str:
|
|
4670
|
+
"""Materialize the withheld row from the PRIOR journaled event (#689).
|
|
4671
|
+
|
|
4672
|
+
The outcome is the INSERT's own answer, never an assumption that calling
|
|
4673
|
+
it worked. `_insert_meter_rate_change` returns False for a payload
|
|
4674
|
+
`_meter_rate_change_row` cannot normalise — a missing `provider` or
|
|
4675
|
+
`effective_from`, or a non-numeric rate — and reporting
|
|
4676
|
+
`CONVERGE_APPLIED` there would print a line claiming a convergence that
|
|
4677
|
+
inserted nothing, which is exactly what acceptance 6 forbids. A False
|
|
4678
|
+
return here cannot mean "a row already existed": the natural-key recheck
|
|
4679
|
+
at the top of `record_meter_rate_change` ran inside this same transaction
|
|
4680
|
+
and found none.
|
|
4681
|
+
"""
|
|
4682
|
+
prior = _prior_meter_rate_change_event(conn, event_id)
|
|
4683
|
+
if prior is None:
|
|
4684
|
+
return CONVERGE_DROPPED
|
|
4685
|
+
return (CONVERGE_APPLIED if _insert_meter_rate_change(conn, prior)
|
|
4686
|
+
else CONVERGE_DROPPED)
|
|
4687
|
+
|
|
4688
|
+
|
|
4689
|
+
def _prior_meter_rate_change_event(conn, event_id):
|
|
4690
|
+
"""The prior effective event to materialize a withheld row from, or None.
|
|
4691
|
+
|
|
4692
|
+
A TOMBSTONED selection stores `event_json` as NULL and
|
|
4693
|
+
`_effective_event_for_convergence` fails closed and raises on it. Here a
|
|
4694
|
+
tombstone is a terminal negative latch rather than an error: a completed
|
|
4695
|
+
correction saying this identity should have no row must not be undone by
|
|
4696
|
+
materializing one.
|
|
4697
|
+
"""
|
|
4698
|
+
row = conn.execute(
|
|
4699
|
+
"SELECT status FROM journal_effective_events WHERE event_id = ?",
|
|
4700
|
+
(event_id,)).fetchone()
|
|
4701
|
+
if row is None or str(row[0]) != "active":
|
|
4702
|
+
return None
|
|
4703
|
+
return _effective_event_for_convergence(conn, event_id)
|
|
4501
4704
|
|
|
4502
4705
|
|
|
4503
4706
|
def _load_meter_rate_change():
|
|
@@ -5268,9 +5471,16 @@ def _record_live_effective_event(conn, evt) -> bool:
|
|
|
5268
5471
|
return False
|
|
5269
5472
|
|
|
5270
5473
|
|
|
5271
|
-
def _record_dropped_conflict(ctx, evt)
|
|
5272
|
-
"""Count
|
|
5273
|
-
|
|
5474
|
+
def _record_dropped_conflict(ctx, evt):
|
|
5475
|
+
"""Count one withheld emission and RETURN its selection, so the caller can
|
|
5476
|
+
render the outcome once convergence has resolved (#689).
|
|
5477
|
+
|
|
5478
|
+
Spec §8 asks for a one-line stderr note per dropped emission and a count on
|
|
5479
|
+
the cycle summary. Recording is separated from rendering because the
|
|
5480
|
+
wording depends on whether a row was actually converged, and for a
|
|
5481
|
+
`table=None` family nothing is. Doing both after convergence would lose the
|
|
5482
|
+
`DroppedConflict` too when convergence raises, not merely the stderr line.
|
|
5483
|
+
"""
|
|
5274
5484
|
selected = _lib_journal.resolve_effective_events([evt]).by_id[evt["id"]]
|
|
5275
5485
|
ctx.conflicts_dropped.append(
|
|
5276
5486
|
DroppedConflict(
|
|
@@ -5279,13 +5489,42 @@ def _record_dropped_conflict(ctx, evt) -> None:
|
|
|
5279
5489
|
rejected_hash=selected.content_hash,
|
|
5280
5490
|
)
|
|
5281
5491
|
)
|
|
5492
|
+
return selected
|
|
5493
|
+
|
|
5494
|
+
|
|
5495
|
+
def _report_dropped_conflict(selected, outcome) -> None:
|
|
5496
|
+
"""Spec §8's one-line stderr note, worded from what convergence did.
|
|
5497
|
+
|
|
5498
|
+
An unrecognised outcome takes the DROPPED wording, because the direction
|
|
5499
|
+
of mistake that matters is claiming a convergence that did not happen.
|
|
5500
|
+
"""
|
|
5501
|
+
tail = _CONVERGE_OUTCOME_WORDING.get(
|
|
5502
|
+
outcome, _CONVERGE_OUTCOME_WORDING[CONVERGE_DROPPED])
|
|
5282
5503
|
print(
|
|
5283
5504
|
f"[journal] withheld a divergent emission for {selected.event_id} "
|
|
5284
|
-
f"rev {selected.rev};
|
|
5505
|
+
f"rev {selected.rev}; {tail}",
|
|
5285
5506
|
file=sys.stderr,
|
|
5286
5507
|
)
|
|
5287
5508
|
|
|
5288
5509
|
|
|
5510
|
+
def _converge_and_report(selected, converge):
|
|
5511
|
+
"""Run one convergence and render §8's line WHATEVER it does (#689).
|
|
5512
|
+
|
|
5513
|
+
The line is in a `finally`, so a convergence that fails closed still
|
|
5514
|
+
leaves the diagnostic behind. Before the record/report split it did,
|
|
5515
|
+
because the whole call ran ahead of convergence; the split moved the
|
|
5516
|
+
rendering after the outcome was known and would otherwise have dropped
|
|
5517
|
+
the line on exactly the path that most needs one. The exception is not
|
|
5518
|
+
caught — it propagates through the `finally` unchanged.
|
|
5519
|
+
"""
|
|
5520
|
+
outcome = CONVERGE_RAISED
|
|
5521
|
+
try:
|
|
5522
|
+
outcome = converge()
|
|
5523
|
+
finally:
|
|
5524
|
+
_report_dropped_conflict(selected, outcome)
|
|
5525
|
+
return outcome
|
|
5526
|
+
|
|
5527
|
+
|
|
5289
5528
|
def _effective_event_for_convergence(conn, event_id) -> dict:
|
|
5290
5529
|
"""The ACTIVE, validated journal record the live row must converge to.
|
|
5291
5530
|
|
|
@@ -5355,6 +5594,18 @@ def _evt_target_columns(conn, evt, spec) -> tuple:
|
|
|
5355
5594
|
|
|
5356
5595
|
CONVERGE_DROPPED = "dropped"
|
|
5357
5596
|
CONVERGE_APPLIED = "converged"
|
|
5597
|
+
#: Convergence raised rather than returning an outcome. Only `_converge_and_
|
|
5598
|
+
#: report`'s `finally` produces it, and the exception still propagates.
|
|
5599
|
+
CONVERGE_RAISED = "raised"
|
|
5600
|
+
|
|
5601
|
+
#: §8's stderr tail per outcome. The APPLIED and DROPPED wordings are frozen —
|
|
5602
|
+
#: `tests/test_meter_rate_change_journal.py` asserts both verbatim.
|
|
5603
|
+
_CONVERGE_OUTCOME_WORDING: dict = {
|
|
5604
|
+
CONVERGE_APPLIED: "converged the row from the journaled event",
|
|
5605
|
+
CONVERGE_DROPPED: "the journaled event stands and there was no row to "
|
|
5606
|
+
"converge",
|
|
5607
|
+
CONVERGE_RAISED: "convergence failed and the row was left unchanged",
|
|
5608
|
+
}
|
|
5358
5609
|
|
|
5359
5610
|
|
|
5360
5611
|
def _converge_row_from_effective(conn, event_id, *, table=None, rowid=None) -> str:
|
|
@@ -6282,6 +6533,9 @@ def _run_cycle(conn: sqlite3.Connection, *, reconcile_config=None,
|
|
|
6282
6533
|
# the next cycle's step 4a.
|
|
6283
6534
|
ctx = IngestContext(conn=conn, batch=batch,
|
|
6284
6535
|
config=(_load_config_once() if batch else None))
|
|
6536
|
+
# #695: None unless step 4b''' runs, so a cycle with no `meter_rate_change`
|
|
6537
|
+
# descriptor reports exactly what it did before this change.
|
|
6538
|
+
rate_change_result = None
|
|
6285
6539
|
conn.execute("BEGIN IMMEDIATE")
|
|
6286
6540
|
try:
|
|
6287
6541
|
# 4a. Replay journal evt lines (a prior cycle's emission that landed past
|
|
@@ -6323,14 +6577,16 @@ def _run_cycle(conn: sqlite3.Connection, *, reconcile_config=None,
|
|
|
6323
6577
|
# already decided under the calibration file's leaf lock and released
|
|
6324
6578
|
# before any stats lock was taken (step 2), is journaled and applied
|
|
6325
6579
|
# here — inside this transaction, journal-first, with its notification
|
|
6326
|
-
# queued to `ctx.
|
|
6580
|
+
# queued to `ctx.deferred_alerts` for the CALLER to dispatch (#695).
|
|
6327
6581
|
# It sits beside the Codex leg rather than in the pipeline because its
|
|
6328
6582
|
# trigger is a persistence transition rather than a journal record.
|
|
6329
6583
|
if meter_rate_change is not None:
|
|
6330
|
-
record_meter_rate_change(
|
|
6584
|
+
rate_change_result = record_meter_rate_change(
|
|
6331
6585
|
ctx, meter_rate_change["transition"],
|
|
6332
6586
|
notify=bool(meter_rate_change.get("notify")),
|
|
6333
|
-
created_at=str(meter_rate_change["created_at"])
|
|
6587
|
+
created_at=str(meter_rate_change["created_at"]),
|
|
6588
|
+
notification_owed=bool(
|
|
6589
|
+
meter_rate_change.get("notification_owed")))
|
|
6334
6590
|
# 4c. Journal + stamp the natural-keyed rows the pipeline inserted.
|
|
6335
6591
|
# Early-out (Task 6 gate P2): the ONLY source of `journal_id IS NULL`
|
|
6336
6592
|
# rows is a Task-5 chokepoint called from a step-4b pipeline hook —
|
|
@@ -6397,7 +6653,9 @@ def _run_cycle(conn: sqlite3.Connection, *, reconcile_config=None,
|
|
|
6397
6653
|
return IngestResult(ran=True, consumed=len(records), malformed=malformed,
|
|
6398
6654
|
events_emitted=ctx.events_emitted, alerts=alerts,
|
|
6399
6655
|
conflicts_dropped=(len(ctx.conflicts_dropped)
|
|
6400
|
-
+ len(preflight_conflicts))
|
|
6656
|
+
+ len(preflight_conflicts)),
|
|
6657
|
+
meter_rate_change_result=rate_change_result,
|
|
6658
|
+
deferred_alerts=list(ctx.deferred_alerts))
|
|
6401
6659
|
|
|
6402
6660
|
|
|
6403
6661
|
def _run_stats_ingest_once(
|
package/bin/_cctally_quota.py
CHANGED
|
@@ -317,6 +317,11 @@ PROJECTION_DYNAMIC_READ_SITES: "dict[str, int]" = {
|
|
|
317
317
|
"_cctally_dashboard.py": 3,
|
|
318
318
|
"_cctally_dashboard_envelope.py": 5,
|
|
319
319
|
"_cctally_db.py": 7,
|
|
320
|
+
# NOT a read. The worker-loader failure says `cannot load cctally worker
|
|
321
|
+
# from {path}`, which the case-insensitive `FROM\s+{` pattern cannot tell
|
|
322
|
+
# from a dynamic SQL target. The module's SQL table names remain literal,
|
|
323
|
+
# so no `PROJECTION_DYNAMIC_READ_ACTIONS` classification applies.
|
|
324
|
+
"_cctally_diagnosis_sources.py": 1,
|
|
320
325
|
"_cctally_doctor.py": 1,
|
|
321
326
|
"_cctally_five_hour.py": 1,
|
|
322
327
|
"_cctally_journal.py": 18,
|
|
@@ -331,9 +336,19 @@ PROJECTION_DYNAMIC_READ_SITES: "dict[str, int]" = {
|
|
|
331
336
|
"_cctally_release.py": 4,
|
|
332
337
|
"_cctally_setup.py": 3,
|
|
333
338
|
"_cctally_tui.py": 1,
|
|
339
|
+
# One, in `_codex_rollup_revision`: the table is selected from a hardcoded
|
|
340
|
+
# pair of Codex conversation rollups. Neither is a quota projection table,
|
|
341
|
+
# so no `PROJECTION_DYNAMIC_READ_ACTIONS` classification applies.
|
|
342
|
+
"_lib_codex_conversation_query.py": 1,
|
|
334
343
|
"_lib_conversation_query.py": 1,
|
|
335
344
|
"_lib_conversation_retention.py": 2,
|
|
336
345
|
"_lib_doctor.py": 1,
|
|
346
|
+
# Three: `_target_has_cursor_gap` selects one of the two accounting source
|
|
347
|
+
# tables, while `_conversation_source_paths` and `_conversation_target_risk`
|
|
348
|
+
# select one of the two transcript source tables. Every choice is hardcoded;
|
|
349
|
+
# none is a quota projection table, so no
|
|
350
|
+
# `PROJECTION_DYNAMIC_READ_ACTIONS` classification applies.
|
|
351
|
+
"_lib_ingest_frontier.py": 3,
|
|
337
352
|
"_lib_snapshot_cache.py": 1,
|
|
338
353
|
"_lib_subscription_weeks.py": 1,
|
|
339
354
|
# NOT a read. `bin/_lib_test_estate.py` opens no database and issues no SQL;
|