cctally 1.116.1 → 1.117.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +57 -3
- package/bin/_cctally_cache.py +5720 -885
- package/bin/_cctally_claude_attribution.py +997 -0
- package/bin/_cctally_completion.py +101 -28
- package/bin/_cctally_core.py +235 -12
- package/bin/_cctally_dashboard.py +263 -17
- package/bin/_cctally_dashboard_conversation.py +107 -21
- package/bin/_cctally_dashboard_envelope.py +44 -0
- package/bin/_cctally_db.py +928 -80
- package/bin/_cctally_diagnosis_sources.py +78 -13
- package/bin/_cctally_doctor.py +211 -86
- package/bin/_cctally_five_hour.py +446 -22
- package/bin/_cctally_forecast.py +302 -6
- package/bin/_cctally_journal.py +2418 -158
- package/bin/_cctally_milestone_history.py +379 -91
- package/bin/_cctally_oauth_samples.py +185 -7
- package/bin/_cctally_parser.py +38 -0
- package/bin/_cctally_project.py +34 -8
- package/bin/_cctally_quarantine.py +821 -343
- package/bin/_cctally_quota.py +69 -4
- package/bin/_cctally_quota_calibration.py +677 -3
- package/bin/_cctally_quota_model.py +927 -127
- package/bin/_cctally_record.py +922 -445
- package/bin/_cctally_rederive.py +119 -15
- package/bin/_cctally_statusline.py +1048 -176
- package/bin/_cctally_store.py +544 -110
- package/bin/_cctally_tui.py +455 -42
- package/bin/_cctally_update.py +77 -14
- package/bin/_lib_account_protocol.py +227 -19
- package/bin/_lib_claude_attribution.py +421 -0
- package/bin/_lib_clone_copy.py +30 -0
- package/bin/_lib_codex_conversation_query.py +45 -9
- package/bin/_lib_codex_conversation_watch.py +72 -13
- package/bin/_lib_codex_rollout_container.py +234 -19
- package/bin/_lib_codex_worker_lifecycle.py +374 -0
- package/bin/_lib_credit.py +22 -13
- package/bin/_lib_dashboard_sources.py +257 -18
- package/bin/_lib_detector_access_registry.py +1335 -166
- package/bin/_lib_doctor.py +276 -92
- package/bin/_lib_ingest_frontier.py +269 -36
- package/bin/_lib_journal.py +273 -22
- package/bin/_lib_jsonl.py +12 -0
- package/bin/_lib_meter_rate_change.py +72 -0
- package/bin/_lib_outcome_quarantine.py +917 -554
- package/bin/_lib_provider_activity.py +24 -6
- package/bin/_lib_quota_calibration.py +57 -3
- package/bin/_lib_quota_copy.py +7 -2
- package/bin/_lib_quota_lineage.py +613 -0
- package/bin/_lib_quota_model.py +464 -33
- package/bin/_lib_rate_change_delivery.py +16 -0
- package/bin/_lib_reclaim_planner.py +6 -4
- package/bin/_lib_rederive.py +2046 -21
- package/bin/_lib_selector_state.py +131 -3
- package/bin/_lib_snapshot_cache.py +8 -0
- package/bin/_lib_stale_proof.py +113 -4
- package/bin/_lib_statusline.py +72 -3
- package/bin/_lib_statusline_candidates.py +246 -42
- package/bin/_lib_tick_stats.py +10 -5
- package/bin/_lib_write_io.py +419 -6
- package/bin/cctally +28 -11
- package/dashboard/static/assets/{ConversationsView-Dflb_myA.js → ConversationsView-oqNsWzWV.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-DFX5CfIs.js → DoctorModal-B0DnU9d9.js} +1 -1
- package/dashboard/static/assets/ModalRoot-CYua2wP_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-C3PJy1xx.js → ProjectsDrillPanel-DSxXQSa-.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-BczsWKhD.js → SourceDetailModal-BleDpQqG.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-Os-Lo5cs.js → UpdateModal-CMqaAkSK.js} +1 -1
- package/dashboard/static/assets/{index-CYZ3NLqF.js → index-BjQV-0-D.js} +3 -3
- package/dashboard/static/assets/index-D34eYhAc.css +1 -0
- package/dashboard/static/assets/{outlineNavigation-k-zIif6W.js → outlineNavigation-CO-VWBm7.js} +1 -1
- package/dashboard/static/assets/{useKeymap-CW7Up0Ay.js → useKeymap-CUgEnnua.js} +1 -1
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +5 -1
- package/dashboard/static/assets/ModalRoot-HlxoagIa.js +0 -1
- package/dashboard/static/assets/index-BKITk1CT.css +0 -1
|
@@ -23,6 +23,7 @@ import json
|
|
|
23
23
|
import os
|
|
24
24
|
import sqlite3
|
|
25
25
|
import sys
|
|
26
|
+
import textwrap
|
|
26
27
|
|
|
27
28
|
import _cctally_core
|
|
28
29
|
import _cctally_store
|
|
@@ -90,7 +91,12 @@ def published_coefficient_eras() -> list:
|
|
|
90
91
|
every call, so the published table can never drift from the one the
|
|
91
92
|
kernel weights with.
|
|
92
93
|
"""
|
|
93
|
-
|
|
94
|
+
# #991: each era also names the reference family its meter-evidence
|
|
95
|
+
# rate bounds are relative to, and the bounds; the unvalidated history
|
|
96
|
+
# has neither.
|
|
97
|
+
unvalidated = dict(UNVALIDATED_ERA_RECORD)
|
|
98
|
+
unvalidated.update(referenceFamily=None, familyRateBounds=[])
|
|
99
|
+
out = [unvalidated]
|
|
94
100
|
for era in qm.COEFFICIENT_ERAS:
|
|
95
101
|
out.append({
|
|
96
102
|
"id": era.id,
|
|
@@ -99,10 +105,20 @@ def published_coefficient_eras() -> list:
|
|
|
99
105
|
"state": "validated",
|
|
100
106
|
"provenance": era.provenance,
|
|
101
107
|
"tokenClassWeights": era.weight_map(),
|
|
108
|
+
"referenceFamily": era.reference_family,
|
|
109
|
+
"familyRateBounds": _rate_bounds_json(era),
|
|
102
110
|
})
|
|
103
111
|
return out
|
|
104
112
|
|
|
105
113
|
|
|
114
|
+
def _rate_bounds_json(era) -> list:
|
|
115
|
+
"""An era's family rate bounds on the wire, in table order."""
|
|
116
|
+
if era is None:
|
|
117
|
+
return []
|
|
118
|
+
return [{"family": b.family, "low": b.low, "high": b.high,
|
|
119
|
+
"provenance": b.provenance} for b in era.family_rate_bounds]
|
|
120
|
+
|
|
121
|
+
|
|
106
122
|
def select_era(now: "dt.datetime") -> "qm.CoefficientEra | None":
|
|
107
123
|
"""The one era an analysis at `now` is fitted in (#978 spec 4.3).
|
|
108
124
|
|
|
@@ -202,6 +218,12 @@ class LoadResult:
|
|
|
202
218
|
#: through `analyse(extra_statuses=...)` otherwise.
|
|
203
219
|
status: "CalibrationStatus | None" = None
|
|
204
220
|
cause: "str | None" = None
|
|
221
|
+
#: #1015 spec 5.2/5.5: the correction facts and the completed batch ids
|
|
222
|
+
#: (the fit's `basis`), read inside the same coherence bracket, on the
|
|
223
|
+
#: same stats connection and generation as the fit inputs. None facts
|
|
224
|
+
#: mean "not read" (an unusable load).
|
|
225
|
+
correction_facts: object = None
|
|
226
|
+
correction_basis: frozenset = frozenset()
|
|
205
227
|
|
|
206
228
|
|
|
207
229
|
def _account_clause(column: str, account_key):
|
|
@@ -532,6 +554,7 @@ def load_population(account_key, *, since, now, era) -> LoadResult:
|
|
|
532
554
|
stats_conn, account_key, start)
|
|
533
555
|
entries, newest, fast_excluded = _read_cache_component(
|
|
534
556
|
cache_conn, account_key, start, era_end)
|
|
557
|
+
facts = _read_lineage_facts(stats_conn, account_key)
|
|
535
558
|
except sqlite3.Error:
|
|
536
559
|
return LoadResult(analysis_start=start,
|
|
537
560
|
status=CalibrationStatus.UNAVAILABLE,
|
|
@@ -554,8 +577,15 @@ def load_population(account_key, *, since, now, era) -> LoadResult:
|
|
|
554
577
|
return LoadResult(analysis_start=start,
|
|
555
578
|
status=CalibrationStatus.UNAVAILABLE,
|
|
556
579
|
cause="index-not-current")
|
|
580
|
+
if facts is None:
|
|
581
|
+
# A current index whose correction projections cannot be
|
|
582
|
+
# read (a damaged index): a fit published now could not
|
|
583
|
+
# name the corrections its inputs reflect (#1015).
|
|
584
|
+
return LoadResult(analysis_start=start,
|
|
585
|
+
status=CalibrationStatus.UNAVAILABLE,
|
|
586
|
+
cause="store-unavailable")
|
|
557
587
|
payload = (earliest, snapshots, credits, diagnostics,
|
|
558
|
-
entries, newest, fast_excluded)
|
|
588
|
+
entries, newest, fast_excluded, facts)
|
|
559
589
|
break
|
|
560
590
|
if payload is None:
|
|
561
591
|
# A store that moves twice while we read it is under active
|
|
@@ -574,7 +604,7 @@ def load_population(account_key, *, since, now, era) -> LoadResult:
|
|
|
574
604
|
pass
|
|
575
605
|
|
|
576
606
|
earliest, snapshots, credits, diagnostics, entries, newest, \
|
|
577
|
-
fast_excluded = payload
|
|
607
|
+
fast_excluded, facts = payload
|
|
578
608
|
snapshots = _confine(snapshots, era_end)
|
|
579
609
|
credits = _confine(credits, era_end)
|
|
580
610
|
entries = _confine(entries, era_end)
|
|
@@ -591,11 +621,21 @@ def load_population(account_key, *, since, now, era) -> LoadResult:
|
|
|
591
621
|
analysis_start=start, newest_entry_at=newest,
|
|
592
622
|
diagnostics=diagnostics,
|
|
593
623
|
status=CalibrationStatus.UNVALIDATED_COEFFICIENT_ERA,
|
|
594
|
-
cause="history-predates-supported-composition"
|
|
624
|
+
cause="history-predates-supported-composition",
|
|
625
|
+
correction_facts=facts, correction_basis=facts.basis)
|
|
595
626
|
return LoadResult(
|
|
596
627
|
entries=tuple(entries), snapshots=tuple(snapshots),
|
|
597
628
|
credits=tuple(credits), newest_entry_at=newest,
|
|
598
|
-
analysis_start=start, diagnostics=diagnostics
|
|
629
|
+
analysis_start=start, diagnostics=diagnostics,
|
|
630
|
+
correction_facts=facts, correction_basis=facts.basis)
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def _read_lineage_facts(stats_conn, account_key):
|
|
634
|
+
"""The correction facts for one account's scope, on the bracket's own
|
|
635
|
+
stats connection (#1015 spec 5.2), or None when they cannot be read."""
|
|
636
|
+
import _cctally_quota_calibration
|
|
637
|
+
return _cctally_quota_calibration.read_correction_facts(
|
|
638
|
+
stats_conn, account_scope=_state_key(account_key))
|
|
599
639
|
|
|
600
640
|
|
|
601
641
|
# ---------------------------------------------------------------------------
|
|
@@ -1062,6 +1102,28 @@ def claim_delivery(identity) -> str:
|
|
|
1062
1102
|
return d.CLAIM_WON
|
|
1063
1103
|
|
|
1064
1104
|
|
|
1105
|
+
def release_delivery_claim(identity) -> bool:
|
|
1106
|
+
"""Hand back this run's own won claim for `identity`, whose notification
|
|
1107
|
+
was NOT handed off (#1015 spec 5.7: the supersession recheck could not
|
|
1108
|
+
take the stats maintenance read guard). The identity is owed again, so
|
|
1109
|
+
the owed sweep of a later run retries it. False when the ledger could
|
|
1110
|
+
not be rewritten; the claim then stands, exactly as a crash after the
|
|
1111
|
+
mark would leave it."""
|
|
1112
|
+
d = _rate_change_delivery()
|
|
1113
|
+
with delivery_lock():
|
|
1114
|
+
loaded = load_delivery_state()
|
|
1115
|
+
if not loaded.usable:
|
|
1116
|
+
return False
|
|
1117
|
+
if tuple(identity) not in d.decided_set(loaded.state):
|
|
1118
|
+
return True
|
|
1119
|
+
try:
|
|
1120
|
+
save_delivery_state(d.without_decided(loaded.state,
|
|
1121
|
+
[tuple(identity)]))
|
|
1122
|
+
except OSError:
|
|
1123
|
+
return False
|
|
1124
|
+
return True
|
|
1125
|
+
|
|
1126
|
+
|
|
1065
1127
|
def seed_delivery_state() -> LoadedDelivery:
|
|
1066
1128
|
"""Mark every currently recorded identity as decided, once (#695 sec 4).
|
|
1067
1129
|
|
|
@@ -1124,6 +1186,28 @@ class PersistMode:
|
|
|
1124
1186
|
#: every regime. None when the clock precedes every validated era, which
|
|
1125
1187
|
#: makes the run persist nothing.
|
|
1126
1188
|
coefficient_era: "str | None"
|
|
1189
|
+
#: #1015 spec 5.5: the correction facts read inside `load_population`'s
|
|
1190
|
+
#: coherence bracket, which `reduce_state` assesses every stored regime
|
|
1191
|
+
#: against, and the completed batch ids a fit written by this run
|
|
1192
|
+
#: records as its `basis`. None facts mean "not assessed": a constructed
|
|
1193
|
+
#: mode keeps the pre-#1015 reduction (lineage is still written).
|
|
1194
|
+
correction_facts: object = None
|
|
1195
|
+
correction_basis: frozenset = frozenset()
|
|
1196
|
+
|
|
1197
|
+
|
|
1198
|
+
def persist_mode(result, *, now) -> PersistMode:
|
|
1199
|
+
"""The `PersistMode` one account's sweep reduces under (#1015 spec 5.5).
|
|
1200
|
+
|
|
1201
|
+
The correction facts and the basis come from the account's own
|
|
1202
|
+
`LoadResult`, so they describe the generation the fit was computed from
|
|
1203
|
+
and are never read again after fitting."""
|
|
1204
|
+
return PersistMode(
|
|
1205
|
+
kind=result.mode_kind, account_key=result.account_key, now=now,
|
|
1206
|
+
fingerprint=qm.QUOTA_MODEL_CONSTANTS_FINGERPRINT,
|
|
1207
|
+
regime_start=result.load.analysis_start,
|
|
1208
|
+
coefficient_era=None if result.era is None else result.era.id,
|
|
1209
|
+
correction_facts=result.load.correction_facts,
|
|
1210
|
+
correction_basis=frozenset(result.load.correction_basis or ()))
|
|
1127
1211
|
|
|
1128
1212
|
|
|
1129
1213
|
def _interval_json(interval):
|
|
@@ -1151,12 +1235,20 @@ def _regime_from(evidence, *, effective_from, mode, status):
|
|
|
1151
1235
|
}
|
|
1152
1236
|
|
|
1153
1237
|
|
|
1154
|
-
def _decorate(regime, analysis):
|
|
1155
|
-
"""Attach the composition the regime was fitted under.
|
|
1238
|
+
def _decorate(regime, analysis, *, fit_family_shares):
|
|
1239
|
+
"""Attach the composition the regime was fitted under.
|
|
1240
|
+
|
|
1241
|
+
`fit_family_shares` is the fit-weighted family composition of the fit
|
|
1242
|
+
THIS regime records (#991 spec 4.5): the baseline q for a baseline
|
|
1243
|
+
regime, the published q for a successor or a no-split regime. A
|
|
1244
|
+
predecessor is never decorated again, so it keeps what it stored.
|
|
1245
|
+
"""
|
|
1156
1246
|
regime["familyShares"] = dict(analysis.family_shares)
|
|
1157
1247
|
regime["classShares"] = dict(analysis.class_shares)
|
|
1158
1248
|
regime["familyRadius"] = analysis.family_radius
|
|
1159
1249
|
regime["classRadius"] = analysis.class_radius
|
|
1250
|
+
regime["fitFamilyShares"] = (None if fit_family_shares is None
|
|
1251
|
+
else dict(fit_family_shares))
|
|
1160
1252
|
return regime
|
|
1161
1253
|
|
|
1162
1254
|
|
|
@@ -1222,6 +1314,18 @@ _REGIME_REVISION_KEY = "wpr"
|
|
|
1222
1314
|
_REGIME_REVISION_DIGEST_KEY = "wprDigest"
|
|
1223
1315
|
_REGIME_WRITER_KEYS = (_REGIME_WRITER_KEY, _REGIME_WRITER_DIGEST_KEY,
|
|
1224
1316
|
_REGIME_REVISION_KEY, _REGIME_REVISION_DIGEST_KEY)
|
|
1317
|
+
# #1019/#1014 spec 4.7 C5: and revision-3 provenance. 1.116.1 writes a regime
|
|
1318
|
+
# whose `wpg` and `wpr` digests verify, so every regime this binary writes
|
|
1319
|
+
# also carries `wpx` and a revision-3 digest over its content without any of
|
|
1320
|
+
# the six writer keys. 1.116.1 strips only its four keys when it stamps, so
|
|
1321
|
+
# its in-place `update()` keeps a stale `wpxDigest`, and a regime it created
|
|
1322
|
+
# has none; neither is recovered as current-writer state. The `wpr` and `wpg`
|
|
1323
|
+
# digests are computed as before (over the `wpx` keys too), so 1.116.1 still
|
|
1324
|
+
# verifies this binary's stamps.
|
|
1325
|
+
_REGIME_SUCCESSOR_KEY = "wpx"
|
|
1326
|
+
_REGIME_SUCCESSOR_DIGEST_KEY = "wpxDigest"
|
|
1327
|
+
_REGIME_ALL_WRITER_KEYS = _REGIME_WRITER_KEYS + (
|
|
1328
|
+
_REGIME_SUCCESSOR_KEY, _REGIME_SUCCESSOR_DIGEST_KEY)
|
|
1225
1329
|
|
|
1226
1330
|
|
|
1227
1331
|
def _regime_content_digest(regime: dict) -> str:
|
|
@@ -1245,11 +1349,26 @@ def _regime_revision_digest(regime: dict) -> str:
|
|
|
1245
1349
|
return "sha256:" + hashlib.sha256(data.encode("utf-8")).hexdigest()
|
|
1246
1350
|
|
|
1247
1351
|
|
|
1352
|
+
def _regime_successor_digest(regime: dict) -> str:
|
|
1353
|
+
import _lib_account_protocol
|
|
1354
|
+
core = {k: v for k, v in regime.items()
|
|
1355
|
+
if k not in _REGIME_ALL_WRITER_KEYS}
|
|
1356
|
+
data = json.dumps(
|
|
1357
|
+
{"revision": _lib_account_protocol.SUCCESSOR_REVISION,
|
|
1358
|
+
"regime": core},
|
|
1359
|
+
sort_keys=True, separators=(",", ":"), ensure_ascii=False,
|
|
1360
|
+
default=str)
|
|
1361
|
+
return "sha256:" + hashlib.sha256(data.encode("utf-8")).hexdigest()
|
|
1362
|
+
|
|
1363
|
+
|
|
1248
1364
|
def stamp_regime(regime: dict) -> dict:
|
|
1249
|
-
"""``regime`` stamped as written by the current protocol generation
|
|
1250
|
-
revision."""
|
|
1365
|
+
"""``regime`` stamped as written by the current protocol generation,
|
|
1366
|
+
revision and successor revision."""
|
|
1251
1367
|
import _lib_account_protocol
|
|
1252
|
-
out = {k: v for k, v in regime.items()
|
|
1368
|
+
out = {k: v for k, v in regime.items()
|
|
1369
|
+
if k not in _REGIME_ALL_WRITER_KEYS}
|
|
1370
|
+
out[_REGIME_SUCCESSOR_KEY] = _lib_account_protocol.SUCCESSOR_REVISION
|
|
1371
|
+
out[_REGIME_SUCCESSOR_DIGEST_KEY] = _regime_successor_digest(out)
|
|
1253
1372
|
out[_REGIME_REVISION_KEY] = _lib_account_protocol.PROTOCOL_REVISION
|
|
1254
1373
|
out[_REGIME_REVISION_DIGEST_KEY] = _regime_revision_digest(out)
|
|
1255
1374
|
out[_REGIME_WRITER_KEY] = _lib_account_protocol.PROTOCOL_GENERATION
|
|
@@ -1268,7 +1387,12 @@ def regime_writer_is_current(regime) -> bool:
|
|
|
1268
1387
|
{_lib_account_protocol.REVISION_KEY:
|
|
1269
1388
|
regime.get(_REGIME_REVISION_KEY)})
|
|
1270
1389
|
and regime.get(_REGIME_REVISION_DIGEST_KEY)
|
|
1271
|
-
== _regime_revision_digest(regime)
|
|
1390
|
+
== _regime_revision_digest(regime)
|
|
1391
|
+
and _lib_account_protocol.has_successor_revision(
|
|
1392
|
+
{_lib_account_protocol.SUCCESSOR_REVISION_KEY:
|
|
1393
|
+
regime.get(_REGIME_SUCCESSOR_KEY)})
|
|
1394
|
+
and regime.get(_REGIME_SUCCESSOR_DIGEST_KEY)
|
|
1395
|
+
== _regime_successor_digest(regime))
|
|
1272
1396
|
|
|
1273
1397
|
|
|
1274
1398
|
def stamp_current_writes(stored: dict, reduced: dict, account_key) -> dict:
|
|
@@ -1293,25 +1417,100 @@ def stamp_current_writes(stored: dict, reduced: dict, account_key) -> dict:
|
|
|
1293
1417
|
return out
|
|
1294
1418
|
|
|
1295
1419
|
|
|
1296
|
-
def recovery_candidates(state: dict, account_key, *, detected_at
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1420
|
+
def recovery_candidates(state: dict, account_key, *, detected_at,
|
|
1421
|
+
facts=None) -> tuple:
|
|
1422
|
+
"""The persisted transitions #689 recovery may journal.
|
|
1423
|
+
|
|
1424
|
+
Every qualified pair of CLEAN regimes (#1015 spec 5.6) whose SUCCESSOR —
|
|
1425
|
+
the regime that constitutes the transition — carries a verifying
|
|
1426
|
+
current-writer stamp AND a verifying frozen `transitionLineage`. A
|
|
1427
|
+
transition a previous-release process persisted (or edited in place) is
|
|
1428
|
+
withheld. The predecessor is ordinary history: every regime from before
|
|
1429
|
+
the upgrade is legacy-written, and the first current change after it must
|
|
1430
|
+
still be recoverable.
|
|
1431
|
+
|
|
1432
|
+
#1015 spec 5.6/5.7: recovery trusts the transition's own lineage, never
|
|
1433
|
+
the successor's `fitLineage` (a later no-change refit replaces that). An
|
|
1434
|
+
identity that lineage finds superseded under `facts` is not recovered,
|
|
1435
|
+
so a refit's newer basis can never rehabilitate an earlier detection.
|
|
1436
|
+
What is returned is the frozen DESCRIPTOR, so the recovered event's
|
|
1437
|
+
numbers (rates, severity, evidence), windows and basis are the
|
|
1438
|
+
detection's own. A writer-verified successor with NO `transitionLineage`
|
|
1439
|
+
(written by a release before #1015) is recovered as a legacy transition
|
|
1440
|
+
instead: from the regime pair, unless a correction up to this run's clock
|
|
1441
|
+
supersedes it. `detected_at` is still the recovery clock, as #689's
|
|
1442
|
+
contract states for a recovered row; the detection's own clock stays in
|
|
1443
|
+
the frozen lineage the event journals.
|
|
1444
|
+
"""
|
|
1303
1445
|
mrc = _cctally()._load_sibling("_lib_meter_rate_change")
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1446
|
+
ql = _lineage_kernel()
|
|
1447
|
+
import _cctally_quota_calibration as qcg
|
|
1448
|
+
import _lib_accounts
|
|
1449
|
+
scope = _state_key(account_key)
|
|
1450
|
+
alert_key = _alert_account_key(account_key)
|
|
1451
|
+
regimes = clean_regimes(stored_regimes(state, account_key), facts, scope)
|
|
1452
|
+
out = []
|
|
1453
|
+
legacy_starts = set()
|
|
1454
|
+
pairs = mrc._adjacent_pairs(regimes)
|
|
1455
|
+
for start in sorted(pairs):
|
|
1456
|
+
_predecessor, successor = pairs[start]
|
|
1457
|
+
if not regime_writer_is_current(successor):
|
|
1458
|
+
continue
|
|
1459
|
+
lineage = successor.get(ql.TRANSITION_LINEAGE_KEY)
|
|
1460
|
+
if lineage is None:
|
|
1461
|
+
# F8: a writer-verified successor a release before #1015 wrote
|
|
1462
|
+
# (#1019 stamps, no frozen lineage) recovers as a legacy
|
|
1463
|
+
# transition (spec 5.7): basis [] and a window ending at this
|
|
1464
|
+
# recovery run's clock, so any earlier correction in its scope
|
|
1465
|
+
# supersedes it; its values come from the regime pair, exactly
|
|
1466
|
+
# as recovery read them before #1015.
|
|
1467
|
+
legacy = ql.legacy_transition_lineage(
|
|
1468
|
+
account_key=alert_key, detected_at=detected_at,
|
|
1469
|
+
unattributed=_lib_accounts.UNATTRIBUTED)
|
|
1470
|
+
if facts is None or ql.assess(
|
|
1471
|
+
legacy, facts,
|
|
1472
|
+
coord_key=qcg._coordinate_order).state == "clean":
|
|
1473
|
+
legacy_starts.add(start)
|
|
1474
|
+
continue
|
|
1475
|
+
if not ql.transition_lineage_verifies(lineage):
|
|
1476
|
+
continue
|
|
1477
|
+
if facts is not None and ql.assess(
|
|
1478
|
+
lineage, facts,
|
|
1479
|
+
coord_key=qcg._coordinate_order).state != "clean":
|
|
1480
|
+
continue
|
|
1481
|
+
transition = _transition_from_descriptor(lineage, detected_at)
|
|
1482
|
+
if transition is not None:
|
|
1483
|
+
out.append(transition)
|
|
1484
|
+
if legacy_starts:
|
|
1485
|
+
out.extend(
|
|
1486
|
+
t for t in mrc.enumerate_transitions(
|
|
1487
|
+
regimes, provider="claude", account_key=alert_key,
|
|
1488
|
+
detected_at=detected_at)
|
|
1489
|
+
if mrc._instant(t.effective_from) in legacy_starts)
|
|
1490
|
+
out.sort(key=lambda t: mrc._instant(t.effective_from))
|
|
1491
|
+
return tuple(out)
|
|
1492
|
+
|
|
1493
|
+
|
|
1494
|
+
def _transition_from_descriptor(lineage, detected_at):
|
|
1495
|
+
"""The `RateChangeTransition` a frozen `transitionLineage` describes,
|
|
1496
|
+
stamped at `detected_at`, or None when its descriptor cannot be read."""
|
|
1497
|
+
mrc = _cctally()._load_sibling("_lib_meter_rate_change")
|
|
1498
|
+
d = lineage.get("descriptor") or {}
|
|
1499
|
+
try:
|
|
1500
|
+
days = d.get("baseline_withheld_days")
|
|
1501
|
+
return mrc.RateChangeTransition(
|
|
1502
|
+
provider=str(d["provider"]), account_key=str(d["account_key"]),
|
|
1503
|
+
effective_from=str(d["effective_from"]),
|
|
1504
|
+
previous_units_per_point=float(d["previous_units_per_point"]),
|
|
1505
|
+
new_units_per_point=float(d["new_units_per_point"]),
|
|
1506
|
+
severity=str(d["severity"]), detected_at=str(detected_at),
|
|
1507
|
+
withholding_status=d.get("withholding_status"),
|
|
1508
|
+
detector_input_causes=d.get("detector_input_causes"),
|
|
1509
|
+
composition_provenance=d.get("composition_provenance"),
|
|
1510
|
+
baseline_withheld_days=None if days is None else int(days),
|
|
1511
|
+
fit_lineage=dict(lineage))
|
|
1512
|
+
except (KeyError, TypeError, ValueError):
|
|
1513
|
+
return None
|
|
1315
1514
|
|
|
1316
1515
|
|
|
1317
1516
|
def recorded_regime(state: dict, account_key) -> "dict | None":
|
|
@@ -1442,6 +1641,35 @@ def reduce_state(stored: dict, analysis, mode: PersistMode) -> dict:
|
|
|
1442
1641
|
return stored
|
|
1443
1642
|
key = _state_key(mode.account_key)
|
|
1444
1643
|
regimes = stored_regimes(stored, mode.account_key)
|
|
1644
|
+
stored_bucket = (stored.get("accounts") or {}).get(key)
|
|
1645
|
+
bucket = dict(stored_bucket) if isinstance(stored_bucket, dict) else {}
|
|
1646
|
+
archive = [dict(a) for a in bucket.get(_ARCHIVE_KEY) or ()
|
|
1647
|
+
if isinstance(a, dict)]
|
|
1648
|
+
now_iso = mode.now.astimezone(UTC).isoformat()
|
|
1649
|
+
facts = mode.correction_facts
|
|
1650
|
+
# #1015 spec 5.5 item 1: every stored regime is assessed against the
|
|
1651
|
+
# facts, over the state this reducer was handed (the locked re-read), and
|
|
1652
|
+
# a tainted one is annotated whatever else this run does.
|
|
1653
|
+
assessments = _assess_regimes(regimes, facts, key)
|
|
1654
|
+
annotated = _annotate(regimes, assessments, now_iso)
|
|
1655
|
+
tainted = {id(r) for r, a in zip(regimes, assessments)
|
|
1656
|
+
if not _is_clean(r, a)}
|
|
1657
|
+
before_clean = [dict(r) for r in regimes if id(r) not in tainted]
|
|
1658
|
+
|
|
1659
|
+
def finish(changed: bool) -> dict:
|
|
1660
|
+
if not changed:
|
|
1661
|
+
return stored
|
|
1662
|
+
out_bucket = dict(bucket)
|
|
1663
|
+
out_bucket["regimes"] = regimes
|
|
1664
|
+
if archive:
|
|
1665
|
+
out_bucket[_ARCHIVE_KEY] = archive
|
|
1666
|
+
if out_bucket == stored_bucket:
|
|
1667
|
+
return stored
|
|
1668
|
+
accounts = dict(stored.get("accounts") or {})
|
|
1669
|
+
accounts[key] = out_bucket
|
|
1670
|
+
return {"schemaVersion": CALIBRATION_STATE_SCHEMA_VERSION,
|
|
1671
|
+
"accounts": accounts}
|
|
1672
|
+
|
|
1445
1673
|
confirmed = (analysis.verdict is Verdict.RATE_CHANGE_DETECTED
|
|
1446
1674
|
and analysis.detector.split_date is not None)
|
|
1447
1675
|
fitted_ok = (analysis.status is CalibrationStatus.OK
|
|
@@ -1455,21 +1683,27 @@ def reduce_state(stored: dict, analysis, mode: PersistMode) -> dict:
|
|
|
1455
1683
|
permitted = _transition_persistence_admitted(analysis)
|
|
1456
1684
|
confirmed = confirmed or permitted
|
|
1457
1685
|
successor_ok = successor_ok or permitted
|
|
1686
|
+
open_regime = _open_regime(regimes)
|
|
1458
1687
|
if not fitted_ok and not successor_ok:
|
|
1459
|
-
|
|
1688
|
+
# #1015 spec 5.5 item 2: a withheld analysis over a tainted open
|
|
1689
|
+
# regime does not close it; it records the attempt, so the refusal
|
|
1690
|
+
# can state why no recalibration happened.
|
|
1691
|
+
attempted = (facts is not None and open_regime is not None
|
|
1692
|
+
and id(open_regime) in tainted
|
|
1693
|
+
and _record_attempt(open_regime, analysis, mode))
|
|
1694
|
+
return finish(annotated or attempted)
|
|
1460
1695
|
mode_era = qm.era_by_id(mode.coefficient_era)
|
|
1461
1696
|
if mode_era is None:
|
|
1462
1697
|
# No validated era: nothing this run fitted could name one.
|
|
1463
|
-
return
|
|
1698
|
+
return finish(annotated)
|
|
1464
1699
|
|
|
1465
|
-
open_regime = _open_regime(regimes)
|
|
1466
1700
|
# #978 spec 4.5: a run whose clock selects an EARLIER era than the open
|
|
1467
1701
|
# regime's — a `CCTALLY_AS_OF` inside the Opus 5 era against a stored
|
|
1468
1702
|
# Opus 5.5 calibration — never replaces the present with the past.
|
|
1469
1703
|
open_era = (qm.era_by_id(open_regime.get("coefficientEra"))
|
|
1470
1704
|
if open_regime is not None else None)
|
|
1471
1705
|
if open_era is not None and open_era.start > mode_era.start:
|
|
1472
|
-
return
|
|
1706
|
+
return finish(annotated)
|
|
1473
1707
|
# A stored regime fitted under different constants, or in a different
|
|
1474
1708
|
# coefficient era (#978), is marked stale and left otherwise untouched.
|
|
1475
1709
|
# It is marked HERE, together with the append below, because section 20
|
|
@@ -1496,28 +1730,36 @@ def reduce_state(stored: dict, analysis, mode: PersistMode) -> dict:
|
|
|
1496
1730
|
if open_regime is not None else None)
|
|
1497
1731
|
predecessor = (open_regime if open_from is not None
|
|
1498
1732
|
and open_from < boundary else None)
|
|
1499
|
-
|
|
1733
|
+
baseline_available = analysis.baseline_fit.state == "available"
|
|
1734
|
+
if predecessor is not None and id(predecessor) in tainted:
|
|
1735
|
+
# #1015 spec 5.5 item 4: a tainted open regime is closed at the
|
|
1736
|
+
# boundary with its status unchanged and is never a comparison
|
|
1737
|
+
# predecessor. This run's own clean baseline is persisted beside
|
|
1738
|
+
# it, so the successor's predecessor is a current-run fit.
|
|
1739
|
+
predecessor["effectiveUntil"] = boundary_iso
|
|
1740
|
+
if baseline_available:
|
|
1741
|
+
regimes.append(_baseline_regime(analysis, mode, boundary))
|
|
1742
|
+
elif predecessor is not None:
|
|
1500
1743
|
predecessor["effectiveUntil"] = boundary_iso
|
|
1501
1744
|
predecessor["status"] = CalibrationStatus.OK.value
|
|
1502
|
-
elif open_regime is None and
|
|
1745
|
+
elif open_regime is None and baseline_available:
|
|
1503
1746
|
# The first run that discovers a split persists both sides, so a
|
|
1504
1747
|
# store with no prior regime still records what the rate WAS.
|
|
1505
|
-
|
|
1506
|
-
analysis.baseline_fit, effective_from=mode.regime_start,
|
|
1507
|
-
mode=mode, status=CalibrationStatus.OK.value), analysis)
|
|
1508
|
-
baseline["effectiveUntil"] = boundary_iso
|
|
1509
|
-
regimes.append(baseline)
|
|
1748
|
+
regimes.append(_baseline_regime(analysis, mode, boundary))
|
|
1510
1749
|
if successor_ok:
|
|
1511
1750
|
successor = _decorate(_regime_from(
|
|
1512
1751
|
analysis.watch_fit, effective_from=boundary, mode=mode,
|
|
1513
|
-
status=analysis.status.value), analysis
|
|
1752
|
+
status=analysis.status.value), analysis,
|
|
1753
|
+
fit_family_shares=analysis.fit_family_shares)
|
|
1514
1754
|
existing = next((r for r in regimes
|
|
1515
1755
|
if r.get("effectiveFrom") == boundary_iso
|
|
1516
1756
|
and r.get("effectiveUntil") is None), None)
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1757
|
+
target = _place_fit(regimes, existing, successor, tainted,
|
|
1758
|
+
archive, now_iso)
|
|
1759
|
+
_attach_fit_lineage(
|
|
1760
|
+
target, mode, "successor", s=boundary,
|
|
1761
|
+
estimator=_estimator(
|
|
1762
|
+
getattr(analysis, "published_day_bounds", None)))
|
|
1521
1763
|
elif fitted_ok:
|
|
1522
1764
|
fresh = _decorate(_regime_from(
|
|
1523
1765
|
analysis.fitted,
|
|
@@ -1525,18 +1767,236 @@ def reduce_state(stored: dict, analysis, mode: PersistMode) -> dict:
|
|
|
1525
1767
|
parse_instant(open_regime["effectiveFrom"])
|
|
1526
1768
|
if open_regime is not None
|
|
1527
1769
|
and open_regime.get("effectiveFrom") else mode.regime_start),
|
|
1528
|
-
mode=mode, status=CalibrationStatus.OK.value), analysis
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1770
|
+
mode=mode, status=CalibrationStatus.OK.value), analysis,
|
|
1771
|
+
fit_family_shares=analysis.fit_family_shares)
|
|
1772
|
+
target = _place_fit(regimes, open_regime, fresh, tainted, archive,
|
|
1773
|
+
now_iso)
|
|
1774
|
+
_attach_fit_lineage(
|
|
1775
|
+
target, mode, "nosplit",
|
|
1776
|
+
estimator=_estimator(
|
|
1777
|
+
getattr(analysis, "published_day_bounds", None)))
|
|
1533
1778
|
else:
|
|
1534
|
-
return
|
|
1779
|
+
return finish(annotated)
|
|
1780
|
+
|
|
1781
|
+
_attach_transition_lineage(before_clean, regimes, analysis, mode)
|
|
1782
|
+
return finish(True)
|
|
1535
1783
|
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1784
|
+
|
|
1785
|
+
# ---------------------------------------------------------------------------
|
|
1786
|
+
# #1015: correction-to-fit lineage in the reducer (spec 5.2, 5.5, 5.6).
|
|
1787
|
+
# ---------------------------------------------------------------------------
|
|
1788
|
+
_ANNOTATION_KEY = "supersededByCorrection"
|
|
1789
|
+
_ATTEMPT_KEY = "recalibrationAttempt"
|
|
1790
|
+
_ARCHIVE_KEY = "archivedFits"
|
|
1791
|
+
|
|
1792
|
+
|
|
1793
|
+
def _lineage_kernel():
|
|
1794
|
+
import _lib_quota_lineage
|
|
1795
|
+
return _lib_quota_lineage
|
|
1796
|
+
|
|
1797
|
+
|
|
1798
|
+
def _assess_regimes(regimes, facts, scope) -> list:
|
|
1799
|
+
"""One assessment per regime, or None for each when `facts` is None
|
|
1800
|
+
(a constructed mode that does not assess)."""
|
|
1801
|
+
if facts is None:
|
|
1802
|
+
return [None] * len(regimes)
|
|
1803
|
+
import _cctally_quota_calibration as qcg
|
|
1804
|
+
return [qcg.assess_regime(r, facts, scope=scope) for r in regimes]
|
|
1805
|
+
|
|
1806
|
+
|
|
1807
|
+
def _is_clean(regime, assessment) -> bool:
|
|
1808
|
+
"""Spec 5.6: only a regime assessed `clean` and never annotated takes
|
|
1809
|
+
part in pairing, detection, recovery and the marker. The annotation is
|
|
1810
|
+
terminal: a fit's basis never grows, so a tainted fit stays tainted."""
|
|
1811
|
+
if isinstance(regime.get(_ANNOTATION_KEY), dict):
|
|
1812
|
+
return False
|
|
1813
|
+
return assessment is None or assessment.state == "clean"
|
|
1814
|
+
|
|
1815
|
+
|
|
1816
|
+
def clean_regimes(regimes, facts, scope) -> list:
|
|
1817
|
+
"""`regimes` without every tainted, unavailable or annotated regime."""
|
|
1818
|
+
return [r for r, a in zip(regimes, _assess_regimes(regimes, facts, scope))
|
|
1819
|
+
if _is_clean(r, a)]
|
|
1820
|
+
|
|
1821
|
+
|
|
1822
|
+
def _correction_json(corrections) -> list:
|
|
1823
|
+
return [{"opId": str(op), "observations": int(count)}
|
|
1824
|
+
for op, count in corrections]
|
|
1825
|
+
|
|
1826
|
+
|
|
1827
|
+
def _annotate(regimes, assessments, now_iso) -> bool:
|
|
1828
|
+
"""Spec 5.5 item 1. True when an annotation was added."""
|
|
1829
|
+
changed = False
|
|
1830
|
+
for regime, assessment in zip(regimes, assessments):
|
|
1831
|
+
if assessment is None or assessment.state != "tainted":
|
|
1832
|
+
continue
|
|
1833
|
+
if isinstance(regime.get(_ANNOTATION_KEY), dict):
|
|
1834
|
+
continue
|
|
1835
|
+
lineage = regime.get(_lineage_kernel().FIT_LINEAGE_KEY)
|
|
1836
|
+
fit_id = (lineage.get("fitId") if isinstance(lineage, dict)
|
|
1837
|
+
and _lineage_kernel().fit_lineage_verifies(regime)
|
|
1838
|
+
else _legacy_fit_id(regime))
|
|
1839
|
+
regime[_ANNOTATION_KEY] = {
|
|
1840
|
+
"fitId": fit_id,
|
|
1841
|
+
"corrections": _correction_json(assessment.corrections),
|
|
1842
|
+
"assessedAt": now_iso}
|
|
1843
|
+
changed = True
|
|
1844
|
+
return changed
|
|
1845
|
+
|
|
1846
|
+
|
|
1847
|
+
def _legacy_fit_id(regime) -> str:
|
|
1848
|
+
ql = _lineage_kernel()
|
|
1849
|
+
return ql.legacy_lineage(regime, None)["fitId"]
|
|
1850
|
+
|
|
1851
|
+
|
|
1852
|
+
def _record_attempt(regime, analysis, mode) -> bool:
|
|
1853
|
+
"""Spec 5.5 item 2. True when the attempt record changed. An attempt
|
|
1854
|
+
with the same status over the same basis is the same attempt, so a
|
|
1855
|
+
repeated withheld sweep leaves the record (and the file) unchanged."""
|
|
1856
|
+
status = getattr(getattr(analysis, "status", None), "value", None)
|
|
1857
|
+
basis = sorted(str(b) for b in mode.correction_basis)
|
|
1858
|
+
prior = regime.get(_ATTEMPT_KEY)
|
|
1859
|
+
if isinstance(prior, dict) and prior.get("status") == status \
|
|
1860
|
+
and prior.get("basis") == basis:
|
|
1861
|
+
return False
|
|
1862
|
+
regime[_ATTEMPT_KEY] = {"status": status, "basis": basis,
|
|
1863
|
+
"asOf": mode.now.astimezone(UTC).isoformat()}
|
|
1864
|
+
return True
|
|
1865
|
+
|
|
1866
|
+
|
|
1867
|
+
def _place_fit(regimes, target, fresh, tainted, archive, now_iso) -> dict:
|
|
1868
|
+
"""Write one fresh fit over `target` (or append it) and return the
|
|
1869
|
+
record that now holds it.
|
|
1870
|
+
|
|
1871
|
+
A clean target is updated in place, as before #1015. A tainted one is
|
|
1872
|
+
first archived whole, with its annotation, outside `regimes` so it can
|
|
1873
|
+
never form an adjacent pair, and is then REPLACED at the same
|
|
1874
|
+
`effectiveFrom` with no annotation and no attempt (spec 5.5 item 3).
|
|
1875
|
+
`transitionLineage` is immutable and survives either way (spec 5.7).
|
|
1876
|
+
"""
|
|
1877
|
+
if target is None:
|
|
1878
|
+
regimes.append(fresh)
|
|
1879
|
+
return fresh
|
|
1880
|
+
if id(target) in tainted:
|
|
1881
|
+
archive.append({"regime": dict(target), "archivedAt": now_iso})
|
|
1882
|
+
frozen = target.get(_lineage_kernel().TRANSITION_LINEAGE_KEY)
|
|
1883
|
+
target.clear()
|
|
1884
|
+
target.update(fresh)
|
|
1885
|
+
if frozen is not None:
|
|
1886
|
+
target[_lineage_kernel().TRANSITION_LINEAGE_KEY] = frozen
|
|
1887
|
+
return target
|
|
1888
|
+
target.update(fresh)
|
|
1889
|
+
return target
|
|
1890
|
+
|
|
1891
|
+
|
|
1892
|
+
def _baseline_regime(analysis, mode, boundary) -> dict:
|
|
1893
|
+
baseline = _decorate(_regime_from(
|
|
1894
|
+
analysis.baseline_fit, effective_from=mode.regime_start,
|
|
1895
|
+
mode=mode, status=CalibrationStatus.OK.value), analysis,
|
|
1896
|
+
fit_family_shares=analysis.baseline_fit_family_shares)
|
|
1897
|
+
baseline["effectiveUntil"] = boundary.isoformat()
|
|
1898
|
+
_attach_fit_lineage(
|
|
1899
|
+
baseline, mode, "baseline", s=boundary,
|
|
1900
|
+
estimator=_estimator(getattr(analysis, "baseline_day_bounds", None)))
|
|
1901
|
+
return baseline
|
|
1902
|
+
|
|
1903
|
+
|
|
1904
|
+
def _estimator(bounds) -> "dict | None":
|
|
1905
|
+
"""`fitLineage.estimator`: the half-open UTC span of the fit's days."""
|
|
1906
|
+
if not bounds:
|
|
1907
|
+
return None
|
|
1908
|
+
first, last = bounds
|
|
1909
|
+
start = dt.datetime.combine(first, dt.time(0, 0), tzinfo=UTC)
|
|
1910
|
+
until = dt.datetime.combine(last, dt.time(0, 0), tzinfo=UTC) \
|
|
1911
|
+
+ dt.timedelta(days=1)
|
|
1912
|
+
return {"from": start.isoformat(), "until": until.isoformat()}
|
|
1913
|
+
|
|
1914
|
+
|
|
1915
|
+
def _attach_fit_lineage(regime, mode, kind, *, s=None, estimator) -> None:
|
|
1916
|
+
"""Spec 5.2: the immutable `fitLineage` of the fit `regime` now holds,
|
|
1917
|
+
computed over its final content."""
|
|
1918
|
+
ql = _lineage_kernel()
|
|
1919
|
+
regime.pop(ql.FIT_LINEAGE_KEY, None)
|
|
1920
|
+
windows = ql.dependency_windows(kind, a=mode.regime_start, s=s,
|
|
1921
|
+
t=mode.now)
|
|
1922
|
+
regime[ql.FIT_LINEAGE_KEY] = ql.fit_lineage(
|
|
1923
|
+
scope=_state_key(mode.account_key), basis=mode.correction_basis,
|
|
1924
|
+
windows=windows, estimator=estimator, regime=regime)
|
|
1925
|
+
|
|
1926
|
+
|
|
1927
|
+
def _transition_descriptor(transition) -> dict:
|
|
1928
|
+
"""Every value the `meter_rate_change` event journals other than the
|
|
1929
|
+
lineage itself, exactly as detected (spec 5.7)."""
|
|
1930
|
+
return {
|
|
1931
|
+
"provider": transition.provider,
|
|
1932
|
+
"account_key": transition.account_key,
|
|
1933
|
+
"effective_from": transition.effective_from,
|
|
1934
|
+
"previous_units_per_point": transition.previous_units_per_point,
|
|
1935
|
+
"new_units_per_point": transition.new_units_per_point,
|
|
1936
|
+
"severity": transition.severity,
|
|
1937
|
+
"detected_at": transition.detected_at,
|
|
1938
|
+
"withholding_status": transition.withholding_status,
|
|
1939
|
+
"detector_input_causes": transition.detector_input_causes,
|
|
1940
|
+
"composition_provenance": transition.composition_provenance,
|
|
1941
|
+
"baseline_withheld_days": transition.baseline_withheld_days,
|
|
1942
|
+
}
|
|
1943
|
+
|
|
1944
|
+
|
|
1945
|
+
def _stamp_disclosure(transitions, analysis) -> tuple:
|
|
1946
|
+
"""The detection-time disclosure stamped onto fresh transitions (#688,
|
|
1947
|
+
#690, #692), computed once from the analysis."""
|
|
1948
|
+
withheld = (analysis.status.value
|
|
1949
|
+
if _transition_persistence_admitted(analysis) else None)
|
|
1950
|
+
causes, provenance, baseline_days = _analysis_evidence(analysis)
|
|
1951
|
+
return tuple(
|
|
1952
|
+
dataclasses.replace(
|
|
1953
|
+
t, withholding_status=withheld, detector_input_causes=causes,
|
|
1954
|
+
composition_provenance=provenance,
|
|
1955
|
+
baseline_withheld_days=baseline_days)
|
|
1956
|
+
for t in transitions)
|
|
1957
|
+
|
|
1958
|
+
|
|
1959
|
+
def _fresh_transitions(before_clean, after_regimes, analysis, mode,
|
|
1960
|
+
scope) -> tuple:
|
|
1961
|
+
"""The transitions this reduction creates over CLEAN regimes only."""
|
|
1962
|
+
mrc = _cctally()._load_sibling("_lib_meter_rate_change")
|
|
1963
|
+
after_clean = clean_regimes(after_regimes, mode.correction_facts, scope)
|
|
1964
|
+
return _stamp_disclosure(mrc.detect_transitions(
|
|
1965
|
+
before_clean, after_clean, provider="claude",
|
|
1966
|
+
account_key=_alert_account_key(mode.account_key),
|
|
1967
|
+
detected_at=mode.now.astimezone(UTC).isoformat()), analysis), \
|
|
1968
|
+
after_clean
|
|
1969
|
+
|
|
1970
|
+
|
|
1971
|
+
def _attach_transition_lineage(before_clean, regimes, analysis, mode) -> None:
|
|
1972
|
+
"""Spec 5.7: the successor of every transition this reduction creates
|
|
1973
|
+
receives its frozen `transitionLineage` in the same reduction: the union
|
|
1974
|
+
of both sides' dependency windows and the detector coverage `[A, T]`,
|
|
1975
|
+
the detecting analysis's scope and basis, and the descriptor."""
|
|
1976
|
+
ql = _lineage_kernel()
|
|
1977
|
+
mrc = _cctally()._load_sibling("_lib_meter_rate_change")
|
|
1978
|
+
import _cctally_quota_calibration as qcg
|
|
1979
|
+
scope = _state_key(mode.account_key)
|
|
1980
|
+
fresh, after_clean = _fresh_transitions(
|
|
1981
|
+
before_clean, regimes, analysis, mode, scope)
|
|
1982
|
+
if not fresh:
|
|
1983
|
+
return
|
|
1984
|
+
pairs = mrc._adjacent_pairs(after_clean)
|
|
1985
|
+
coverage = ql.window(mode.regime_start, mode.now, until_inclusive=True)
|
|
1986
|
+
for transition in fresh:
|
|
1987
|
+
predecessor, successor = pairs[mrc._instant(transition.effective_from)]
|
|
1988
|
+
if ql.TRANSITION_LINEAGE_KEY in successor:
|
|
1989
|
+
continue
|
|
1990
|
+
windows = list(ql.regime_lineage(
|
|
1991
|
+
predecessor, scope=scope,
|
|
1992
|
+
era_start=qcg._era_start(predecessor))["dependencyWindows"])
|
|
1993
|
+
windows += list(ql.regime_lineage(
|
|
1994
|
+
successor, scope=scope,
|
|
1995
|
+
era_start=qcg._era_start(successor))["dependencyWindows"])
|
|
1996
|
+
windows.append(coverage)
|
|
1997
|
+
successor[ql.TRANSITION_LINEAGE_KEY] = ql.transition_lineage(
|
|
1998
|
+
windows=windows, scope=scope, basis=mode.correction_basis,
|
|
1999
|
+
descriptor=_transition_descriptor(transition))
|
|
1540
2000
|
|
|
1541
2001
|
|
|
1542
2002
|
def reset_calibration(account_key, *, now=None) -> bool:
|
|
@@ -1614,10 +2074,12 @@ def persist_and_detect(analysis, mode: PersistMode) -> tuple:
|
|
|
1614
2074
|
# elements, for the same reason: `cmd_quota` iterates the third
|
|
1615
2075
|
# as well.
|
|
1616
2076
|
return None, (), ()
|
|
1617
|
-
|
|
2077
|
+
scope = _state_key(mode.account_key)
|
|
2078
|
+
# #1015 spec 5.6: the before and after states are filtered to their
|
|
2079
|
+
# CLEAN regimes identically, so a tainted regime never pairs.
|
|
2080
|
+
before = clean_regimes(stored_regimes(loaded.state, mode.account_key),
|
|
2081
|
+
mode.correction_facts, scope)
|
|
1618
2082
|
reduced = reduce_state(loaded.state, analysis, mode)
|
|
1619
|
-
mrc = _cctally()._load_sibling("_lib_meter_rate_change")
|
|
1620
|
-
account_key = _alert_account_key(mode.account_key)
|
|
1621
2083
|
detected_at = mode.now.astimezone(UTC).isoformat()
|
|
1622
2084
|
transitions: tuple = ()
|
|
1623
2085
|
if reduced is not loaded.state:
|
|
@@ -1626,41 +2088,47 @@ def persist_and_detect(analysis, mode: PersistMode) -> tuple:
|
|
|
1626
2088
|
save_calibrations(reduced)
|
|
1627
2089
|
# #688: `detect_transitions` compares two stored states and does
|
|
1628
2090
|
# not know WHY this run was admitted, so the disclosure is
|
|
1629
|
-
# stamped onto what it returns rather than passed into it
|
|
1630
|
-
#
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
#
|
|
1635
|
-
# analysis
|
|
1636
|
-
#
|
|
1637
|
-
#
|
|
1638
|
-
#
|
|
1639
|
-
#
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
transitions = tuple(
|
|
1645
|
-
dataclasses.replace(
|
|
1646
|
-
t, withholding_status=withheld,
|
|
1647
|
-
detector_input_causes=causes,
|
|
1648
|
-
composition_provenance=provenance,
|
|
1649
|
-
baseline_withheld_days=baseline_days)
|
|
1650
|
-
for t in mrc.detect_transitions(
|
|
1651
|
-
before, stored_regimes(reduced, mode.account_key),
|
|
1652
|
-
provider="claude", account_key=account_key,
|
|
1653
|
-
detected_at=detected_at))
|
|
2091
|
+
# stamped onto what it returns rather than passed into it
|
|
2092
|
+
# (`_stamp_disclosure`). #690 / #692: the rest of the disclosure,
|
|
2093
|
+
# captured from the SAME analysis and at the same moment.
|
|
2094
|
+
# `withholding_status` stays gated on admission, because it names
|
|
2095
|
+
# the status a run was withheld under and an ordinary confirmed
|
|
2096
|
+
# change was not withheld at all. The other three are stamped
|
|
2097
|
+
# whenever the analysis publishes them, on both paths.
|
|
2098
|
+
#
|
|
2099
|
+
# #1015 spec 5.7: each fresh transition carries the frozen
|
|
2100
|
+
# `transitionLineage` the reducer wrote on its successor, which
|
|
2101
|
+
# the v2 payload journals as `fit_lineage`.
|
|
2102
|
+
fresh, after_clean = _fresh_transitions(
|
|
2103
|
+
before, stored_regimes(reduced, mode.account_key), analysis,
|
|
2104
|
+
mode, scope)
|
|
2105
|
+
transitions = _with_transition_lineage(fresh, after_clean)
|
|
1654
2106
|
# #689: enumerated UNCONDITIONALLY, and that is the whole point. The
|
|
1655
2107
|
# run that lost its recording already wrote the regimes, so every
|
|
1656
2108
|
# later run finds the state unchanged and takes neither branch above.
|
|
1657
2109
|
# Recovery therefore cannot be derived from freshness; it comes from
|
|
1658
2110
|
# what is persisted right now.
|
|
1659
2111
|
candidates = recovery_candidates(
|
|
1660
|
-
reduced, mode.account_key, detected_at=detected_at
|
|
2112
|
+
reduced, mode.account_key, detected_at=detected_at,
|
|
2113
|
+
facts=mode.correction_facts)
|
|
1661
2114
|
return loaded.quarantined, transitions, candidates
|
|
1662
2115
|
|
|
1663
2116
|
|
|
2117
|
+
def _with_transition_lineage(transitions, regimes) -> tuple:
|
|
2118
|
+
"""Each transition with its successor's frozen `transitionLineage`."""
|
|
2119
|
+
mrc = _cctally()._load_sibling("_lib_meter_rate_change")
|
|
2120
|
+
ql = _lineage_kernel()
|
|
2121
|
+
pairs = mrc._adjacent_pairs(regimes)
|
|
2122
|
+
out = []
|
|
2123
|
+
for transition in transitions:
|
|
2124
|
+
pair = pairs.get(mrc._instant(transition.effective_from))
|
|
2125
|
+
lineage = (pair[1].get(ql.TRANSITION_LINEAGE_KEY)
|
|
2126
|
+
if pair is not None else None)
|
|
2127
|
+
out.append(dataclasses.replace(transition, fit_lineage=lineage)
|
|
2128
|
+
if isinstance(lineage, dict) else transition)
|
|
2129
|
+
return tuple(out)
|
|
2130
|
+
|
|
2131
|
+
|
|
1664
2132
|
#: Keys per presence query. Three bind parameters each, so this stays far
|
|
1665
2133
|
#: below SQLite's variable limit however long the regime list grows.
|
|
1666
2134
|
_PRESENCE_CHUNK = 200
|
|
@@ -1765,6 +2233,25 @@ def unrecorded_rate_change_transitions(transitions) -> tuple:
|
|
|
1765
2233
|
if (str(status) != "active" or int(rev) != 0
|
|
1766
2234
|
or event_json is None):
|
|
1767
2235
|
recorded.add(by_id[str(event_id)])
|
|
2236
|
+
# #1015 spec 5.7: a superseded identity is terminally recorded.
|
|
2237
|
+
# R6: one snapshot, and only the rows each candidate's lineage needs.
|
|
2238
|
+
import _cctally_quota_calibration as qcg
|
|
2239
|
+
began = not conn.in_transaction
|
|
2240
|
+
if began:
|
|
2241
|
+
conn.execute("BEGIN")
|
|
2242
|
+
try:
|
|
2243
|
+
facts = qcg.correction_facts_source(conn)
|
|
2244
|
+
if facts is not None:
|
|
2245
|
+
for t in candidates:
|
|
2246
|
+
if t.identity() not in recorded and \
|
|
2247
|
+
qcg.rate_change_supersession(
|
|
2248
|
+
conn, t.identity(), lineage=t.fit_lineage,
|
|
2249
|
+
detected_at=t.detected_at,
|
|
2250
|
+
facts=facts) is not None:
|
|
2251
|
+
recorded.add(t.identity())
|
|
2252
|
+
finally:
|
|
2253
|
+
if began and conn.in_transaction:
|
|
2254
|
+
conn.execute("ROLLBACK")
|
|
1768
2255
|
return tuple(t for t in candidates if t.identity() not in recorded)
|
|
1769
2256
|
except (Exception, StatsRebuildDeferred): # noqa: BLE001
|
|
1770
2257
|
return ()
|
|
@@ -1886,17 +2373,61 @@ def _settle_rate_change(transition, outcome, status) -> "str | None":
|
|
|
1886
2373
|
if result is None or not result.notification_decided:
|
|
1887
2374
|
return
|
|
1888
2375
|
if not status.usable:
|
|
1889
|
-
|
|
2376
|
+
# An unusable ledger records nothing, so a deferred handoff leaves
|
|
2377
|
+
# the identity owed for the sweep once the ledger is usable again.
|
|
2378
|
+
if deferred and _superseded_at_handoff(transition) is False:
|
|
1890
2379
|
(jr.ALERT_DISPATCHER or jr._dispatch_pending_alerts)(deferred)
|
|
1891
2380
|
return
|
|
1892
2381
|
claim = claim_delivery(transition.identity())
|
|
1893
2382
|
if claim != d.CLAIM_WON:
|
|
1894
2383
|
return claim
|
|
2384
|
+
# #1015 spec 5.7: the ledger is claimed and its leaf lock released
|
|
2385
|
+
# (`claim_delivery` returns from its `with` block), and only then is
|
|
2386
|
+
# supersession rechecked under the stats maintenance read guard. A
|
|
2387
|
+
# superseded identity is settled — the decision stays recorded and is
|
|
2388
|
+
# never cleared to enable a refire — with no notification. A recheck
|
|
2389
|
+
# that cannot take the read guard (a maintenance owner holds it) defers
|
|
2390
|
+
# the handoff: nothing is dispatched, this run's own claim is handed
|
|
2391
|
+
# back, and the identity stays owed for a later run's sweep.
|
|
1895
2392
|
if deferred:
|
|
1896
|
-
|
|
2393
|
+
superseded = _superseded_at_handoff(transition)
|
|
2394
|
+
if superseded is None:
|
|
2395
|
+
release_delivery_claim(transition.identity())
|
|
2396
|
+
return HANDOFF_DEFERRED
|
|
2397
|
+
if not superseded:
|
|
2398
|
+
(jr.ALERT_DISPATCHER or jr._dispatch_pending_alerts)(deferred)
|
|
1897
2399
|
return claim
|
|
1898
2400
|
|
|
1899
2401
|
|
|
2402
|
+
#: `_settle_rate_change`'s outcome when the won claim was handed back
|
|
2403
|
+
#: because the supersession recheck could not take its read guard.
|
|
2404
|
+
HANDOFF_DEFERRED = "handoff_deferred"
|
|
2405
|
+
|
|
2406
|
+
|
|
2407
|
+
def _superseded_at_handoff(transition) -> "bool | None":
|
|
2408
|
+
"""Spec 5.7's last check before a notification is handed off.
|
|
2409
|
+
|
|
2410
|
+
The fail-fast reader takes the stats maintenance flock SHARED and
|
|
2411
|
+
non-blocking, which is the read guard: a destructive maintenance owner
|
|
2412
|
+
is not raced. When that guard (or a published index) is unavailable the
|
|
2413
|
+
answer is None, and the caller defers the handoff rather than
|
|
2414
|
+
dispatching unchecked. Correction facts that cannot be read on an open
|
|
2415
|
+
index answer False — the emitter already refused a superseded identity
|
|
2416
|
+
inside the recording cycle, so this only narrows the window between
|
|
2417
|
+
that commit and the handoff."""
|
|
2418
|
+
import _cctally_quota_calibration as qcg
|
|
2419
|
+
import _cctally_store
|
|
2420
|
+
conn = _cctally_store.stats_open_readonly_failfast()
|
|
2421
|
+
if conn is None:
|
|
2422
|
+
return None
|
|
2423
|
+
try:
|
|
2424
|
+
return qcg.rate_change_supersession(
|
|
2425
|
+
conn, transition.identity(), lineage=transition.fit_lineage,
|
|
2426
|
+
detected_at=transition.detected_at) is not None
|
|
2427
|
+
finally:
|
|
2428
|
+
conn.close()
|
|
2429
|
+
|
|
2430
|
+
|
|
1900
2431
|
def _owed_rate_change_transitions(account_keys, status, *, now,
|
|
1901
2432
|
settled=()) -> tuple:
|
|
1902
2433
|
"""Durable rows with no delivery decision, as descriptors (#695 §5.1).
|
|
@@ -1927,6 +2458,9 @@ def _owed_rate_change_transitions(account_keys, status, *, now,
|
|
|
1927
2458
|
recorded = recorded_rate_change_identities(account_keys)
|
|
1928
2459
|
except (Exception, StatsRebuildDeferred): # noqa: BLE001
|
|
1929
2460
|
return ()
|
|
2461
|
+
# #1015 spec 5.7: a superseded identity is never owed a notification.
|
|
2462
|
+
superseded = _superseded_identities(account_keys)
|
|
2463
|
+
recorded = tuple(i for i in recorded if tuple(i) not in superseded)
|
|
1930
2464
|
detected_at = now.astimezone(UTC).isoformat()
|
|
1931
2465
|
return tuple(
|
|
1932
2466
|
mrc.RateChangeTransition(
|
|
@@ -1938,6 +2472,98 @@ def _owed_rate_change_transitions(account_keys, status, *, now,
|
|
|
1938
2472
|
d.decided_set(status.state) | {tuple(i) for i in settled}))
|
|
1939
2473
|
|
|
1940
2474
|
|
|
2475
|
+
def _superseded_identities(account_keys) -> frozenset:
|
|
2476
|
+
"""Every effectively superseded identity of `account_keys`, read on a
|
|
2477
|
+
guarded connection, or the empty set when it cannot be read."""
|
|
2478
|
+
import _cctally_quota_calibration as qcg
|
|
2479
|
+
from _cctally_db import StatsRebuildDeferred
|
|
2480
|
+
conn = None
|
|
2481
|
+
try:
|
|
2482
|
+
conn = _cctally_core.open_db()
|
|
2483
|
+
state = qcg.read_rate_change_supersessions(
|
|
2484
|
+
conn, account_keys=account_keys)
|
|
2485
|
+
return frozenset(state.by_identity)
|
|
2486
|
+
except (Exception, StatsRebuildDeferred): # noqa: BLE001
|
|
2487
|
+
return frozenset()
|
|
2488
|
+
finally:
|
|
2489
|
+
if conn is not None:
|
|
2490
|
+
conn.close()
|
|
2491
|
+
|
|
2492
|
+
|
|
2493
|
+
def reconcile_rate_change_supersessions(account_keys, *, now) -> int:
|
|
2494
|
+
"""Journal the explicit supersession of every effectively superseded
|
|
2495
|
+
transition identity of `account_keys` that has none (#1015 spec 5.7).
|
|
2496
|
+
|
|
2497
|
+
It runs after `persist_and_detect` released the calibration lock and
|
|
2498
|
+
before any transition is recorded, on every automatic invocation,
|
|
2499
|
+
withheld analyses included, and it reads the durable rows rather than
|
|
2500
|
+
the calibration file, so an identity whose account was reset is
|
|
2501
|
+
reconciled too. Through the sole stats writer, as
|
|
2502
|
+
`record_rate_change_transition` is; a crash between the sidecar write and
|
|
2503
|
+
the journal append is repaired by the next invocation. Returns how many
|
|
2504
|
+
supersessions were offered. A store that cannot be read or written is
|
|
2505
|
+
not an error here: the command reports what it found.
|
|
2506
|
+
"""
|
|
2507
|
+
import _cctally_journal as jr
|
|
2508
|
+
import _cctally_quota_calibration as qcg
|
|
2509
|
+
import _lib_journal
|
|
2510
|
+
from _cctally_db import StatsRebuildDeferred
|
|
2511
|
+
mrc = _cctally()._load_sibling("_lib_meter_rate_change")
|
|
2512
|
+
conn = None
|
|
2513
|
+
try:
|
|
2514
|
+
conn = _cctally_core.open_db()
|
|
2515
|
+
state = qcg.read_rate_change_supersessions(
|
|
2516
|
+
conn, account_keys=account_keys)
|
|
2517
|
+
pending = []
|
|
2518
|
+
for identity, supersession in sorted(state.by_identity.items()):
|
|
2519
|
+
if supersession.explicit:
|
|
2520
|
+
continue
|
|
2521
|
+
row = conn.execute(
|
|
2522
|
+
"SELECT detected_at_utc, fit_lineage FROM "
|
|
2523
|
+
"meter_rate_change_events WHERE provider = ? AND "
|
|
2524
|
+
"account_key = ? AND effective_from = ?", identity).fetchone()
|
|
2525
|
+
if row is None:
|
|
2526
|
+
continue
|
|
2527
|
+
lineage = qcg.transition_lineage_for(identity[1], row[0], row[1])
|
|
2528
|
+
if lineage.get("legacy"):
|
|
2529
|
+
lineage = {"legacy": True,
|
|
2530
|
+
"window": lineage["dependencyWindows"][0]}
|
|
2531
|
+
target = None
|
|
2532
|
+
for prefix in (mrc.EVT_ID_PREFIX_V2, mrc.EVT_ID_PREFIX):
|
|
2533
|
+
eid = _lib_journal.evt_id(prefix, *identity)
|
|
2534
|
+
if conn.execute(
|
|
2535
|
+
"SELECT 1 FROM journal_effective_events "
|
|
2536
|
+
"WHERE event_id = ?", (eid,)).fetchone() is not None:
|
|
2537
|
+
target = eid
|
|
2538
|
+
break
|
|
2539
|
+
pending.append({
|
|
2540
|
+
"provider": identity[0], "account_key": identity[1],
|
|
2541
|
+
"effective_from": identity[2], "target_event_id": target,
|
|
2542
|
+
"fit_lineage": lineage,
|
|
2543
|
+
"corrections": [{"opId": op, "observations": n}
|
|
2544
|
+
for op, n in supersession.observations],
|
|
2545
|
+
"superseded_at_utc": now.astimezone(UTC).isoformat()})
|
|
2546
|
+
except (Exception, StatsRebuildDeferred) as exc: # noqa: BLE001
|
|
2547
|
+
eprint(f"quota: could not read metering-rate supersessions: {exc}")
|
|
2548
|
+
return 0
|
|
2549
|
+
finally:
|
|
2550
|
+
if conn is not None:
|
|
2551
|
+
conn.close()
|
|
2552
|
+
if not pending:
|
|
2553
|
+
return 0
|
|
2554
|
+
try:
|
|
2555
|
+
jr.run_stats_ingest(
|
|
2556
|
+
mode="authoritative",
|
|
2557
|
+
meter_rate_change={
|
|
2558
|
+
"supersessions": pending,
|
|
2559
|
+
"created_at": now.astimezone(UTC).isoformat()})
|
|
2560
|
+
except (Exception, StatsRebuildDeferred) as exc: # noqa: BLE001
|
|
2561
|
+
eprint("quota: could not record the metering-rate supersession: "
|
|
2562
|
+
f"{exc}")
|
|
2563
|
+
return 0
|
|
2564
|
+
return len(pending)
|
|
2565
|
+
|
|
2566
|
+
|
|
1941
2567
|
def persist(analysis, mode: PersistMode) -> "str | None":
|
|
1942
2568
|
"""`persist_and_detect`'s quarantine half, for callers with no stats leg.
|
|
1943
2569
|
|
|
@@ -2024,6 +2650,31 @@ class AccountResult:
|
|
|
2024
2650
|
#: The coefficient era the clock selected, or None before every
|
|
2025
2651
|
#: validated era (#978). The method metadata names it.
|
|
2026
2652
|
era: "qm.CoefficientEra | None" = None
|
|
2653
|
+
#: #1015 spec 5.6: the recorded prior's lineage assessment,
|
|
2654
|
+
#: `{"state": "clean"|"recalibration-needed"|"unavailable",
|
|
2655
|
+
#: "corrections": [...]}`, or None when nothing is recorded.
|
|
2656
|
+
recorded_lineage: "dict | None" = None
|
|
2657
|
+
|
|
2658
|
+
|
|
2659
|
+
def recorded_prior_lineage(recorded, facts, account_key) -> "dict | None":
|
|
2660
|
+
"""The recorded prior presented with its assessment (spec 5.6)."""
|
|
2661
|
+
if recorded is None:
|
|
2662
|
+
return None
|
|
2663
|
+
annotation = recorded.get(_ANNOTATION_KEY)
|
|
2664
|
+
if isinstance(annotation, dict):
|
|
2665
|
+
corrections = annotation.get("corrections")
|
|
2666
|
+
return {"state": qm.RECALIBRATION_NEEDED,
|
|
2667
|
+
"corrections": list(corrections)
|
|
2668
|
+
if isinstance(corrections, list) else []}
|
|
2669
|
+
if facts is None:
|
|
2670
|
+
return {"state": "unavailable", "corrections": []}
|
|
2671
|
+
import _cctally_quota_calibration as qcg
|
|
2672
|
+
assessment = qcg.assess_regime(recorded, facts,
|
|
2673
|
+
scope=_state_key(account_key))
|
|
2674
|
+
if assessment.state == "tainted":
|
|
2675
|
+
return {"state": qm.RECALIBRATION_NEEDED,
|
|
2676
|
+
"corrections": _correction_json(assessment.corrections)}
|
|
2677
|
+
return {"state": assessment.state, "corrections": []}
|
|
2027
2678
|
|
|
2028
2679
|
|
|
2029
2680
|
def recorded_prior_is_stale(recorded, era) -> bool:
|
|
@@ -2122,7 +2773,9 @@ def analyse_account(account_key, *, now, since, watch_from,
|
|
|
2122
2773
|
account_key=account_key,
|
|
2123
2774
|
label=account_display_label(account_key),
|
|
2124
2775
|
analysis=reported, load=load, week=week, recorded=recorded,
|
|
2125
|
-
stale_prior=stale_prior, mode_kind=mode_kind, clean=clean, era=era
|
|
2776
|
+
stale_prior=stale_prior, mode_kind=mode_kind, clean=clean, era=era,
|
|
2777
|
+
recorded_lineage=recorded_prior_lineage(
|
|
2778
|
+
recorded, load.correction_facts, account_key))
|
|
2126
2779
|
|
|
2127
2780
|
|
|
2128
2781
|
# ---------------------------------------------------------------------------
|
|
@@ -2159,7 +2812,7 @@ def _evidence_json(evidence) -> dict:
|
|
|
2159
2812
|
}
|
|
2160
2813
|
|
|
2161
2814
|
|
|
2162
|
-
def _recorded_json(recorded, stale_prior) -> "dict | None":
|
|
2815
|
+
def _recorded_json(recorded, stale_prior, lineage=None) -> "dict | None":
|
|
2163
2816
|
if recorded is None:
|
|
2164
2817
|
return None
|
|
2165
2818
|
out = dict(recorded)
|
|
@@ -2167,9 +2820,66 @@ def _recorded_json(recorded, stale_prior) -> "dict | None":
|
|
|
2167
2820
|
if out.get(key) is not None:
|
|
2168
2821
|
out[key] = _iso(parse_instant(out[key], key))
|
|
2169
2822
|
out["fingerprintMatchesCurrent"] = not stale_prior
|
|
2823
|
+
if lineage is not None:
|
|
2824
|
+
# #1015 spec 5.6: additive; the analysis status and exit code are
|
|
2825
|
+
# unchanged.
|
|
2826
|
+
out["lineage"] = lineage
|
|
2170
2827
|
return out
|
|
2171
2828
|
|
|
2172
2829
|
|
|
2830
|
+
def _sorted_shares(shares) -> "dict | None":
|
|
2831
|
+
if shares is None:
|
|
2832
|
+
return None
|
|
2833
|
+
return {k: shares[k] for k in sorted(shares)}
|
|
2834
|
+
|
|
2835
|
+
|
|
2836
|
+
def _family_materiality_json(analysis, era) -> dict:
|
|
2837
|
+
"""`composition.familyMateriality` (#991 spec 4.8): the reference the
|
|
2838
|
+
evidence-bounded family test measures against, and the evidence."""
|
|
2839
|
+
return {
|
|
2840
|
+
"reference": "fit-weighted",
|
|
2841
|
+
"fitFamilyShares": _sorted_shares(analysis.fit_family_shares),
|
|
2842
|
+
"referenceFamily": None if era is None else era.reference_family,
|
|
2843
|
+
"familyRateBounds": _rate_bounds_json(era),
|
|
2844
|
+
"maxRelativeEffect": qm.MIX_SUPPORT["family_max_relative_effect"],
|
|
2845
|
+
}
|
|
2846
|
+
|
|
2847
|
+
|
|
2848
|
+
def _forecast_probe_json(analysis) -> dict:
|
|
2849
|
+
"""`composition.forecastProbe` (#991 spec 4.8).
|
|
2850
|
+
|
|
2851
|
+
The vectors are the exact forecast aggregate, published for the
|
|
2852
|
+
`no-reference` and `probed` states. Every measurement is the evaluator's
|
|
2853
|
+
and is null unless the state is `probed`; `supported` is the analysis'
|
|
2854
|
+
`forecast_supported` in every state.
|
|
2855
|
+
"""
|
|
2856
|
+
support = (analysis.forecast_support
|
|
2857
|
+
if analysis.forecast_probe_state == "probed" else None)
|
|
2858
|
+
effect = None if support is None else support.family_effect
|
|
2859
|
+
unavailable = None
|
|
2860
|
+
if effect is not None and effect.unavailable is not None:
|
|
2861
|
+
unavailable = {"reason": effect.unavailable.value,
|
|
2862
|
+
"families": sorted(effect.families)}
|
|
2863
|
+
return {
|
|
2864
|
+
"state": analysis.forecast_probe_state,
|
|
2865
|
+
"familyShares": _sorted_shares(analysis.forecast_family_shares),
|
|
2866
|
+
"classShares": _sorted_shares(analysis.forecast_class_shares),
|
|
2867
|
+
"familyDistance": None if support is None else support.family_distance,
|
|
2868
|
+
"familyDistanceReference": "baseline-family-shares",
|
|
2869
|
+
"familyWithinRadius": (None if support is None
|
|
2870
|
+
else support.family_within_radius),
|
|
2871
|
+
"familyEffectBound": None if effect is None else effect.bound,
|
|
2872
|
+
"familyEffectReference": "fit-family-shares",
|
|
2873
|
+
"familyEffectUnavailable": unavailable,
|
|
2874
|
+
"familyAdmittedBy": (None if support is None
|
|
2875
|
+
else support.family_admitted_by),
|
|
2876
|
+
"classDistance": None if support is None else support.class_distance,
|
|
2877
|
+
"classWithinRadius": (None if support is None
|
|
2878
|
+
else support.class_within_radius),
|
|
2879
|
+
"supported": analysis.forecast_supported,
|
|
2880
|
+
}
|
|
2881
|
+
|
|
2882
|
+
|
|
2173
2883
|
def account_payload(result: AccountResult) -> dict:
|
|
2174
2884
|
"""One account's complete payload, camelCase throughout."""
|
|
2175
2885
|
analysis = result.analysis
|
|
@@ -2197,7 +2907,8 @@ def account_payload(result: AccountResult) -> dict:
|
|
|
2197
2907
|
"invocation": result.mode_kind,
|
|
2198
2908
|
},
|
|
2199
2909
|
"calibration": {
|
|
2200
|
-
"recorded": _recorded_json(result.recorded, result.stale_prior
|
|
2910
|
+
"recorded": _recorded_json(result.recorded, result.stale_prior,
|
|
2911
|
+
result.recorded_lineage),
|
|
2201
2912
|
"fitted": _evidence_json(analysis.fitted),
|
|
2202
2913
|
},
|
|
2203
2914
|
"analysis": {
|
|
@@ -2223,6 +2934,8 @@ def account_payload(result: AccountResult) -> dict:
|
|
|
2223
2934
|
"familyRadiusEmpirical": diagnostics.get("familyRadiusEmpirical"),
|
|
2224
2935
|
"classRadiusEmpirical": diagnostics.get("classRadiusEmpirical"),
|
|
2225
2936
|
"radiusRule": "max(observed spread, materiality floor)",
|
|
2937
|
+
"familyMateriality": _family_materiality_json(analysis, era),
|
|
2938
|
+
"forecastProbe": _forecast_probe_json(analysis),
|
|
2226
2939
|
},
|
|
2227
2940
|
"health": {
|
|
2228
2941
|
"entriesThrough": _iso(load.newest_entry_at),
|
|
@@ -2357,6 +3070,12 @@ def render_text(results, *, now, decorated) -> list:
|
|
|
2357
3070
|
recorded_value = result.recorded.get("unitsPerPoint")
|
|
2358
3071
|
stale = " (stale — fitted under different constants)" \
|
|
2359
3072
|
if result.stale_prior else ""
|
|
3073
|
+
if (result.recorded_lineage or {}).get("state") \
|
|
3074
|
+
== qm.RECALIBRATION_NEEDED:
|
|
3075
|
+
# #1015 spec 5.6: U3's sentence on a tainted prior.
|
|
3076
|
+
import _lib_quota_copy
|
|
3077
|
+
stale += (" — " + _lib_quota_copy._LONG_FORM[
|
|
3078
|
+
qm.RECALIBRATION_NEEDED])
|
|
2360
3079
|
out.append(f" {'recorded prior':26} "
|
|
2361
3080
|
f"{_fmt_units(recorded_value)}{stale}")
|
|
2362
3081
|
out.append(" The fitted value is not a provider budget. It is one "
|
|
@@ -2395,6 +3114,7 @@ def render_text(results, *, now, decorated) -> list:
|
|
|
2395
3114
|
f"{_fmt_radius(analysis.class_radius)}"
|
|
2396
3115
|
f" (observed "
|
|
2397
3116
|
f"{_fmt_radius(analysis.diagnostics.get('classRadiusEmpirical'))})")
|
|
3117
|
+
out.extend(_forecast_family_lines(analysis))
|
|
2398
3118
|
out.append(f" {qm.DEDICATED_POOL_SCOPE_NOTE}")
|
|
2399
3119
|
out.append("")
|
|
2400
3120
|
out.append("DETECTOR")
|
|
@@ -2444,6 +3164,59 @@ def _fmt_radius(value) -> str:
|
|
|
2444
3164
|
return "-" if value is None else f"{value:.4f}"
|
|
2445
3165
|
|
|
2446
3166
|
|
|
3167
|
+
#: The width the COMPOSITION SUPPORT lines were approved at (#991 spec 4.9):
|
|
3168
|
+
#: the `not bounded:` line wraps onto a continuation line indented to the
|
|
3169
|
+
#: value column when it would run past it.
|
|
3170
|
+
_COMPOSITION_TEXT_WIDTH = 80
|
|
3171
|
+
#: The value column of every ` <label:26> value` line.
|
|
3172
|
+
_VALUE_COLUMN = 2 + 26 + 1
|
|
3173
|
+
|
|
3174
|
+
#: The distance line for each probe state the evaluator did not reach.
|
|
3175
|
+
_UNPROBED_DISTANCE = {
|
|
3176
|
+
"not-probed": "- (no forecast population)",
|
|
3177
|
+
"unaggregatable": "- (forecast population not aggregatable)",
|
|
3178
|
+
"no-reference": "- (no reference composition)",
|
|
3179
|
+
}
|
|
3180
|
+
|
|
3181
|
+
|
|
3182
|
+
def _forecast_family_lines(analysis) -> list:
|
|
3183
|
+
"""The two forecast-family lines of COMPOSITION SUPPORT (#991 spec 4.9)."""
|
|
3184
|
+
support = (analysis.forecast_support
|
|
3185
|
+
if analysis.forecast_probe_state == "probed" else None)
|
|
3186
|
+
if support is None:
|
|
3187
|
+
return [f" {'forecast family distance':26} "
|
|
3188
|
+
f"{_UNPROBED_DISTANCE.get(analysis.forecast_probe_state, '-')}",
|
|
3189
|
+
f" {'forecast family effect':26} -"]
|
|
3190
|
+
side = "inside" if support.family_within_radius else "outside"
|
|
3191
|
+
lines = [f" {'forecast family distance':26} "
|
|
3192
|
+
f"{_fmt_radius(support.family_distance)}"
|
|
3193
|
+
f" ({side} the family radius)"]
|
|
3194
|
+
effect = support.family_effect
|
|
3195
|
+
limit = f"{float(qm.MIX_SUPPORT['family_max_relative_effect']) * 100:.2f}%"
|
|
3196
|
+
label = f" {'forecast family effect':26} "
|
|
3197
|
+
if support.family_within_radius:
|
|
3198
|
+
lines.append(f"{label}- (not needed: inside the family radius)")
|
|
3199
|
+
elif support.family_admitted_by == "materiality":
|
|
3200
|
+
lines.append(f"{label}{effect.bound * 100:.2f}% (within {limit}: "
|
|
3201
|
+
f"admitted by family rate bounds)")
|
|
3202
|
+
elif effect.bound is not None:
|
|
3203
|
+
lines.append(f"{label}{effect.bound * 100:.2f}% (above {limit})")
|
|
3204
|
+
elif effect.unavailable is \
|
|
3205
|
+
qm.FamilyEffectUnavailable.UNEVIDENCED_FAMILY_CHANGE:
|
|
3206
|
+
text = (f"not bounded: {', '.join(sorted(effect.families))} "
|
|
3207
|
+
f"changed share without rate bounds")
|
|
3208
|
+
wrapped = textwrap.wrap(
|
|
3209
|
+
text, width=_COMPOSITION_TEXT_WIDTH - _VALUE_COLUMN,
|
|
3210
|
+
break_long_words=False, break_on_hyphens=False)
|
|
3211
|
+
lines.append(f"{label}{wrapped[0]}")
|
|
3212
|
+
lines.extend(" " * _VALUE_COLUMN + rest for rest in wrapped[1:])
|
|
3213
|
+
else:
|
|
3214
|
+
reason = (effect.unavailable.value if effect.unavailable is not None
|
|
3215
|
+
else "-")
|
|
3216
|
+
lines.append(f"{label}- (not evaluated: {reason})")
|
|
3217
|
+
return lines
|
|
3218
|
+
|
|
3219
|
+
|
|
2447
3220
|
# ---------------------------------------------------------------------------
|
|
2448
3221
|
# The command (spec sections 8, 17, 23, 24).
|
|
2449
3222
|
# ---------------------------------------------------------------------------
|
|
@@ -2465,6 +3238,68 @@ def _reset(args, keys, *, now) -> int:
|
|
|
2465
3238
|
return 0
|
|
2466
3239
|
|
|
2467
3240
|
|
|
3241
|
+
@dataclasses.dataclass(frozen=True)
|
|
3242
|
+
class SweepOutcome:
|
|
3243
|
+
"""What one account's sweep did (#1015 spec 5.5)."""
|
|
3244
|
+
|
|
3245
|
+
#: The quarantine path `persist_and_detect` reported, or None.
|
|
3246
|
+
quarantined: "str | None"
|
|
3247
|
+
#: The fresh transitions this sweep's persistence created.
|
|
3248
|
+
transitions: tuple
|
|
3249
|
+
#: Every identity this sweep offered to `_settle_rate_change`.
|
|
3250
|
+
settled: frozenset
|
|
3251
|
+
|
|
3252
|
+
|
|
3253
|
+
def sweep_account(result: AccountResult, *, now, delivery) -> SweepOutcome:
|
|
3254
|
+
"""One account's automatic sweep, in `cmd_quota`'s order (spec 5.5):
|
|
3255
|
+
build `PersistMode`, `persist_and_detect`, supersession reconciliation
|
|
3256
|
+
(5.7), fresh recording, #689 recovery, settlement.
|
|
3257
|
+
|
|
3258
|
+
Persistence runs AFTER the stores are closed, because the calibration
|
|
3259
|
+
lock is a leaf in the lock order and takes nothing else. #661 S2 §6.5
|
|
3260
|
+
steps 1-2: the descriptor is decided under the calibration file's leaf
|
|
3261
|
+
lock, and that lock is released before any stats lock is taken —
|
|
3262
|
+
`persist_and_detect` returns from its `with` block first.
|
|
3263
|
+
"""
|
|
3264
|
+
import _lib_accounts
|
|
3265
|
+
mode = persist_mode(result, now=now)
|
|
3266
|
+
quarantined, transitions, candidates = persist_and_detect(
|
|
3267
|
+
result.clean, mode)
|
|
3268
|
+
# #1015 spec 5.7: every effectively superseded transition identity of
|
|
3269
|
+
# this account (and of the `unattributed` sentinel a merged fit records
|
|
3270
|
+
# under) gets its explicit supersession event BEFORE anything is
|
|
3271
|
+
# recorded, on every automatic invocation, withheld analyses included.
|
|
3272
|
+
if mode.kind == "automatic":
|
|
3273
|
+
reconcile_rate_change_supersessions(
|
|
3274
|
+
(_alert_account_key(result.account_key),
|
|
3275
|
+
_lib_accounts.UNATTRIBUTED), now=now)
|
|
3276
|
+
settled: set = set()
|
|
3277
|
+
for transition in transitions:
|
|
3278
|
+
# §6.5 steps 3-6: through the sole stats writer, so the append, the
|
|
3279
|
+
# row and the cursor commit together and the notification dispatches
|
|
3280
|
+
# only after that commit. One write can create more than one adjacent
|
|
3281
|
+
# pair, and each is recorded: the row's UNIQUE key dedups, so a pair
|
|
3282
|
+
# a prior run already recorded folds to a no-op rather than a second
|
|
3283
|
+
# alert.
|
|
3284
|
+
settled.add(transition.identity())
|
|
3285
|
+
_settle_rate_change(
|
|
3286
|
+
transition, record_rate_change_transition(transition, now=now),
|
|
3287
|
+
delivery)
|
|
3288
|
+
# #689: a transition an EARLIER run persisted and then failed to record.
|
|
3289
|
+
# It can never be fresh again, so it is recovered from the persisted
|
|
3290
|
+
# state instead. Fresh descriptors are excluded because they were just
|
|
3291
|
+
# recorded above, and only the remainder pays for the presence lookup.
|
|
3292
|
+
fresh = {t.identity() for t in transitions}
|
|
3293
|
+
for transition in unrecorded_rate_change_transitions(
|
|
3294
|
+
tuple(c for c in candidates if c.identity() not in fresh)):
|
|
3295
|
+
settled.add(transition.identity())
|
|
3296
|
+
_settle_rate_change(
|
|
3297
|
+
transition, record_rate_change_transition(transition, now=now),
|
|
3298
|
+
delivery)
|
|
3299
|
+
return SweepOutcome(quarantined=quarantined, transitions=transitions,
|
|
3300
|
+
settled=frozenset(settled))
|
|
3301
|
+
|
|
3302
|
+
|
|
2468
3303
|
def cmd_quota(args) -> int:
|
|
2469
3304
|
"""`cctally quota` — weekly quota consumption and metering-rate change.
|
|
2470
3305
|
|
|
@@ -2517,46 +3352,11 @@ def cmd_quota(args) -> int:
|
|
|
2517
3352
|
result = analyse_account(
|
|
2518
3353
|
account_key, now=now, since=since, watch_from=watch_from,
|
|
2519
3354
|
stored_state=loaded.state)
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
mode = PersistMode(
|
|
2523
|
-
kind=result.mode_kind, account_key=account_key, now=now,
|
|
2524
|
-
fingerprint=qm.QUOTA_MODEL_CONSTANTS_FINGERPRINT,
|
|
2525
|
-
regime_start=result.load.analysis_start,
|
|
2526
|
-
coefficient_era=None if result.era is None else result.era.id)
|
|
2527
|
-
# §6.5 step 1-2: the descriptor is decided under the calibration
|
|
2528
|
-
# file's leaf lock, and that lock is released before any stats lock is
|
|
2529
|
-
# taken — `persist_and_detect` returns from its `with` block first.
|
|
2530
|
-
quarantined, transitions, candidates = persist_and_detect(
|
|
2531
|
-
result.clean, mode)
|
|
2532
|
-
quarantined = quarantined or loaded.quarantined
|
|
3355
|
+
outcome = sweep_account(result, now=now, delivery=delivery)
|
|
3356
|
+
quarantined = outcome.quarantined or loaded.quarantined
|
|
2533
3357
|
results.append(dataclasses.replace(result, quarantined=quarantined))
|
|
2534
3358
|
swept_keys.append(_alert_account_key(account_key))
|
|
2535
|
-
|
|
2536
|
-
# §6.5 steps 3-6: through the sole stats writer, so the append,
|
|
2537
|
-
# the row and the cursor commit together and the notification
|
|
2538
|
-
# dispatches only after that commit. One write can create more
|
|
2539
|
-
# than one adjacent pair, and each is recorded: the row's UNIQUE
|
|
2540
|
-
# key dedups, so a pair a prior run already recorded folds to a
|
|
2541
|
-
# no-op rather than a second alert.
|
|
2542
|
-
settled.add(transition.identity())
|
|
2543
|
-
_settle_rate_change(
|
|
2544
|
-
transition,
|
|
2545
|
-
record_rate_change_transition(transition, now=now),
|
|
2546
|
-
delivery)
|
|
2547
|
-
# #689: a transition an EARLIER run persisted and then failed to
|
|
2548
|
-
# record. It can never be fresh again, so it is recovered from the
|
|
2549
|
-
# persisted state instead. Fresh descriptors are excluded because
|
|
2550
|
-
# they were just recorded above, and only the remainder pays for the
|
|
2551
|
-
# presence lookup.
|
|
2552
|
-
fresh = {t.identity() for t in transitions}
|
|
2553
|
-
for transition in unrecorded_rate_change_transitions(
|
|
2554
|
-
tuple(c for c in candidates if c.identity() not in fresh)):
|
|
2555
|
-
settled.add(transition.identity())
|
|
2556
|
-
_settle_rate_change(
|
|
2557
|
-
transition,
|
|
2558
|
-
record_rate_change_transition(transition, now=now),
|
|
2559
|
-
delivery)
|
|
3359
|
+
settled |= outcome.settled
|
|
2560
3360
|
|
|
2561
3361
|
# #696: the `unattributed` sentinel joins `swept_keys` on every
|
|
2562
3362
|
# successful analysis, bare or filtered, whatever accounts this run
|