cctally 1.116.1 → 1.117.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +57 -3
- package/bin/_cctally_cache.py +5720 -885
- package/bin/_cctally_claude_attribution.py +997 -0
- package/bin/_cctally_completion.py +101 -28
- package/bin/_cctally_core.py +235 -12
- package/bin/_cctally_dashboard.py +263 -17
- package/bin/_cctally_dashboard_conversation.py +107 -21
- package/bin/_cctally_dashboard_envelope.py +44 -0
- package/bin/_cctally_db.py +928 -80
- package/bin/_cctally_diagnosis_sources.py +78 -13
- package/bin/_cctally_doctor.py +211 -86
- package/bin/_cctally_five_hour.py +446 -22
- package/bin/_cctally_forecast.py +302 -6
- package/bin/_cctally_journal.py +2418 -158
- package/bin/_cctally_milestone_history.py +379 -91
- package/bin/_cctally_oauth_samples.py +185 -7
- package/bin/_cctally_parser.py +38 -0
- package/bin/_cctally_project.py +34 -8
- package/bin/_cctally_quarantine.py +821 -343
- package/bin/_cctally_quota.py +69 -4
- package/bin/_cctally_quota_calibration.py +677 -3
- package/bin/_cctally_quota_model.py +927 -127
- package/bin/_cctally_record.py +922 -445
- package/bin/_cctally_rederive.py +119 -15
- package/bin/_cctally_statusline.py +1048 -176
- package/bin/_cctally_store.py +544 -110
- package/bin/_cctally_tui.py +455 -42
- package/bin/_cctally_update.py +77 -14
- package/bin/_lib_account_protocol.py +227 -19
- package/bin/_lib_claude_attribution.py +421 -0
- package/bin/_lib_clone_copy.py +30 -0
- package/bin/_lib_codex_conversation_query.py +45 -9
- package/bin/_lib_codex_conversation_watch.py +72 -13
- package/bin/_lib_codex_rollout_container.py +234 -19
- package/bin/_lib_codex_worker_lifecycle.py +374 -0
- package/bin/_lib_credit.py +22 -13
- package/bin/_lib_dashboard_sources.py +257 -18
- package/bin/_lib_detector_access_registry.py +1335 -166
- package/bin/_lib_doctor.py +276 -92
- package/bin/_lib_ingest_frontier.py +269 -36
- package/bin/_lib_journal.py +273 -22
- package/bin/_lib_jsonl.py +12 -0
- package/bin/_lib_meter_rate_change.py +72 -0
- package/bin/_lib_outcome_quarantine.py +917 -554
- package/bin/_lib_provider_activity.py +24 -6
- package/bin/_lib_quota_calibration.py +57 -3
- package/bin/_lib_quota_copy.py +7 -2
- package/bin/_lib_quota_lineage.py +613 -0
- package/bin/_lib_quota_model.py +464 -33
- package/bin/_lib_rate_change_delivery.py +16 -0
- package/bin/_lib_reclaim_planner.py +6 -4
- package/bin/_lib_rederive.py +2046 -21
- package/bin/_lib_selector_state.py +131 -3
- package/bin/_lib_snapshot_cache.py +8 -0
- package/bin/_lib_stale_proof.py +113 -4
- package/bin/_lib_statusline.py +72 -3
- package/bin/_lib_statusline_candidates.py +246 -42
- package/bin/_lib_tick_stats.py +10 -5
- package/bin/_lib_write_io.py +419 -6
- package/bin/cctally +28 -11
- package/dashboard/static/assets/{ConversationsView-Dflb_myA.js → ConversationsView-oqNsWzWV.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-DFX5CfIs.js → DoctorModal-B0DnU9d9.js} +1 -1
- package/dashboard/static/assets/ModalRoot-CYua2wP_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-C3PJy1xx.js → ProjectsDrillPanel-DSxXQSa-.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-BczsWKhD.js → SourceDetailModal-BleDpQqG.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-Os-Lo5cs.js → UpdateModal-CMqaAkSK.js} +1 -1
- package/dashboard/static/assets/{index-CYZ3NLqF.js → index-BjQV-0-D.js} +3 -3
- package/dashboard/static/assets/index-D34eYhAc.css +1 -0
- package/dashboard/static/assets/{outlineNavigation-k-zIif6W.js → outlineNavigation-CO-VWBm7.js} +1 -1
- package/dashboard/static/assets/{useKeymap-CW7Up0Ay.js → useKeymap-CUgEnnua.js} +1 -1
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +5 -1
- package/dashboard/static/assets/ModalRoot-HlxoagIa.js +0 -1
- package/dashboard/static/assets/index-BKITk1CT.css +0 -1
|
@@ -20,9 +20,13 @@ from __future__ import annotations
|
|
|
20
20
|
|
|
21
21
|
import dataclasses
|
|
22
22
|
import json
|
|
23
|
+
import sqlite3
|
|
23
24
|
|
|
24
25
|
import _cctally_quota_model
|
|
26
|
+
import _lib_journal
|
|
25
27
|
import _lib_quota_model as qm
|
|
28
|
+
import _lib_quota_lineage as ql
|
|
29
|
+
from _lib_quota_lineage import BatchFact, CorrectionFacts, CorrectionRow
|
|
26
30
|
from _lib_quota_calibration import ( # re-exported for callers and tests
|
|
27
31
|
AppliedQuota, ApplyRejection, RegimeRejection, ValidatedRegime,
|
|
28
32
|
apply_regime, validate_regime,
|
|
@@ -30,8 +34,12 @@ from _lib_quota_calibration import ( # re-exported for callers and tests
|
|
|
30
34
|
|
|
31
35
|
__all__ = [
|
|
32
36
|
"AppliedQuota", "ApplyRejection", "CalibrationRead", "RegimeRejection",
|
|
33
|
-
"ValidatedRegime", "apply_regime", "
|
|
34
|
-
"
|
|
37
|
+
"ValidatedRegime", "apply_regime", "assess_regime",
|
|
38
|
+
"read_calibration_file", "read_correction_facts", "refusal_status",
|
|
39
|
+
"validate_regime", "Supersession", "SupersessionState",
|
|
40
|
+
"assess_active_rate_change", "clean_active_rate_change", "marker_regimes",
|
|
41
|
+
"rate_change_supersession", "read_rate_change_supersessions",
|
|
42
|
+
"transition_lineage_for",
|
|
35
43
|
]
|
|
36
44
|
|
|
37
45
|
|
|
@@ -72,7 +80,8 @@ def _state_key(account_key) -> str:
|
|
|
72
80
|
def read_calibration_file(*, account_key, path=None,
|
|
73
81
|
expected_fingerprint=None,
|
|
74
82
|
expected_revision=None,
|
|
75
|
-
require_prediction_ready=True
|
|
83
|
+
require_prediction_ready=True,
|
|
84
|
+
stats_conn=None) -> CalibrationRead:
|
|
76
85
|
"""The open regime for one account, validated, or a typed cause.
|
|
77
86
|
|
|
78
87
|
`expected_fingerprint` and `expected_revision` default to THIS binary's
|
|
@@ -82,6 +91,16 @@ def read_calibration_file(*, account_key, path=None,
|
|
|
82
91
|
`require_prediction_ready` defaults to True, so every projection consumer
|
|
83
92
|
is refused a regime S1 marked `detection-only`. A detection consumer —
|
|
84
93
|
section 6's rate-change alert — passes False and reads the same record.
|
|
94
|
+
|
|
95
|
+
#1015 spec 5.4: a regime that validates is then assessed against the
|
|
96
|
+
published correction facts. A tainted (or already annotated) regime is
|
|
97
|
+
refused as `RECALIBRATION_NEEDED`, carrying in `regime_status` the
|
|
98
|
+
status of its `recalibrationAttempt` when that attempt's basis is the
|
|
99
|
+
current completed basis; unreadable lineage is `LINEAGE_UNAVAILABLE`.
|
|
100
|
+
`stats_conn` is a current stats connection the caller already holds;
|
|
101
|
+
without one the facts are read through the fail-fast read-only opener,
|
|
102
|
+
which never recovers or waits. No stats access happens unless a regime
|
|
103
|
+
validated.
|
|
85
104
|
"""
|
|
86
105
|
if path is None:
|
|
87
106
|
path = _cctally_quota_model.calibration_path()
|
|
@@ -143,4 +162,659 @@ def read_calibration_file(*, account_key, path=None,
|
|
|
143
162
|
stored = open_regime.get("status")
|
|
144
163
|
status = str(stored) if stored is not None else None
|
|
145
164
|
return CalibrationRead(None, outcome, status)
|
|
165
|
+
scope = _state_key(account_key)
|
|
166
|
+
lineage = regime_lineage_of(open_regime, scope=scope)
|
|
167
|
+
facts = read_correction_facts(stats_conn, account_scope=scope,
|
|
168
|
+
lineages=[lineage])
|
|
169
|
+
assessment = ql.assess(lineage, facts, coord_key=_coordinate_order)
|
|
170
|
+
if assessment.state == "tainted" or isinstance(
|
|
171
|
+
open_regime.get("supersededByCorrection"), dict):
|
|
172
|
+
return CalibrationRead(None, RegimeRejection.RECALIBRATION_NEEDED,
|
|
173
|
+
refusal_status(open_regime, facts))
|
|
174
|
+
if assessment.state != "clean":
|
|
175
|
+
return CalibrationRead(None, RegimeRejection.LINEAGE_UNAVAILABLE)
|
|
146
176
|
return CalibrationRead(outcome, None)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _era_start(regime):
|
|
180
|
+
"""The start of the regime's own coefficient era, or None (an unreadable
|
|
181
|
+
era leaves the legacy window's start open)."""
|
|
182
|
+
era = qm.era_by_id(regime.get("coefficientEra")) \
|
|
183
|
+
if isinstance(regime, dict) else None
|
|
184
|
+
return None if era is None else qm.era_bounds(era)[0]
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def regime_lineage_of(regime, *, scope) -> dict:
|
|
188
|
+
"""The lineage one stored regime is assessed under (spec 5.3): its own
|
|
189
|
+
verifying `fitLineage`, else the conservative legacy lineage over its
|
|
190
|
+
bucket's scope."""
|
|
191
|
+
return ql.regime_lineage(regime, scope=scope,
|
|
192
|
+
era_start=_era_start(regime))
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def assess_regime(regime, facts, *, scope) -> "ql.Assessment":
|
|
196
|
+
"""One stored regime's lineage assessment (spec 5.3) under
|
|
197
|
+
:func:`regime_lineage_of`. None facts are `unavailable`."""
|
|
198
|
+
return ql.assess(regime_lineage_of(regime, scope=scope), facts,
|
|
199
|
+
coord_key=_coordinate_order)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def refusal_status(regime, facts) -> "str | None":
|
|
203
|
+
"""The `recalibrationAttempt.status` a refusal states, when that
|
|
204
|
+
attempt's basis is the current completed basis; None otherwise."""
|
|
205
|
+
attempt = regime.get("recalibrationAttempt") \
|
|
206
|
+
if isinstance(regime, dict) else None
|
|
207
|
+
if not isinstance(attempt, dict) or facts is None:
|
|
208
|
+
return None
|
|
209
|
+
basis = attempt.get("basis")
|
|
210
|
+
if not isinstance(basis, list) or \
|
|
211
|
+
frozenset(str(b) for b in basis) != facts.basis:
|
|
212
|
+
return None
|
|
213
|
+
status = attempt.get("status")
|
|
214
|
+
return None if status is None else str(status)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
# --------------------------------------------------------------------------
|
|
218
|
+
# #1015: the correction facts a fit's lineage is assessed against
|
|
219
|
+
# --------------------------------------------------------------------------
|
|
220
|
+
|
|
221
|
+
#: The distinct reviewed ops named by the history, by a loose scan of its
|
|
222
|
+
#: primary key (one index probe per op, never a scan of the decision rows).
|
|
223
|
+
_HISTORY_OPS_SQL = (
|
|
224
|
+
"WITH RECURSIVE d(op) AS ("
|
|
225
|
+
"SELECT MIN(decision_op_id) FROM claude_weekly_correction_history "
|
|
226
|
+
"UNION ALL "
|
|
227
|
+
"SELECT (SELECT MIN(decision_op_id) FROM claude_weekly_correction_history "
|
|
228
|
+
"WHERE decision_op_id > d.op) FROM d WHERE d.op IS NOT NULL) "
|
|
229
|
+
"SELECT op FROM d WHERE op IS NOT NULL"
|
|
230
|
+
)
|
|
231
|
+
_OP_COORDS_SQL = (
|
|
232
|
+
"SELECT decision_op_id, op_segment, op_offset "
|
|
233
|
+
"FROM claude_weekly_correction_ops"
|
|
234
|
+
)
|
|
235
|
+
#: Fails to prepare when the observation index is missing (an index older
|
|
236
|
+
#: than epoch 1022), even when no correction row needs reading.
|
|
237
|
+
_INDEX_PROBE_SQL = (
|
|
238
|
+
"SELECT observation_id, account_key, captured_at_utc "
|
|
239
|
+
"FROM claude_weekly_observation_index LIMIT 0"
|
|
240
|
+
)
|
|
241
|
+
#: One reviewed op's decision rows joined to their observation's index row,
|
|
242
|
+
#: narrowed to what a `_lib_quota_lineage.RowDemand` keeps. A row is kept
|
|
243
|
+
#: when it is unresolved (no index row); when its capture is not one of the
|
|
244
|
+
#: canonical UTC text forms or not a valid calendar instant (the kernel reads
|
|
245
|
+
#: it as unresolved, so its account and window are never consulted); or when
|
|
246
|
+
#: it is in the caller's account scope and inside the demand's account and
|
|
247
|
+
#: its second-floored capture box. The box compares the first 19 characters
|
|
248
|
+
#: of a canonical UTC text (`YYYY-MM-DDTHH:MM:SS`), whose text order is time
|
|
249
|
+
#: order, against second-floored bounds, so it only ever widens the exact
|
|
250
|
+
#: windows `assess` applies afterwards.
|
|
251
|
+
_CORRECTION_ROWS_SQL = (
|
|
252
|
+
"SELECT h.observation_id, h.disposition, i.account_key, i.captured_at_utc "
|
|
253
|
+
"FROM claude_weekly_correction_history h "
|
|
254
|
+
"LEFT JOIN claude_weekly_observation_index i "
|
|
255
|
+
"ON i.observation_id = h.observation_id "
|
|
256
|
+
"WHERE h.decision_op_id = :op AND ("
|
|
257
|
+
"i.observation_id IS NULL "
|
|
258
|
+
"OR ((:scope = '*' OR i.account_key = :scope) "
|
|
259
|
+
"AND (:account IS NULL OR i.account_key = :account) "
|
|
260
|
+
"AND (:lo IS NULL OR i.captured_at_utc >= :lo) "
|
|
261
|
+
"AND (:hi IS NULL OR i.captured_at_utc < :hi)) "
|
|
262
|
+
"OR ((:scope = '*' OR i.account_key = :scope) AND NOT ("
|
|
263
|
+
"substr(i.captured_at_utc, 1, 19) GLOB "
|
|
264
|
+
"'[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]' "
|
|
265
|
+
"AND (substr(i.captured_at_utc, 20) IN ('Z', '+00:00') "
|
|
266
|
+
"OR substr(i.captured_at_utc, 20) GLOB '.[0-9][0-9][0-9][0-9][0-9][0-9]Z' "
|
|
267
|
+
"OR substr(i.captured_at_utc, 20) GLOB '.[0-9][0-9][0-9][0-9][0-9][0-9]+00:00' "
|
|
268
|
+
"OR substr(i.captured_at_utc, 20) GLOB '.[0-9][0-9][0-9]Z' "
|
|
269
|
+
"OR substr(i.captured_at_utc, 20) GLOB '.[0-9][0-9][0-9]+00:00') "
|
|
270
|
+
"AND substr(i.captured_at_utc, 1, 4) <> '0000' "
|
|
271
|
+
"AND datetime(substr(i.captured_at_utc, 1, 19)) "
|
|
272
|
+
"= replace(substr(i.captured_at_utc, 1, 19), 'T', ' '))))"
|
|
273
|
+
)
|
|
274
|
+
_BATCH_ROWS_SQL = (
|
|
275
|
+
"SELECT batch_id, commit_segment, commit_offset "
|
|
276
|
+
"FROM correction_batch_lineage"
|
|
277
|
+
)
|
|
278
|
+
_FOOTPRINT_ROWS_SQL = (
|
|
279
|
+
"SELECT batch_id, provider, account_key FROM correction_batch_footprint"
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _coordinate_order(coord) -> tuple:
|
|
284
|
+
"""Journal order of a ``(segment, offset)`` coordinate; the same key as
|
|
285
|
+
`_cctally_journal.journal_coordinate_key`, without importing the journal
|
|
286
|
+
glue into a per-prompt reader."""
|
|
287
|
+
return (_lib_journal.segment_sort_key(coord[0]), int(coord[1]))
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _box_text(instant, *, ceil: bool) -> "str | None":
|
|
291
|
+
"""A second-floored box bound as the 19-character canonical prefix;
|
|
292
|
+
``ceil`` moves it one second up so the exclusive comparison keeps every
|
|
293
|
+
capture inside that second. None (open) when it cannot be formed."""
|
|
294
|
+
import datetime as dt
|
|
295
|
+
floored = instant.replace(microsecond=0)
|
|
296
|
+
if ceil:
|
|
297
|
+
try:
|
|
298
|
+
floored += dt.timedelta(seconds=1)
|
|
299
|
+
except OverflowError:
|
|
300
|
+
return None
|
|
301
|
+
return (f"{floored.year:04d}-{floored.month:02d}-{floored.day:02d}T"
|
|
302
|
+
f"{floored.hour:02d}:{floored.minute:02d}:{floored.second:02d}")
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
class _FactsBase:
|
|
306
|
+
"""The cheap, always-read half of one snapshot's correction facts: every
|
|
307
|
+
completed batch with its footprint, and each reviewed op's coordinate.
|
|
308
|
+
``None`` from :func:`_read_base` when a decision names an op with no
|
|
309
|
+
coordinate (D3) or a projection is missing."""
|
|
310
|
+
|
|
311
|
+
__slots__ = ("batches", "basis", "op_coords")
|
|
312
|
+
|
|
313
|
+
def __init__(self, batches, op_coords):
|
|
314
|
+
self.batches = batches
|
|
315
|
+
self.basis = frozenset(batch.batch_id for batch in batches)
|
|
316
|
+
self.op_coords = op_coords
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _read_base(conn: sqlite3.Connection) -> "_FactsBase | None":
|
|
320
|
+
footprints: dict = {}
|
|
321
|
+
for batch_id, provider, account_key in conn.execute(_FOOTPRINT_ROWS_SQL):
|
|
322
|
+
footprints.setdefault(str(batch_id), set()).add(
|
|
323
|
+
(str(provider), str(account_key)))
|
|
324
|
+
batches = sorted(
|
|
325
|
+
(BatchFact(str(batch_id), (str(segment), int(offset)),
|
|
326
|
+
frozenset(footprints.get(str(batch_id), ())))
|
|
327
|
+
for batch_id, segment, offset in conn.execute(_BATCH_ROWS_SQL)),
|
|
328
|
+
key=lambda batch: (_coordinate_order(batch.commit_coord),
|
|
329
|
+
batch.batch_id))
|
|
330
|
+
known = {str(op): (str(segment), int(offset))
|
|
331
|
+
for op, segment, offset in conn.execute(_OP_COORDS_SQL)}
|
|
332
|
+
op_coords = {}
|
|
333
|
+
for (op,) in conn.execute(_HISTORY_OPS_SQL):
|
|
334
|
+
coord = known.get(str(op))
|
|
335
|
+
if coord is None:
|
|
336
|
+
# A decision the index cannot place in journal order cannot be
|
|
337
|
+
# ordered against a batch: the assessment is unavailable, never
|
|
338
|
+
# silently narrower — whatever the lineages being assessed.
|
|
339
|
+
return None
|
|
340
|
+
op_coords[str(op)] = coord
|
|
341
|
+
conn.execute(_INDEX_PROBE_SQL).fetchall()
|
|
342
|
+
return _FactsBase(tuple(batches), op_coords)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _read_rows(conn: sqlite3.Connection, base: _FactsBase,
|
|
346
|
+
account_scope: str, demand) -> tuple:
|
|
347
|
+
"""The correction rows ``demand`` keeps, in journal order of the op."""
|
|
348
|
+
if demand is None:
|
|
349
|
+
return ()
|
|
350
|
+
lo = hi = None
|
|
351
|
+
if demand.empty:
|
|
352
|
+
lo, hi = "~", None # no capture text sorts at or above "~"
|
|
353
|
+
else:
|
|
354
|
+
if demand.start is not None:
|
|
355
|
+
lo = _box_text(demand.start, ceil=False)
|
|
356
|
+
if demand.until is not None:
|
|
357
|
+
hi = _box_text(demand.until, ceil=True)
|
|
358
|
+
params = {"scope": account_scope, "account": demand.account,
|
|
359
|
+
"lo": lo, "hi": hi}
|
|
360
|
+
corrections = []
|
|
361
|
+
for op_id, coord in base.op_coords.items():
|
|
362
|
+
if demand.op_before is not None and \
|
|
363
|
+
_coordinate_order(coord) >= demand.op_before:
|
|
364
|
+
continue
|
|
365
|
+
params["op"] = op_id
|
|
366
|
+
for observation_id, disposition, account_key, captured_at in \
|
|
367
|
+
conn.execute(_CORRECTION_ROWS_SQL, params):
|
|
368
|
+
corrections.append(CorrectionRow(
|
|
369
|
+
op_id, str(observation_id), str(disposition),
|
|
370
|
+
None if account_key is None else str(account_key),
|
|
371
|
+
None if captured_at is None else str(captured_at), coord))
|
|
372
|
+
corrections.sort(key=lambda row: (_coordinate_order(row.op_coord),
|
|
373
|
+
row.op_id, row.observation_id))
|
|
374
|
+
return tuple(corrections)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _facts_from(conn, base, account_scope, lineages) -> CorrectionFacts:
|
|
378
|
+
demand = ql.ALL_ROWS if lineages is None else ql.row_demand(
|
|
379
|
+
lineages, base.batches, coord_key=_coordinate_order)
|
|
380
|
+
return CorrectionFacts(
|
|
381
|
+
_read_rows(conn, base, account_scope, demand), base.batches,
|
|
382
|
+
base.basis)
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _read_facts(conn: sqlite3.Connection, account_scope: str,
|
|
386
|
+
lineages=None) -> "CorrectionFacts | None":
|
|
387
|
+
base = _read_base(conn)
|
|
388
|
+
if base is None:
|
|
389
|
+
return None
|
|
390
|
+
return _facts_from(conn, base, account_scope, lineages)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
class _LazyFacts:
|
|
394
|
+
"""One read snapshot's correction facts, read only as far as the
|
|
395
|
+
lineages assessed so far need (R6). :meth:`facts_for` returns facts that
|
|
396
|
+
assess every lineage ever passed to it exactly as the full facts would;
|
|
397
|
+
a lineage outside that set widens the demand and re-reads the rows.
|
|
398
|
+
`base` is None when the facts are unavailable."""
|
|
399
|
+
|
|
400
|
+
def __init__(self, conn, account_scope):
|
|
401
|
+
self._conn = conn
|
|
402
|
+
self._scope = account_scope
|
|
403
|
+
self.base = _read_base(conn)
|
|
404
|
+
self._covered: list = []
|
|
405
|
+
self._facts = None
|
|
406
|
+
|
|
407
|
+
def facts_for(self, lineages) -> "CorrectionFacts | None":
|
|
408
|
+
if self.base is None:
|
|
409
|
+
return None
|
|
410
|
+
new = [lin for lin in lineages
|
|
411
|
+
if not any(lin == seen for seen in self._covered)]
|
|
412
|
+
if new or self._facts is None:
|
|
413
|
+
self._covered.extend(new)
|
|
414
|
+
self._facts = _facts_from(self._conn, self.base, self._scope,
|
|
415
|
+
self._covered)
|
|
416
|
+
return self._facts
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def correction_facts_source(conn: sqlite3.Connection,
|
|
420
|
+
account_scope: str = "*") -> "_LazyFacts | None":
|
|
421
|
+
"""A lazy correction-facts source over ``conn`` for consumers that learn
|
|
422
|
+
their lineages one at a time (R6), or None when the facts are
|
|
423
|
+
unavailable (a decision without an op coordinate, a missing projection,
|
|
424
|
+
a read error). The caller owns the read snapshot: hold one transaction
|
|
425
|
+
across every use."""
|
|
426
|
+
try:
|
|
427
|
+
source = _LazyFacts(conn, account_scope)
|
|
428
|
+
except sqlite3.Error:
|
|
429
|
+
return None
|
|
430
|
+
return None if source.base is None else source
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def read_correction_facts(conn: "sqlite3.Connection | None" = None, *,
|
|
434
|
+
account_scope: str,
|
|
435
|
+
lineages=None) -> "CorrectionFacts | None":
|
|
436
|
+
"""The published correction facts for one account scope, or None.
|
|
437
|
+
|
|
438
|
+
#1015 spec 5.3. ``account_scope`` is an account key, or ``"*"`` for the
|
|
439
|
+
merged view. The corrections are every reviewed decision row whose
|
|
440
|
+
observation the index attributes to that scope, plus every unresolved
|
|
441
|
+
reference (no index row; ``account_key`` and ``captured_at`` None), which
|
|
442
|
+
the lineage predicate must see to call the fit unavailable. The batches
|
|
443
|
+
are every completed correction batch with its footprint, in journal
|
|
444
|
+
order; the basis is their ids.
|
|
445
|
+
|
|
446
|
+
``lineages`` (R6) narrows the corrections to the rows those lineages can
|
|
447
|
+
be affected by (`_lib_quota_lineage.row_demand`): the returned facts
|
|
448
|
+
assess each of them exactly as the full facts do, and no other lineage.
|
|
449
|
+
A set whose every basis holds each batch touching Claude reads no
|
|
450
|
+
correction row at all. The batches, the basis and the D3 refusal are
|
|
451
|
+
those of the full facts in every case.
|
|
452
|
+
|
|
453
|
+
It reads only the published stats projections, in one read snapshot.
|
|
454
|
+
With ``conn`` None it opens through
|
|
455
|
+
`_cctally_store.stats_open_readonly_failfast` and closes what it opened,
|
|
456
|
+
returning None whenever that opener does; a caller's connection is left
|
|
457
|
+
open and its transaction state untouched. None also when a projection is
|
|
458
|
+
missing (an index older than epoch 1022) or a decision row has no op
|
|
459
|
+
coordinate.
|
|
460
|
+
"""
|
|
461
|
+
owned = conn is None
|
|
462
|
+
if owned:
|
|
463
|
+
import _cctally_store
|
|
464
|
+
conn = _cctally_store.stats_open_readonly_failfast()
|
|
465
|
+
if conn is None:
|
|
466
|
+
return None
|
|
467
|
+
try:
|
|
468
|
+
began = not conn.in_transaction
|
|
469
|
+
if began:
|
|
470
|
+
conn.execute("BEGIN")
|
|
471
|
+
try:
|
|
472
|
+
return _read_facts(conn, account_scope,
|
|
473
|
+
None if lineages is None else list(lineages))
|
|
474
|
+
finally:
|
|
475
|
+
if began and conn.in_transaction:
|
|
476
|
+
conn.execute("ROLLBACK")
|
|
477
|
+
except sqlite3.Error:
|
|
478
|
+
return None
|
|
479
|
+
finally:
|
|
480
|
+
if owned:
|
|
481
|
+
conn.close()
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
# --------------------------------------------------------------------------
|
|
485
|
+
# #1015 spec 5.6/5.7: effective transition supersession and the marker
|
|
486
|
+
# --------------------------------------------------------------------------
|
|
487
|
+
|
|
488
|
+
#: The one supersession reason this binary writes or derives.
|
|
489
|
+
SUPERSESSION_REASON = "reviewed-correction"
|
|
490
|
+
|
|
491
|
+
_EXPLICIT_SUPERSESSION_SQL = (
|
|
492
|
+
"SELECT provider, account_key, effective_from, reason, corrections, "
|
|
493
|
+
"superseded_at_utc FROM meter_rate_change_supersessions"
|
|
494
|
+
)
|
|
495
|
+
_TRANSITION_LINEAGE_SQL = (
|
|
496
|
+
"SELECT provider, account_key, effective_from, detected_at_utc, "
|
|
497
|
+
"fit_lineage FROM meter_rate_change_events"
|
|
498
|
+
)
|
|
499
|
+
_IDENTITY_CLAUSE = " WHERE provider = ? AND account_key = ? AND effective_from = ?"
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
@dataclasses.dataclass(frozen=True)
|
|
503
|
+
class Supersession:
|
|
504
|
+
"""One transition identity's effective supersession (spec 5.7).
|
|
505
|
+
|
|
506
|
+
`superseded_at` is the explicit event's sweep clock, or None for a
|
|
507
|
+
supersession derived from the transition's lineage alone (no event yet).
|
|
508
|
+
`corrections` are the op ids; `observations` pairs each with its
|
|
509
|
+
observation count."""
|
|
510
|
+
|
|
511
|
+
reason: str
|
|
512
|
+
superseded_at: "str | None"
|
|
513
|
+
corrections: tuple
|
|
514
|
+
observations: tuple
|
|
515
|
+
explicit: bool
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
@dataclasses.dataclass(frozen=True)
|
|
519
|
+
class SupersessionState:
|
|
520
|
+
"""Every effectively superseded transition identity, or `available`
|
|
521
|
+
False when the store or its correction facts could not be read."""
|
|
522
|
+
|
|
523
|
+
by_identity: dict
|
|
524
|
+
available: bool
|
|
525
|
+
#: Identities whose lineage assessment was `unavailable` (an unresolved
|
|
526
|
+
#: reference): never asserted superseded, never asserted clean.
|
|
527
|
+
unavailable: frozenset = frozenset()
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def _explicit_supersession(reason, corrections_json, superseded_at):
|
|
531
|
+
try:
|
|
532
|
+
parsed = json.loads(corrections_json or "[]")
|
|
533
|
+
except ValueError:
|
|
534
|
+
parsed = []
|
|
535
|
+
pairs = tuple((str(c.get("opId")), int(c.get("observations") or 0))
|
|
536
|
+
for c in parsed if isinstance(c, dict))
|
|
537
|
+
return Supersession(
|
|
538
|
+
reason=str(reason or SUPERSESSION_REASON),
|
|
539
|
+
superseded_at=None if superseded_at is None else str(superseded_at),
|
|
540
|
+
corrections=tuple(op for op, _n in pairs), observations=pairs,
|
|
541
|
+
explicit=True)
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def transition_lineage_for(account_key, detected_at, fit_lineage=None) -> dict:
|
|
545
|
+
"""The lineage one transition is assessed under: its frozen
|
|
546
|
+
`transitionLineage` when it verifies (a dict or its canonical JSON), else
|
|
547
|
+
the conservative legacy lineage `(-inf, detected_at]` over the row's
|
|
548
|
+
account, the `unattributed` sentinel reading as every account."""
|
|
549
|
+
import _lib_accounts
|
|
550
|
+
lineage = fit_lineage
|
|
551
|
+
if isinstance(lineage, str):
|
|
552
|
+
try:
|
|
553
|
+
lineage = json.loads(lineage)
|
|
554
|
+
except ValueError:
|
|
555
|
+
lineage = None
|
|
556
|
+
if ql.transition_lineage_verifies(lineage):
|
|
557
|
+
return dict(lineage)
|
|
558
|
+
return ql.legacy_transition_lineage(
|
|
559
|
+
account_key=account_key, detected_at=detected_at,
|
|
560
|
+
unattributed=_lib_accounts.UNATTRIBUTED)
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def _derived_supersession(lineage, facts) -> "Supersession | str | None":
|
|
564
|
+
assessment = ql.assess(lineage, facts, coord_key=_coordinate_order)
|
|
565
|
+
if assessment.state == "unavailable":
|
|
566
|
+
return "unavailable"
|
|
567
|
+
if assessment.state != "tainted":
|
|
568
|
+
return None
|
|
569
|
+
pairs = tuple((str(op), int(n)) for op, n in assessment.corrections)
|
|
570
|
+
return Supersession(
|
|
571
|
+
reason=SUPERSESSION_REASON, superseded_at=None,
|
|
572
|
+
corrections=tuple(op for op, _n in pairs), observations=pairs,
|
|
573
|
+
explicit=False)
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
class _Unset:
|
|
577
|
+
pass
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
_UNSET = _Unset()
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def rate_change_supersession(conn, identity, *, lineage=None,
|
|
584
|
+
detected_at=None, facts=_UNSET) \
|
|
585
|
+
-> "Supersession | None":
|
|
586
|
+
"""`identity`'s effective supersession on `conn`, or None (spec 5.7).
|
|
587
|
+
|
|
588
|
+
An explicit `meter_rate_change_supersessions` row wins. Otherwise the
|
|
589
|
+
identity is assessed under `lineage` when that frozen lineage verifies,
|
|
590
|
+
else under its row's stored lineage, else (a row with a null lineage)
|
|
591
|
+
under the conservative legacy lineage. An identity with neither a
|
|
592
|
+
verifying lineage nor a row has nothing to assess and is not
|
|
593
|
+
superseded. Unreadable facts, a missing table or any read error answer
|
|
594
|
+
None: the caller keeps its pre-#1015 behaviour and the read paths refuse
|
|
595
|
+
on their own. ``facts`` may be facts covering the resolved lineage, or a
|
|
596
|
+
snapshot's lazy facts source (`correction_facts_source`), which reads
|
|
597
|
+
the rows that lineage needs (R6); unset, they are read for it alone."""
|
|
598
|
+
try:
|
|
599
|
+
began = not conn.in_transaction
|
|
600
|
+
if began:
|
|
601
|
+
conn.execute("BEGIN")
|
|
602
|
+
try:
|
|
603
|
+
explicit = conn.execute(
|
|
604
|
+
"SELECT reason, corrections, superseded_at_utc FROM "
|
|
605
|
+
"meter_rate_change_supersessions" + _IDENTITY_CLAUSE,
|
|
606
|
+
tuple(identity)).fetchone()
|
|
607
|
+
if explicit is not None:
|
|
608
|
+
return _explicit_supersession(*explicit)
|
|
609
|
+
if not ql.transition_lineage_verifies(lineage):
|
|
610
|
+
row = conn.execute(
|
|
611
|
+
"SELECT detected_at_utc, fit_lineage FROM "
|
|
612
|
+
"meter_rate_change_events" + _IDENTITY_CLAUSE,
|
|
613
|
+
tuple(identity)).fetchone()
|
|
614
|
+
if row is None:
|
|
615
|
+
return None
|
|
616
|
+
lineage = transition_lineage_for(identity[1], row[0], row[1])
|
|
617
|
+
if facts is _UNSET:
|
|
618
|
+
facts = _read_facts(conn, "*", [lineage])
|
|
619
|
+
elif isinstance(facts, _LazyFacts):
|
|
620
|
+
facts = facts.facts_for([lineage])
|
|
621
|
+
if facts is None:
|
|
622
|
+
return None
|
|
623
|
+
derived = _derived_supersession(lineage, facts)
|
|
624
|
+
return derived if isinstance(derived, Supersession) else None
|
|
625
|
+
finally:
|
|
626
|
+
if began and conn.in_transaction:
|
|
627
|
+
conn.execute("ROLLBACK")
|
|
628
|
+
except sqlite3.Error:
|
|
629
|
+
return None
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
def read_rate_change_supersessions(conn: "sqlite3.Connection | None" = None,
|
|
633
|
+
*, account_keys=None) -> SupersessionState:
|
|
634
|
+
"""Every effectively superseded transition identity (spec 5.7).
|
|
635
|
+
|
|
636
|
+
One read snapshot over the explicit rows, the transition rows (limited
|
|
637
|
+
to `account_keys` when given) and the correction facts. With `conn`
|
|
638
|
+
None it opens through the fail-fast read-only opener and closes what it
|
|
639
|
+
opened; a caller's connection is left open and its transaction state
|
|
640
|
+
untouched. The derived form makes correction -> rebuild -> read safe
|
|
641
|
+
before any sweep, and a rebuild before or after the explicit event gives
|
|
642
|
+
the same answer."""
|
|
643
|
+
owned = conn is None
|
|
644
|
+
if owned:
|
|
645
|
+
import _cctally_store
|
|
646
|
+
conn = _cctally_store.stats_open_readonly_failfast()
|
|
647
|
+
if conn is None:
|
|
648
|
+
return SupersessionState({}, False)
|
|
649
|
+
keys = None if account_keys is None else {str(k) for k in account_keys}
|
|
650
|
+
try:
|
|
651
|
+
began = not conn.in_transaction
|
|
652
|
+
if began:
|
|
653
|
+
conn.execute("BEGIN")
|
|
654
|
+
try:
|
|
655
|
+
by_identity: dict = {}
|
|
656
|
+
for (provider, account, effective, reason, corrections,
|
|
657
|
+
superseded_at) in conn.execute(_EXPLICIT_SUPERSESSION_SQL):
|
|
658
|
+
if keys is not None and str(account) not in keys:
|
|
659
|
+
continue
|
|
660
|
+
by_identity[(str(provider), str(account), str(effective))] = \
|
|
661
|
+
_explicit_supersession(reason, corrections, superseded_at)
|
|
662
|
+
pending = []
|
|
663
|
+
for provider, account, effective, detected_at, lineage in \
|
|
664
|
+
conn.execute(_TRANSITION_LINEAGE_SQL):
|
|
665
|
+
identity = (str(provider), str(account), str(effective))
|
|
666
|
+
if identity in by_identity or (
|
|
667
|
+
keys is not None and identity[1] not in keys):
|
|
668
|
+
continue
|
|
669
|
+
pending.append((identity, transition_lineage_for(
|
|
670
|
+
identity[1], detected_at, lineage)))
|
|
671
|
+
# R6: the rows the assessed transition lineages need, no more.
|
|
672
|
+
facts = _read_facts(conn, "*", [lin for _i, lin in pending])
|
|
673
|
+
if facts is None:
|
|
674
|
+
return SupersessionState(by_identity, False)
|
|
675
|
+
unavailable = set()
|
|
676
|
+
for identity, lineage in pending:
|
|
677
|
+
derived = _derived_supersession(lineage, facts)
|
|
678
|
+
if derived == "unavailable":
|
|
679
|
+
unavailable.add(identity)
|
|
680
|
+
elif derived is not None:
|
|
681
|
+
by_identity[identity] = derived
|
|
682
|
+
return SupersessionState(by_identity, True, frozenset(unavailable))
|
|
683
|
+
finally:
|
|
684
|
+
if began and conn.in_transaction:
|
|
685
|
+
conn.execute("ROLLBACK")
|
|
686
|
+
except sqlite3.Error:
|
|
687
|
+
return SupersessionState({}, False)
|
|
688
|
+
finally:
|
|
689
|
+
if owned:
|
|
690
|
+
conn.close()
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
#: `assess_active_rate_change` states.
|
|
694
|
+
MARKER_ACTIVE = "active"
|
|
695
|
+
MARKER_NONE = "none"
|
|
696
|
+
MARKER_UNAVAILABLE = "unavailable"
|
|
697
|
+
|
|
698
|
+
|
|
699
|
+
def assess_active_rate_change(account_key=None, *, regimes=None,
|
|
700
|
+
stats_conn=None) -> tuple:
|
|
701
|
+
"""`(state, active)`; see `_assess_marker`."""
|
|
702
|
+
state, active, _clean = _assess_marker(
|
|
703
|
+
account_key, regimes=regimes, stats_conn=stats_conn)
|
|
704
|
+
return state, active
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def _assess_marker(account_key=None, *, regimes=None,
|
|
708
|
+
stats_conn=None) -> tuple:
|
|
709
|
+
"""`(state, active, pair)` of §6.6's marker over CLEAN regimes (spec 5.6).
|
|
710
|
+
|
|
711
|
+
`active_rate_change` over the stored regimes first: when it finds no
|
|
712
|
+
open successor (the steady state) nothing else is read, so the marker
|
|
713
|
+
costs one small read only where a change is active. Otherwise the
|
|
714
|
+
correction facts are read (the caller's stats connection, or the
|
|
715
|
+
fail-fast opener) and the active pair of the clean regimes is found
|
|
716
|
+
newest first (`active_pair_among`), assessing only the regimes that can
|
|
717
|
+
decide it (R6): the same answer as pairing every clean regime, without
|
|
718
|
+
assessing closed history behind the active pair. The surviving active
|
|
719
|
+
transition is suppressed when its identity is superseded. The rows are
|
|
720
|
+
read only as far as the assessed lineages need, starting with the
|
|
721
|
+
unfiltered active pair and its successor's frozen transition lineage.
|
|
722
|
+
Unreadable facts yield `unavailable`: a change that cannot be verified
|
|
723
|
+
is not asserted."""
|
|
724
|
+
import _lib_meter_rate_change as mrc
|
|
725
|
+
glue = _cctally_quota_model
|
|
726
|
+
if regimes is None:
|
|
727
|
+
state = glue.read_stored_state_readonly()
|
|
728
|
+
regimes = [] if state is None else glue.stored_regimes(
|
|
729
|
+
state, account_key)
|
|
730
|
+
regimes = list(regimes or ())
|
|
731
|
+
if mrc.active_rate_change(regimes) is None:
|
|
732
|
+
return MARKER_NONE, None, ()
|
|
733
|
+
owned = stats_conn is None
|
|
734
|
+
conn = stats_conn
|
|
735
|
+
if owned:
|
|
736
|
+
import _cctally_store
|
|
737
|
+
conn = _cctally_store.stats_open_readonly_failfast()
|
|
738
|
+
if conn is None:
|
|
739
|
+
return MARKER_UNAVAILABLE, None, ()
|
|
740
|
+
try:
|
|
741
|
+
began = not conn.in_transaction
|
|
742
|
+
if began:
|
|
743
|
+
conn.execute("BEGIN")
|
|
744
|
+
try:
|
|
745
|
+
return _marker_in_snapshot(conn, account_key, regimes)
|
|
746
|
+
finally:
|
|
747
|
+
if began and conn.in_transaction:
|
|
748
|
+
conn.execute("ROLLBACK")
|
|
749
|
+
except sqlite3.Error:
|
|
750
|
+
return MARKER_UNAVAILABLE, None, ()
|
|
751
|
+
finally:
|
|
752
|
+
if owned:
|
|
753
|
+
conn.close()
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def _marker_in_snapshot(conn, account_key, regimes) -> tuple:
|
|
757
|
+
import _lib_meter_rate_change as mrc
|
|
758
|
+
glue = _cctally_quota_model
|
|
759
|
+
scope = _state_key(account_key)
|
|
760
|
+
facts = _LazyFacts(conn, "*")
|
|
761
|
+
if facts.base is None:
|
|
762
|
+
return MARKER_UNAVAILABLE, None, ()
|
|
763
|
+
lineages: dict = {}
|
|
764
|
+
|
|
765
|
+
def lineage(regime):
|
|
766
|
+
if id(regime) not in lineages:
|
|
767
|
+
lineages[id(regime)] = regime_lineage_of(regime, scope=scope)
|
|
768
|
+
return lineages[id(regime)]
|
|
769
|
+
|
|
770
|
+
def transition(regime):
|
|
771
|
+
frozen = regime.get(ql.TRANSITION_LINEAGE_KEY)
|
|
772
|
+
return frozen if ql.transition_lineage_verifies(frozen) else None
|
|
773
|
+
|
|
774
|
+
# Prefetch what the all-clean answer reads: the unfiltered active pair
|
|
775
|
+
# and its successor's frozen lineage, so the common case is one read.
|
|
776
|
+
unfiltered = mrc.active_pair_among(regimes, lambda _r: True)
|
|
777
|
+
if unfiltered is not None:
|
|
778
|
+
facts.facts_for([lin for lin in (
|
|
779
|
+
lineage(unfiltered[0]), lineage(unfiltered[1]),
|
|
780
|
+
transition(unfiltered[1])) if lin is not None])
|
|
781
|
+
|
|
782
|
+
def is_clean(regime) -> bool:
|
|
783
|
+
if isinstance(regime.get("supersededByCorrection"), dict):
|
|
784
|
+
return False
|
|
785
|
+
return ql.assess(lineage(regime), facts.facts_for([lineage(regime)]),
|
|
786
|
+
coord_key=_coordinate_order).state == "clean"
|
|
787
|
+
|
|
788
|
+
pair = mrc.active_pair_among(regimes, is_clean)
|
|
789
|
+
if pair is None:
|
|
790
|
+
return MARKER_NONE, None, ()
|
|
791
|
+
active = mrc.active_rate_change(pair)
|
|
792
|
+
successor = pair[1]
|
|
793
|
+
identity = ("claude", glue._alert_account_key(account_key),
|
|
794
|
+
active["effective_from"])
|
|
795
|
+
if rate_change_supersession(
|
|
796
|
+
conn, identity, lineage=successor.get(ql.TRANSITION_LINEAGE_KEY),
|
|
797
|
+
facts=facts) is not None:
|
|
798
|
+
return MARKER_NONE, None, ()
|
|
799
|
+
return MARKER_ACTIVE, active, tuple(pair)
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
def clean_active_rate_change(account_key=None, *, regimes=None,
|
|
803
|
+
stats_conn=None) -> "dict | None":
|
|
804
|
+
"""§6.6's marker predicate over clean, unsuperseded regimes, or None
|
|
805
|
+
(none active, superseded, or unverifiable)."""
|
|
806
|
+
_state, active = assess_active_rate_change(
|
|
807
|
+
account_key, regimes=regimes, stats_conn=stats_conn)
|
|
808
|
+
return active
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
def marker_regimes(account_key=None, *, regimes=None,
|
|
812
|
+
stats_conn=None) -> tuple:
|
|
813
|
+
"""The regimes a regime-pair marker consumer may read: the clean active
|
|
814
|
+
pair `(predecessor, successor)` when its transition survives the
|
|
815
|
+
assessment, else (). `active_rate_change` over it is the marker over
|
|
816
|
+
every clean regime (R6: closed history behind the pair is never
|
|
817
|
+
assessed)."""
|
|
818
|
+
_state, _active, clean = _assess_marker(
|
|
819
|
+
account_key, regimes=regimes, stats_conn=stats_conn)
|
|
820
|
+
return clean
|