cctally 1.88.2 → 1.89.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/bin/_cctally_cache.py +827 -37
- package/bin/_cctally_config.py +125 -0
- package/bin/_cctally_core.py +86 -2
- package/bin/_cctally_dashboard_cache_report.py +31 -0
- package/bin/_cctally_dashboard_conversation.py +26 -7
- package/bin/_cctally_dashboard_sources.py +51 -0
- package/bin/_cctally_db.py +626 -0
- package/bin/_cctally_doctor.py +84 -1
- package/bin/_cctally_journal.py +2 -1
- package/bin/_cctally_parser.py +42 -0
- package/bin/_cctally_quota.py +1358 -112
- package/bin/_cctally_record.py +249 -9
- package/bin/_cctally_setup.py +14 -5
- package/bin/_cctally_store.py +16 -1
- package/bin/_cctally_tui.py +16 -2
- package/bin/_cctally_update.py +9 -2
- package/bin/_lib_background_mcp.py +168 -0
- package/bin/_lib_cache_report.py +19 -2
- package/bin/_lib_codex_conversation.py +8 -0
- package/bin/_lib_codex_conversation_query.py +8 -7
- package/bin/_lib_conversation.py +105 -5
- package/bin/_lib_conversation_dispatch.py +15 -4
- package/bin/_lib_conversation_query.py +294 -2
- package/bin/_lib_dashboard_sources.py +5 -1
- package/bin/_lib_doctor.py +202 -1
- package/bin/_lib_jsonl.py +12 -0
- package/bin/_lib_quota_alert_axes.py +188 -0
- package/bin/_lib_quota_ledger.py +274 -0
- package/bin/_lib_snapshot_cache.py +36 -0
- package/bin/cctally +6 -3
- package/dashboard/static/assets/index-BgoYXdus.js +92 -0
- package/dashboard/static/assets/index-Ub8vwz1M.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +4 -1
- package/dashboard/static/assets/index-B0ZCsoxI.css +0 -1
- package/dashboard/static/assets/index-Bvp8mxtz.js +0 -92
package/bin/_cctally_quota.py
CHANGED
|
@@ -9,13 +9,13 @@ stats generation from the physical cache.
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
import datetime as dt
|
|
12
|
-
import hashlib
|
|
13
12
|
import json
|
|
14
13
|
import secrets
|
|
15
14
|
import sqlite3
|
|
16
15
|
import sys
|
|
16
|
+
import time
|
|
17
17
|
from dataclasses import dataclass
|
|
18
|
-
from typing import Callable, Iterable, Mapping
|
|
18
|
+
from typing import Callable, Iterable, Mapping, Sequence
|
|
19
19
|
|
|
20
20
|
import _cctally_core
|
|
21
21
|
import _lib_accounts
|
|
@@ -56,11 +56,63 @@ from _lib_codex_pools import (
|
|
|
56
56
|
codex_model_scoped_quota_pool,
|
|
57
57
|
is_model_scoped_codex_quota,
|
|
58
58
|
)
|
|
59
|
+
import _lib_quota_alert_axes as _axes
|
|
60
|
+
import _lib_quota_ledger as _ledger
|
|
59
61
|
|
|
60
62
|
|
|
61
63
|
UTC = dt.timezone.utc
|
|
62
64
|
_DASHBOARD_PROJECTION_CERTIFICATE_KEY = "codex_quota_projection_certificate"
|
|
63
|
-
|
|
65
|
+
# 2 -> 3 (public #5): the root physical signature is now COMPOSED from per-group
|
|
66
|
+
# digests instead of digested over a whole root's observation tuples. The
|
|
67
|
+
# semantics are unchanged — still an exact-equality function of the physical
|
|
68
|
+
# evidence — but the VALUE is not, so every certificate written under the old
|
|
69
|
+
# scheme has to be rejected rather than compared. The version also invalidates
|
|
70
|
+
# the ledger mechanism itself: a classification change alters interpreted keys
|
|
71
|
+
# with no row mutation to observe, so a bump queues one complete pass.
|
|
72
|
+
_CODEX_QUOTA_INTERPRETATION_VERSION = 3
|
|
73
|
+
|
|
74
|
+
#: Above this many dirty loading units, a bounded pass stops being a saving: it
|
|
75
|
+
#: is one indexed query per unit (times the snap closure) against a single
|
|
76
|
+
#: unbounded scan, and the sweep's ``IN`` list stops fitting comfortably in
|
|
77
|
+
#: SQLite's variable budget. A burst that wide is a rebuild or a first ingest,
|
|
78
|
+
#: which is exactly what the full path is for.
|
|
79
|
+
_MAX_INCREMENTAL_UNITS = 128
|
|
80
|
+
|
|
81
|
+
#: Loading-unit keys per SQL ``IN`` chunk in the scoped sweep.
|
|
82
|
+
_SWEEP_KEY_CHUNK = 400
|
|
83
|
+
|
|
84
|
+
#: How long a projection may go without a whole-history verification pass
|
|
85
|
+
#: (spec §2). The scoped sweep structurally cannot see two classes — a block
|
|
86
|
+
#: whose physical group is absent from the cache entirely, and a milestone on a
|
|
87
|
+
#: historic root no longer active — so without a deadline they are repairable
|
|
88
|
+
#: only by an interpretation bump, a rebuild or a burst overflow, none of which
|
|
89
|
+
#: happen on a normal install. One full pass per day against the roughly four
|
|
90
|
+
#: seconds this design removes from every turn. EVERY full pass stamps the
|
|
91
|
+
#: deadline, whatever triggered it, so it is satisfied by whichever caller
|
|
92
|
+
#: reaches it first — a dashboard tick or a `codex quota` invocation pays the
|
|
93
|
+
#: cost off the hook path entirely — except on a hook-only install, where no
|
|
94
|
+
#: such caller exists. There the deadline would land on the blocking hook path
|
|
95
|
+
#: as the one unbounded operation this design otherwise removes, so the hook
|
|
96
|
+
#: passes ``full_pass="defer"`` and the pass is handed to the detached
|
|
97
|
+
#: ``_codex-quota-verify`` worker instead (see ``_defer_codex_quota_verification``).
|
|
98
|
+
CODEX_QUOTA_FULL_VERIFICATION_INTERVAL_SECONDS = 86400
|
|
99
|
+
|
|
100
|
+
#: The hidden self-subcommand that performs a deferred verification pass.
|
|
101
|
+
CODEX_QUOTA_VERIFY_COMMAND = "_codex-quota-verify"
|
|
102
|
+
|
|
103
|
+
#: Marker whose mtime throttles worker spawns. In ``APP_DIR`` rather than
|
|
104
|
+
#: cache.db because the decision is made with no cache write transaction open,
|
|
105
|
+
#: and because a spawn is process state, not projection state.
|
|
106
|
+
CODEX_QUOTA_VERIFY_MARKER_NAME = "codex-quota-verify.last-attempt"
|
|
107
|
+
|
|
108
|
+
#: Minimum spacing between deferred-verification spawns. Stamped on ATTEMPT,
|
|
109
|
+
#: not on success: ``last_full_pass_at`` moves only when a pass COMPLETES, so
|
|
110
|
+
#: every tick between the spawn and the worker's commit still reads as due, and
|
|
111
|
+
#: a success-stamped throttle would put one worker per hook tick on the box.
|
|
112
|
+
#: Comfortably longer than a measured full pass (~4s locally, ~15s on the
|
|
113
|
+
#: reporter's store) while still retrying many times inside the one-day
|
|
114
|
+
#: interval if a worker dies.
|
|
115
|
+
CODEX_QUOTA_VERIFY_SPAWN_THROTTLE_SECONDS = 600
|
|
64
116
|
|
|
65
117
|
|
|
66
118
|
@dataclass(frozen=True)
|
|
@@ -171,6 +223,7 @@ def _store_codex_quota_projection_certificate(
|
|
|
171
223
|
*,
|
|
172
224
|
sequence: int,
|
|
173
225
|
signatures: Mapping[str, str],
|
|
226
|
+
prune_ledger_through: "int | None" = None,
|
|
174
227
|
) -> None:
|
|
175
228
|
"""Stamp exact validated signatures only if cache physical state is unchanged.
|
|
176
229
|
|
|
@@ -178,6 +231,19 @@ def _store_codex_quota_projection_certificate(
|
|
|
178
231
|
A later cache mutation necessarily advances ``sequence``, so a dashboard
|
|
179
232
|
reader fails coherence rather than combining new physical cache data with
|
|
180
233
|
the prior projection certificate.
|
|
234
|
+
|
|
235
|
+
``prune_ledger_through`` (public #5) deletes consumed change-ledger entries
|
|
236
|
+
in the same transaction. Nothing else prunes them, and nothing bounds them:
|
|
237
|
+
a ``cache-sync --rebuild`` on a 211K-observation store wipes and re-ingests,
|
|
238
|
+
which the triggers record as roughly 422K rows (one delete plus one insert
|
|
239
|
+
each). Entries at or below the committed watermark are provably consumed —
|
|
240
|
+
the projection that consumed them is already durable — and ``seq`` is
|
|
241
|
+
``AUTOINCREMENT``, so a pruned high value is never reissued and the
|
|
242
|
+
watermark can never be overtaken from below.
|
|
243
|
+
|
|
244
|
+
The prune runs even when the certificate itself is declined: a sequence that
|
|
245
|
+
advanced mid-pass means new evidence landed, not that the old entries are
|
|
246
|
+
unconsumed.
|
|
181
247
|
"""
|
|
182
248
|
path = _cctally_core.CACHE_DB_PATH
|
|
183
249
|
if not path.exists():
|
|
@@ -186,8 +252,16 @@ def _store_codex_quota_projection_certificate(
|
|
|
186
252
|
conn = sqlite3.connect(path)
|
|
187
253
|
try:
|
|
188
254
|
conn.execute("BEGIN IMMEDIATE")
|
|
255
|
+
if prune_ledger_through:
|
|
256
|
+
try:
|
|
257
|
+
conn.execute(
|
|
258
|
+
"DELETE FROM quota_window_change_log WHERE seq <= ?",
|
|
259
|
+
(int(prune_ledger_through),),
|
|
260
|
+
)
|
|
261
|
+
except sqlite3.OperationalError:
|
|
262
|
+
pass # a cache too old to carry the ledger
|
|
189
263
|
if codex_physical_mutation_seq(conn) != sequence:
|
|
190
|
-
conn.
|
|
264
|
+
conn.commit()
|
|
191
265
|
return
|
|
192
266
|
payload = json.dumps({
|
|
193
267
|
"interpretationVersion": _CODEX_QUOTA_INTERPRETATION_VERSION,
|
|
@@ -219,19 +293,439 @@ def _stats_projection_signatures_match(
|
|
|
219
293
|
match for every active root before the reconcile is allowed to short-circuit.
|
|
220
294
|
A missing row, a mismatch, or any ``sqlite3.Error`` degrades to False, which
|
|
221
295
|
forces the full reconcile (fail-safe).
|
|
296
|
+
|
|
297
|
+
ACCOUNT-AWARE and ORDER-INDEPENDENT (public #5 Task 9). The signature is
|
|
298
|
+
per-root by construction, so every ``(root, account)`` row for a root must
|
|
299
|
+
carry the same value; collapsing the rows into one dictionary let whichever
|
|
300
|
+
row happened to come last decide, which is a real answer only when they
|
|
301
|
+
already agree. Requiring the root's rows to agree — and to exist — turns a
|
|
302
|
+
partially-updated projection from a coin flip into a mismatch, and a
|
|
303
|
+
mismatch is the fail-safe direction.
|
|
222
304
|
"""
|
|
223
305
|
try:
|
|
224
306
|
rows = stats_conn.execute(
|
|
225
|
-
"SELECT source_root_key, physical_signature
|
|
307
|
+
"SELECT source_root_key, account_key, physical_signature "
|
|
308
|
+
" FROM quota_projection_state"
|
|
226
309
|
).fetchall()
|
|
227
310
|
except sqlite3.Error:
|
|
228
311
|
return False
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
312
|
+
by_root: dict[str, set[str]] = {}
|
|
313
|
+
for row in rows:
|
|
314
|
+
by_root.setdefault(str(row[0]), set()).add(str(row[2]))
|
|
315
|
+
for root in active_roots:
|
|
316
|
+
stored = by_root.get(root)
|
|
317
|
+
if not stored or len(stored) != 1:
|
|
318
|
+
return False
|
|
319
|
+
if next(iter(stored)) != cert_sigs.get(root):
|
|
320
|
+
return False
|
|
321
|
+
return True
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
# ── the change ledger's watermark and the state that rides with it ─────────
|
|
325
|
+
|
|
326
|
+
def _ledger_state(stats_conn: sqlite3.Connection) -> dict | None:
|
|
327
|
+
"""Read the incremental projector's own state, or ``None``.
|
|
328
|
+
|
|
329
|
+
``None`` — a missing row, a missing table, any error — means the projector
|
|
330
|
+
has no consumed range it can trust and the next pass must be a complete one.
|
|
331
|
+
Fail-safe by construction: the expensive answer is the correct one.
|
|
332
|
+
"""
|
|
333
|
+
try:
|
|
334
|
+
row = stats_conn.execute(
|
|
335
|
+
"SELECT watermark_seq, interpretation_version, alerts_enabled, "
|
|
336
|
+
" next_evaluation_at_utc, last_full_pass_at "
|
|
337
|
+
" FROM quota_projection_ledger_state WHERE source='codex'"
|
|
338
|
+
).fetchone()
|
|
339
|
+
except sqlite3.Error:
|
|
340
|
+
return None
|
|
341
|
+
if row is None:
|
|
342
|
+
return None
|
|
343
|
+
try:
|
|
344
|
+
return {
|
|
345
|
+
"watermark": int(row[0]),
|
|
346
|
+
"interpretation_version": int(row[1]),
|
|
347
|
+
"alerts_enabled": (
|
|
348
|
+
None if row[2] is None else bool(int(row[2]))),
|
|
349
|
+
"next_evaluation_at": (
|
|
350
|
+
None if row[3] is None else str(row[3])),
|
|
351
|
+
"last_full_pass_at": (
|
|
352
|
+
None if row[4] is None else str(row[4])),
|
|
353
|
+
}
|
|
354
|
+
except (TypeError, ValueError):
|
|
355
|
+
return None
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _full_verification_due(ledger_state: "dict | None", now: dt.datetime) -> bool:
|
|
359
|
+
"""True when the projection is overdue for a whole-history pass (spec §2).
|
|
360
|
+
|
|
361
|
+
Fail-safe in the expensive direction at every step: no state, no stamp, an
|
|
362
|
+
unparsable stamp and a stamp in the future all read as due. A stamp ahead of
|
|
363
|
+
``now`` is a clock that moved backwards, and treating it as satisfied would
|
|
364
|
+
suspend the verification until wall time caught up.
|
|
365
|
+
"""
|
|
366
|
+
if ledger_state is None:
|
|
367
|
+
return True
|
|
368
|
+
stamp = ledger_state.get("last_full_pass_at")
|
|
369
|
+
if not stamp:
|
|
370
|
+
return True
|
|
371
|
+
try:
|
|
372
|
+
last = _parse_utc(str(stamp), "last_full_pass_at")
|
|
373
|
+
except (TypeError, ValueError):
|
|
374
|
+
return True
|
|
375
|
+
elapsed = (now - last).total_seconds()
|
|
376
|
+
if elapsed < 0:
|
|
377
|
+
return True
|
|
378
|
+
return elapsed >= CODEX_QUOTA_FULL_VERIFICATION_INTERVAL_SECONDS
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _log_codex_worker_outcome(
|
|
382
|
+
op: str, outcome: str, detail: str = "", *, error: str = "",
|
|
383
|
+
) -> None:
|
|
384
|
+
"""One durable line about a detached Codex worker, in ``hook-tick.log``.
|
|
385
|
+
|
|
386
|
+
The workers spawned from the hook path have all three streams on
|
|
387
|
+
``/dev/null`` and an exit code nobody observes, so this file is the only
|
|
388
|
+
place their outcome can land. Best-effort and never raising: a diagnostic
|
|
389
|
+
must not be able to fail the operation it describes.
|
|
390
|
+
|
|
391
|
+
``detail`` carries the caller's OWN structured ``k=v`` fragments — counts, a
|
|
392
|
+
duration, a fixed reason token — and is emitted verbatim. Anything derived
|
|
393
|
+
from an exception goes through ``error`` instead, which is rendered LAST and
|
|
394
|
+
defused HERE rather than at the call site. That is what makes the privacy
|
|
395
|
+
guarantee a property of the renderer, the way it already is of
|
|
396
|
+
``_codex_lifecycle_log_line``: no future caller can reintroduce a path, a
|
|
397
|
+
conversation id or a field separator by forgetting to scrub. A caller
|
|
398
|
+
holding the exception should still pass ``_hook_log_error_detail(exc)``,
|
|
399
|
+
which additionally narrows the ``OSError`` family's embedded ``filename``
|
|
400
|
+
away at the source.
|
|
401
|
+
"""
|
|
402
|
+
try:
|
|
403
|
+
from _cctally_record import (
|
|
404
|
+
_hook_log_safe_free_text, _hook_tick_log_line,
|
|
405
|
+
_hook_tick_log_rotate_if_needed,
|
|
406
|
+
)
|
|
407
|
+
stamp = _utc_iso(dt.datetime.now(UTC))
|
|
408
|
+
suffix = ""
|
|
409
|
+
if error:
|
|
410
|
+
safe = _hook_log_safe_free_text(error)
|
|
411
|
+
if safe:
|
|
412
|
+
suffix = f" error={safe}"
|
|
413
|
+
_hook_tick_log_line(
|
|
414
|
+
f"{stamp} provider=codex op={op} result={outcome}"
|
|
415
|
+
+ (f" {detail}" if detail else "") + suffix)
|
|
416
|
+
_hook_tick_log_rotate_if_needed()
|
|
417
|
+
except Exception:
|
|
418
|
+
pass
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _defer_codex_quota_verification() -> str:
|
|
422
|
+
"""Hand the periodic whole-history pass to a detached worker.
|
|
423
|
+
|
|
424
|
+
Returns ``"spawned"``, ``"throttled"`` or ``"failed"``, and WRITES the two
|
|
425
|
+
outcomes an operator would act on to ``hook-tick.log``. No caller
|
|
426
|
+
distinguishes them — every one of them skips the pass regardless, because a
|
|
427
|
+
missed pass is bounded staleness the next tick retries while running it
|
|
428
|
+
inline is the ~14-30s reconcile against Codex's 30-second hook timeout that
|
|
429
|
+
this whole change exists to remove. So logging here is what stops a
|
|
430
|
+
permanently failing hand-off from being invisible; the value stays a return
|
|
431
|
+
for tests.
|
|
432
|
+
|
|
433
|
+
``"throttled"`` is deliberately NOT logged: it is the ordinary state between
|
|
434
|
+
a spawn and the worker's commit, and the Codex lifecycle throttle is 15
|
|
435
|
+
seconds, so logging it would bury the two outcomes that matter.
|
|
436
|
+
|
|
437
|
+
Marker-first, exactly like ``update-check.last-fetch``: the mtime is stamped
|
|
438
|
+
BEFORE the spawn, so a worker that dies cannot make every following tick
|
|
439
|
+
spawn another one. If the marker itself cannot be written we do not spawn at
|
|
440
|
+
all — without it the spawn rate is unbounded, which is worse than a deferred
|
|
441
|
+
verification.
|
|
442
|
+
"""
|
|
443
|
+
marker = _cctally_core.APP_DIR / CODEX_QUOTA_VERIFY_MARKER_NAME
|
|
444
|
+
try:
|
|
445
|
+
age = time.time() - marker.stat().st_mtime
|
|
446
|
+
except OSError:
|
|
447
|
+
age = None
|
|
448
|
+
if age is not None and 0 <= age < CODEX_QUOTA_VERIFY_SPAWN_THROTTLE_SECONDS:
|
|
449
|
+
return "throttled"
|
|
450
|
+
try:
|
|
451
|
+
_cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
|
|
452
|
+
marker.touch()
|
|
453
|
+
except OSError:
|
|
454
|
+
_log_codex_worker_outcome(
|
|
455
|
+
"quota-verify-spawn", "failed", "reason=marker_unwritable")
|
|
456
|
+
return "failed"
|
|
457
|
+
from _cctally_update import _spawn_detached
|
|
458
|
+
if _spawn_detached(CODEX_QUOTA_VERIFY_COMMAND):
|
|
459
|
+
_log_codex_worker_outcome("quota-verify-spawn", "spawned")
|
|
460
|
+
return "spawned"
|
|
461
|
+
_log_codex_worker_outcome("quota-verify-spawn", "failed", "reason=spawn")
|
|
462
|
+
return "failed"
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def cmd_codex_quota_verify_internal(args) -> int:
|
|
466
|
+
"""Hidden ``_codex-quota-verify`` handler: the deferred whole-history pass.
|
|
467
|
+
|
|
468
|
+
Reporting only — no ``alert_eligible_root_keys``, because alert eligibility
|
|
469
|
+
belongs to whoever holds the per-root lifecycle lock and this worker holds
|
|
470
|
+
none. That matches every other off-hook caller of the verification (the
|
|
471
|
+
dashboard tick, ``codex quota``), which is the point: the worker is one of
|
|
472
|
+
those callers, moved off the blocking path.
|
|
473
|
+
|
|
474
|
+
``last_full_pass_at`` is stamped inside the pass's own stats transaction, so
|
|
475
|
+
a worker that is killed leaves the deadline due and the next tick retries.
|
|
476
|
+
Always returns 0 — a detached worker's exit code is observed by nobody, and
|
|
477
|
+
a raised exception would only produce an unread traceback.
|
|
478
|
+
|
|
479
|
+
The outcome goes to ``hook-tick.log``, following the ``_update-check``
|
|
480
|
+
precedent of writing ``update.log`` for exactly this reason. All three of
|
|
481
|
+
this worker's streams are ``/dev/null`` and its exit code is unobserved, so
|
|
482
|
+
a bare ``except: pass`` made a persistently failing verification completely
|
|
483
|
+
invisible — and because the deadline only moves when a pass COMMITS, such a
|
|
484
|
+
worker respawns every throttle window forever with nothing to show for it.
|
|
485
|
+
"""
|
|
486
|
+
started = time.monotonic()
|
|
487
|
+
|
|
488
|
+
def _log(outcome: str, detail: str = "", *, error: str = "") -> None:
|
|
489
|
+
_log_codex_worker_outcome(
|
|
490
|
+
"quota-verify", outcome,
|
|
491
|
+
f"dur_ms={max(0, int((time.monotonic() - started) * 1000))}"
|
|
492
|
+
+ (f" {detail}" if detail else ""),
|
|
493
|
+
error=error)
|
|
494
|
+
|
|
495
|
+
try:
|
|
496
|
+
result = reconcile_codex_quota_projection(force_full=True)
|
|
497
|
+
except Exception as exc:
|
|
498
|
+
# Same defusing as the lifecycle line, for the same two reasons: the
|
|
499
|
+
# `OSError` family's `str()` carries a rollout path, and the log's
|
|
500
|
+
# reader is a last-wins `k=v` comprehension a free-text `=` can beat.
|
|
501
|
+
# `_log_codex_worker_outcome` performs it; this narrows the `filename`
|
|
502
|
+
# away first, which only the caller holding the exception can do.
|
|
503
|
+
from _cctally_record import _hook_log_error_detail
|
|
504
|
+
_log("error", error=_hook_log_error_detail(exc))
|
|
505
|
+
return 0
|
|
506
|
+
_log(
|
|
507
|
+
"success",
|
|
508
|
+
f"blocks={int(getattr(result, 'blocks_upserted', 0) or 0)} "
|
|
509
|
+
f"milestones={int(getattr(result, 'milestones_upserted', 0) or 0)}")
|
|
510
|
+
return 0
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _store_ledger_state(
|
|
514
|
+
conn: sqlite3.Connection, *, watermark: int,
|
|
515
|
+
alerts_enabled: "bool | None", next_evaluation_at: "str | None",
|
|
516
|
+
last_full_pass_at: "str | None",
|
|
517
|
+
) -> None:
|
|
518
|
+
"""Stamp the consumed range, the non-dirtiness alert axes and the deadline.
|
|
519
|
+
|
|
520
|
+
Runs on the caller's transaction, alongside the projection it describes.
|
|
521
|
+
``last_full_pass_at`` is the caller's decision: a full pass passes its own
|
|
522
|
+
``now``, a bounded pass passes the stored value through unchanged.
|
|
523
|
+
"""
|
|
524
|
+
conn.execute(
|
|
525
|
+
"""INSERT INTO quota_projection_ledger_state
|
|
526
|
+
(source, watermark_seq, interpretation_version, alerts_enabled,
|
|
527
|
+
next_evaluation_at_utc, last_full_pass_at)
|
|
528
|
+
VALUES ('codex',?,?,?,?,?)
|
|
529
|
+
ON CONFLICT(source) DO UPDATE SET
|
|
530
|
+
watermark_seq=excluded.watermark_seq,
|
|
531
|
+
interpretation_version=excluded.interpretation_version,
|
|
532
|
+
alerts_enabled=excluded.alerts_enabled,
|
|
533
|
+
next_evaluation_at_utc=excluded.next_evaluation_at_utc,
|
|
534
|
+
last_full_pass_at=excluded.last_full_pass_at""",
|
|
535
|
+
(
|
|
536
|
+
int(watermark), _CODEX_QUOTA_INTERPRETATION_VERSION,
|
|
537
|
+
None if alerts_enabled is None else int(bool(alerts_enabled)),
|
|
538
|
+
next_evaluation_at, last_full_pass_at,
|
|
539
|
+
),
|
|
232
540
|
)
|
|
233
541
|
|
|
234
542
|
|
|
543
|
+
def _ledger_max_seq(cache_conn: sqlite3.Connection) -> int | None:
|
|
544
|
+
"""The ledger's high-water sequence, or ``None`` when unreadable.
|
|
545
|
+
|
|
546
|
+
``None`` (no table — a cache too old to carry the ledger) forces the full
|
|
547
|
+
path, which is exactly today's behaviour and correct, just not incremental.
|
|
548
|
+
|
|
549
|
+
Read from ``sqlite_sequence`` and not only from ``MAX(seq)``, because the
|
|
550
|
+
projector PRUNES consumed entries: after a prune the table is empty and
|
|
551
|
+
``MAX(seq)`` is 0, which would read as "the ledger was reset below the
|
|
552
|
+
watermark" on every single clean tick — and that reset detection is what
|
|
553
|
+
forces a whole-history pass. ``AUTOINCREMENT`` keeps its high-water in
|
|
554
|
+
``sqlite_sequence`` across a ``DELETE`` and only loses it when the table
|
|
555
|
+
itself is dropped and recreated, which is exactly the event being detected.
|
|
556
|
+
``MAX(seq)`` is still folded in so a cache whose sequence row is missing but
|
|
557
|
+
whose rows are not degrades to the safe direction.
|
|
558
|
+
"""
|
|
559
|
+
try:
|
|
560
|
+
row = cache_conn.execute(
|
|
561
|
+
"SELECT COALESCE(MAX(seq), 0) FROM quota_window_change_log"
|
|
562
|
+
).fetchone()
|
|
563
|
+
except sqlite3.Error:
|
|
564
|
+
return None
|
|
565
|
+
high = 0 if row is None else int(row[0])
|
|
566
|
+
try:
|
|
567
|
+
row = cache_conn.execute(
|
|
568
|
+
"SELECT seq FROM sqlite_sequence WHERE name='quota_window_change_log'"
|
|
569
|
+
).fetchone()
|
|
570
|
+
if row is not None and row[0] is not None:
|
|
571
|
+
high = max(high, int(row[0]))
|
|
572
|
+
except (sqlite3.Error, TypeError, ValueError):
|
|
573
|
+
pass
|
|
574
|
+
return high
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def _ledger_rows_after(
|
|
578
|
+
cache_conn: sqlite3.Connection, low: int, high: int,
|
|
579
|
+
) -> list[dict]:
|
|
580
|
+
"""Every ledger entry in ``(low, high]``, as plain dicts for the kernel."""
|
|
581
|
+
if high <= low:
|
|
582
|
+
return []
|
|
583
|
+
columns = ", ".join(
|
|
584
|
+
f"{side}{name}"
|
|
585
|
+
for side in ("old_", "new_")
|
|
586
|
+
for name in _ledger.LEDGER_GROUP_SUFFIXES
|
|
587
|
+
)
|
|
588
|
+
previous = cache_conn.row_factory
|
|
589
|
+
try:
|
|
590
|
+
cache_conn.row_factory = sqlite3.Row
|
|
591
|
+
return [
|
|
592
|
+
dict(row) for row in cache_conn.execute(
|
|
593
|
+
f"SELECT op, {columns} FROM quota_window_change_log "
|
|
594
|
+
" WHERE seq > ? AND seq <= ?", (int(low), int(high)),
|
|
595
|
+
)
|
|
596
|
+
]
|
|
597
|
+
except sqlite3.Error:
|
|
598
|
+
return []
|
|
599
|
+
finally:
|
|
600
|
+
cache_conn.row_factory = previous
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def _armed_identities(
|
|
604
|
+
stats_conn: sqlite3.Connection,
|
|
605
|
+
) -> dict[tuple, tuple[QuotaWindowIdentity, str]]:
|
|
606
|
+
"""Every persisted Codex arming boundary, keyed by its identity tuple.
|
|
607
|
+
|
|
608
|
+
This is where the policy axis reads its STORED fingerprints from: the arming
|
|
609
|
+
row already persists the resolved rule's hash per identity, so no second
|
|
610
|
+
store is needed to detect a rule change.
|
|
611
|
+
"""
|
|
612
|
+
try:
|
|
613
|
+
rows = stats_conn.execute(
|
|
614
|
+
"""SELECT source, source_root_key, account_key, logical_limit_key,
|
|
615
|
+
observed_slot, window_minutes, rule_fingerprint
|
|
616
|
+
FROM quota_alert_arming WHERE source='codex'"""
|
|
617
|
+
).fetchall()
|
|
618
|
+
except sqlite3.Error:
|
|
619
|
+
return {}
|
|
620
|
+
armed: dict[tuple, tuple[QuotaWindowIdentity, str]] = {}
|
|
621
|
+
for row in rows:
|
|
622
|
+
try:
|
|
623
|
+
identity = QuotaWindowIdentity(
|
|
624
|
+
source=str(row[0]), source_root_key=str(row[1]),
|
|
625
|
+
account_key=str(row[2]), logical_limit_key=str(row[3]),
|
|
626
|
+
observed_slot=str(row[4]), window_minutes=int(row[5]),
|
|
627
|
+
)
|
|
628
|
+
except (TypeError, ValueError):
|
|
629
|
+
continue
|
|
630
|
+
key = (
|
|
631
|
+
identity.source, identity.source_root_key, identity.account_key,
|
|
632
|
+
identity.logical_limit_key, identity.observed_slot,
|
|
633
|
+
identity.window_minutes,
|
|
634
|
+
)
|
|
635
|
+
armed[key] = (identity, str(row[6]))
|
|
636
|
+
return armed
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def _resolve_alert_scope(
|
|
640
|
+
stats_conn: sqlite3.Connection, *, ledger_scope: str, now: dt.datetime,
|
|
641
|
+
ledger_state: "dict | None", global_enabled: bool, quota_enabled: bool,
|
|
642
|
+
rules, config, defer_scheduled: bool = False,
|
|
643
|
+
) -> _axes.AlertDirtyScope:
|
|
644
|
+
"""Feed the five-axis kernel from stats.db and the resolved configuration."""
|
|
645
|
+
armed = _armed_identities(stats_conn)
|
|
646
|
+
stored = {key: fingerprint for key, (_ident, fingerprint) in armed.items()}
|
|
647
|
+
resolved = {}
|
|
648
|
+
for key, (identity, _fingerprint) in armed.items():
|
|
649
|
+
rule = resolve_quota_rule(
|
|
650
|
+
identity,
|
|
651
|
+
default_actual_thresholds=config["actual_thresholds"],
|
|
652
|
+
default_projected_thresholds=config["projected_thresholds"],
|
|
653
|
+
rules=rules,
|
|
654
|
+
)
|
|
655
|
+
resolved[key] = quota_rule_fingerprint(
|
|
656
|
+
identity, rule, global_enabled=global_enabled,
|
|
657
|
+
quota_enabled=quota_enabled,
|
|
658
|
+
)
|
|
659
|
+
boundary = None
|
|
660
|
+
if ledger_state is not None and ledger_state["next_evaluation_at"]:
|
|
661
|
+
try:
|
|
662
|
+
boundary = _parse_utc(
|
|
663
|
+
ledger_state["next_evaluation_at"], "next_evaluation_at_utc")
|
|
664
|
+
except (TypeError, ValueError):
|
|
665
|
+
boundary = None
|
|
666
|
+
return _axes.alert_dirty_scope(
|
|
667
|
+
ledger_groups=(1,) if ledger_scope == _axes.SCOPE_GROUPS else (),
|
|
668
|
+
stored_fingerprints=stored,
|
|
669
|
+
resolved_fingerprints=resolved,
|
|
670
|
+
gate_before=(
|
|
671
|
+
None if ledger_state is None else ledger_state["alerts_enabled"]),
|
|
672
|
+
gate_after=bool(global_enabled and quota_enabled),
|
|
673
|
+
now=now,
|
|
674
|
+
next_evaluation_at=boundary,
|
|
675
|
+
defer_scheduled=defer_scheduled,
|
|
676
|
+
)
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def _blocks_missing_reverse_map(stats_conn: sqlite3.Connection) -> bool:
|
|
680
|
+
"""True when any Codex block predates the reverse map.
|
|
681
|
+
|
|
682
|
+
A scoped sweep matches on ``physical_group_key``, so a NULL there would
|
|
683
|
+
silently escape it and the stale block would survive indefinitely. The
|
|
684
|
+
epoch rebuild (1005 for the column, 1006 as shipped) stamps every row, and
|
|
685
|
+
this is the guard that turns the
|
|
686
|
+
one shape it cannot reach — a block written by an older binary against an
|
|
687
|
+
already-current index — into a full pass rather than a missed one.
|
|
688
|
+
"""
|
|
689
|
+
try:
|
|
690
|
+
return bool(stats_conn.execute(
|
|
691
|
+
"SELECT 1 FROM quota_window_blocks "
|
|
692
|
+
" WHERE source='codex' AND physical_group_key IS NULL LIMIT 1"
|
|
693
|
+
).fetchone())
|
|
694
|
+
except sqlite3.Error:
|
|
695
|
+
return True
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
def _normalized_physical_group(group: object) -> tuple[object, ...]:
|
|
699
|
+
"""Coerce one caller-supplied physical group into its bound-parameter form.
|
|
700
|
+
|
|
701
|
+
``window_minutes`` is an INTEGER column, so a string would compare unequal
|
|
702
|
+
under SQLite's type affinity rules and select nothing; the reset is
|
|
703
|
+
normalized to an ISO string because the predicate wraps it in
|
|
704
|
+
``unixepoch()``, which accepts both the ``Z`` and ``+00:00`` spellings the
|
|
705
|
+
cache retains.
|
|
706
|
+
"""
|
|
707
|
+
try:
|
|
708
|
+
root, limit_key, slot, minutes, reset = group # type: ignore[misc]
|
|
709
|
+
except (TypeError, ValueError):
|
|
710
|
+
raise ValueError(
|
|
711
|
+
"physical_groups entries must be (source_root_key, "
|
|
712
|
+
"logical_limit_key, observed_slot, window_minutes, "
|
|
713
|
+
"canonical_reset) 5-tuples"
|
|
714
|
+
) from None
|
|
715
|
+
if isinstance(minutes, bool) or not isinstance(minutes, int):
|
|
716
|
+
raise ValueError("physical_groups window_minutes must be an integer")
|
|
717
|
+
if isinstance(reset, dt.datetime):
|
|
718
|
+
if reset.tzinfo is None or reset.utcoffset() is None:
|
|
719
|
+
raise ValueError("physical_groups reset must be timezone-aware")
|
|
720
|
+
reset = _utc_iso(reset)
|
|
721
|
+
return (str(root), str(limit_key), str(slot), minutes, str(reset))
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def _iter_shard_rows(conn, shards):
|
|
725
|
+
for shard_sql, shard_params in shards:
|
|
726
|
+
yield from conn.execute(shard_sql, shard_params)
|
|
727
|
+
|
|
728
|
+
|
|
235
729
|
def load_codex_quota_observations(
|
|
236
730
|
*,
|
|
237
731
|
source_root_keys: Iterable[str] | None = None,
|
|
@@ -241,6 +735,7 @@ def load_codex_quota_observations(
|
|
|
241
735
|
max_rows: int | None = None,
|
|
242
736
|
physical_signatures: dict[str, str] | None = None,
|
|
243
737
|
canonical_resets_between: "tuple[dt.datetime, dt.datetime] | None" = None,
|
|
738
|
+
physical_groups: "Iterable[tuple[str, str, str, int, str]] | None" = None,
|
|
244
739
|
) -> tuple[QuotaObservation, ...]:
|
|
245
740
|
"""Load only valid root-qualified S1 physical quota rows.
|
|
246
741
|
|
|
@@ -267,6 +762,33 @@ def load_codex_quota_observations(
|
|
|
267
762
|
continuity fold below therefore still sees each retained window whole. That
|
|
268
763
|
is what lets the ingest-side spend-adoption pass bound itself to the windows
|
|
269
764
|
one sync touched instead of materializing all history every hook tick.
|
|
765
|
+
|
|
766
|
+
``physical_groups`` (public #5) is the EXACT group filter the incremental
|
|
767
|
+
projector expands a dirty ledger entry with: an iterable of
|
|
768
|
+
``(source_root_key, logical_limit_key, observed_slot, window_minutes,
|
|
769
|
+
canonical reset ISO)`` tuples, each matched exactly in SQL by its own
|
|
770
|
+
indexed query. Passing an empty iterable selects nothing and returns ``()``;
|
|
771
|
+
``None`` is unbounded, i.e. today's behaviour.
|
|
772
|
+
|
|
773
|
+
It is deliberately NOT ``canonical_resets_between`` in disguise. That bound
|
|
774
|
+
is an inclusive RANGE over one dimension, so across sparse dirty groups it
|
|
775
|
+
loads everything between the extremes, and even a single reset instant pulls
|
|
776
|
+
in every unrelated limit key and slot that happens to share it. The range
|
|
777
|
+
stays for its spend-adoption caller and is not repurposed here.
|
|
778
|
+
|
|
779
|
+
The reset member matches ``COALESCE(canonical_resets_at_utc,
|
|
780
|
+
resets_at_utc)``, never the raw column. Measured on the real store, raw
|
|
781
|
+
grouping yields 4,064 windows where the canonical anchor yields 608 — the
|
|
782
|
+
true block count. Matching the raw column fragments every physical window
|
|
783
|
+
about sevenfold, silently, and nothing fails: the pass simply loads a
|
|
784
|
+
fraction of each group and materializes a wrong block rather than a stale
|
|
785
|
+
one.
|
|
786
|
+
|
|
787
|
+
The tuples are RAW stored coordinates. Interpretation (snapping a jittered
|
|
788
|
+
``window_minutes``, rewriting the limit key from ``observed_model``, folding
|
|
789
|
+
the account over the population) happens below, in Python, exactly as it
|
|
790
|
+
does on the unbounded path — so the caller is responsible for widening a
|
|
791
|
+
dirty group to every raw spelling that snaps onto it before asking for it.
|
|
270
792
|
"""
|
|
271
793
|
for name, value in (
|
|
272
794
|
("captured_at_or_after", captured_at_or_after), ("active_at", active_at),
|
|
@@ -294,6 +816,23 @@ def load_codex_quota_observations(
|
|
|
294
816
|
if max_rows is not None:
|
|
295
817
|
if not isinstance(max_rows, int) or isinstance(max_rows, bool) or max_rows <= 0:
|
|
296
818
|
raise ValueError("max_rows must be a positive integer or None")
|
|
819
|
+
group_filter: tuple[tuple[object, ...], ...] | None = None
|
|
820
|
+
if physical_groups is not None:
|
|
821
|
+
# Neither combination has a coherent meaning, and both would fail
|
|
822
|
+
# QUIETLY: `max_rows` appends its ORDER BY/LIMIT parameters after the
|
|
823
|
+
# group disjunction's, and `physical_signatures` must be accumulated
|
|
824
|
+
# from the COMPLETE root history or the certificate it stamps certifies
|
|
825
|
+
# a fraction of the evidence.
|
|
826
|
+
if max_rows is not None:
|
|
827
|
+
raise ValueError("physical_groups cannot be combined with max_rows")
|
|
828
|
+
if physical_signatures is not None:
|
|
829
|
+
raise ValueError(
|
|
830
|
+
"physical_groups cannot be combined with physical_signatures")
|
|
831
|
+
group_filter = tuple(sorted({
|
|
832
|
+
_normalized_physical_group(group) for group in physical_groups
|
|
833
|
+
}))
|
|
834
|
+
if not group_filter:
|
|
835
|
+
return ()
|
|
297
836
|
requested = None if source_root_keys is None else {str(key) for key in source_root_keys}
|
|
298
837
|
owns_conn = cache_conn is None
|
|
299
838
|
if owns_conn:
|
|
@@ -315,9 +854,6 @@ def load_codex_quota_observations(
|
|
|
315
854
|
}
|
|
316
855
|
return required <= columns
|
|
317
856
|
|
|
318
|
-
has_session_entries = has_columns(
|
|
319
|
-
"codex_session_entries", {"source_path", "line_offset", "model"},
|
|
320
|
-
)
|
|
321
857
|
has_observed_model = has_columns(
|
|
322
858
|
"quota_window_snapshots", {"observed_model"},
|
|
323
859
|
)
|
|
@@ -338,26 +874,21 @@ def load_codex_quota_observations(
|
|
|
338
874
|
"canonical_resets_at_utc" if has_anchor
|
|
339
875
|
else "NULL AS canonical_resets_at_utc"
|
|
340
876
|
)
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
#
|
|
348
|
-
#
|
|
349
|
-
#
|
|
350
|
-
#
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
if has_observed_model
|
|
355
|
-
else (
|
|
356
|
-
"quota_window_snapshots.observed_model"
|
|
357
|
-
if has_observed_model else fallback_model
|
|
358
|
-
)
|
|
877
|
+
# Public #5: the model is read from THIS table alone. There used to be a
|
|
878
|
+
# COALESCE onto the nearest preceding `codex_session_entries.model` at
|
|
879
|
+
# or before the snapshot's byte offset, for rows written before the
|
|
880
|
+
# column existed. That fallback made `quota_window_snapshots` an
|
|
881
|
+
# incomplete dependency set — an accounting row arriving later could
|
|
882
|
+
# move a window into a different model pool with no quota-row mutation
|
|
883
|
+
# for the change ledger to record — so cache migration 039 materialized
|
|
884
|
+
# exactly what it resolved and the fallback was removed. Ingest already
|
|
885
|
+
# stamps the sticky model onto every quota row it emits, so nothing
|
|
886
|
+
# forward-looking depended on it. A row with no determinable model stays
|
|
887
|
+
# NULL and reads as unscoped rather than being fabricated.
|
|
888
|
+
model_expr = (
|
|
889
|
+
"quota_window_snapshots.observed_model AS observed_model"
|
|
890
|
+
if has_observed_model else "NULL AS observed_model"
|
|
359
891
|
)
|
|
360
|
-
model_expr = f"{selected_model} AS observed_model"
|
|
361
892
|
sql = """
|
|
362
893
|
SELECT source, source_root_key, source_path, line_offset,
|
|
363
894
|
captured_at_utc, observed_slot, logical_limit_key, limit_id,
|
|
@@ -422,9 +953,40 @@ def load_codex_quota_observations(
|
|
|
422
953
|
params.append(max_rows)
|
|
423
954
|
else:
|
|
424
955
|
sql += " ORDER BY source_root_key, captured_at_utc, resets_at_utc, source_path, line_offset"
|
|
956
|
+
# ONE SHARD PER GROUP (public #5), not one disjunction over all of them.
|
|
957
|
+
# Measured on a 211K-row / 608-group store: an OR over the five-member
|
|
958
|
+
# equality gives up and SCANs the table (2 groups 45.7ms, 3 groups
|
|
959
|
+
# 60.3ms, 8 groups 114.3ms), while the same groups as separate queries
|
|
960
|
+
# each seek `idx_qws_physical_group` and cost 0.6ms apiece (2 groups
|
|
961
|
+
# 1.3ms, 3 groups 1.8ms, 8 groups 4.8ms). The disjunction is therefore
|
|
962
|
+
# O(all history) — exactly the property this whole change removes — and
|
|
963
|
+
# the shape that keeps the pass proportional to the change is the boring
|
|
964
|
+
# one. It also keeps the bound-variable count trivially inside SQLite's
|
|
965
|
+
# 999 ceiling however many groups a burst dirties.
|
|
966
|
+
#
|
|
967
|
+
# The unbounded path stays a single shard with no extra predicate, so
|
|
968
|
+
# its SQL and its plan are byte-identical to before.
|
|
969
|
+
reset_group_expr = (
|
|
970
|
+
"COALESCE(canonical_resets_at_utc, resets_at_utc)"
|
|
971
|
+
if has_anchor else "resets_at_utc"
|
|
972
|
+
)
|
|
973
|
+
shards: list[tuple[str, tuple[object, ...]]] = []
|
|
974
|
+
if group_filter is None:
|
|
975
|
+
shards.append((sql, tuple(params)))
|
|
976
|
+
else:
|
|
977
|
+
head, _, tail = sql.partition(" ORDER BY ")
|
|
978
|
+
order_by = " ORDER BY " + tail if tail else ""
|
|
979
|
+
clause = (
|
|
980
|
+
" AND (source_root_key=? AND logical_limit_key=? "
|
|
981
|
+
"AND observed_slot=? AND window_minutes=? "
|
|
982
|
+
f"AND unixepoch({reset_group_expr})=unixepoch(?))"
|
|
983
|
+
)
|
|
984
|
+
shard_sql = f"{head}{clause}{order_by}"
|
|
985
|
+
for group in group_filter:
|
|
986
|
+
shards.append((shard_sql, (*params, *group)))
|
|
425
987
|
result: list[QuotaObservation] = []
|
|
426
|
-
signature_tuples: dict[str, list[tuple[object, ...]]] = {}
|
|
427
|
-
for row in conn
|
|
988
|
+
signature_tuples: dict[str, dict[str, list[tuple[object, ...]]]] = {}
|
|
989
|
+
for row in _iter_shard_rows(conn, shards):
|
|
428
990
|
required_text = (
|
|
429
991
|
"source", "source_root_key", "source_path", "captured_at_utc",
|
|
430
992
|
"observed_slot", "logical_limit_key", "resets_at_utc",
|
|
@@ -493,15 +1055,16 @@ def load_codex_quota_observations(
|
|
|
493
1055
|
# must not suppress unrelated valid windows or accounting.
|
|
494
1056
|
continue
|
|
495
1057
|
if physical_signatures is not None:
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
1058
|
+
# Accumulated PER LOADING UNIT, not per root: the root signature
|
|
1059
|
+
# is a digest over its sorted (unit key, unit digest) pairs, so
|
|
1060
|
+
# a flat per-root list would produce a different value from the
|
|
1061
|
+
# one the projector composes and stores — and the certificate
|
|
1062
|
+
# compares them with `==`.
|
|
1063
|
+
signature_tuples.setdefault(
|
|
1064
|
+
identity.source_root_key, {},
|
|
1065
|
+
).setdefault(
|
|
1066
|
+
_observation_unit_text(observation), [],
|
|
1067
|
+
).append(_signature_tuple(observation))
|
|
505
1068
|
if (
|
|
506
1069
|
captured_at_or_after is not None
|
|
507
1070
|
and observation.captured_at < captured_at_or_after
|
|
@@ -509,6 +1072,17 @@ def load_codex_quota_observations(
|
|
|
509
1072
|
):
|
|
510
1073
|
continue
|
|
511
1074
|
result.append(observation)
|
|
1075
|
+
if group_filter is not None and len(shards) > 1:
|
|
1076
|
+
# Each shard is ordered internally; their union is not. Restore the
|
|
1077
|
+
# unbounded path's total order so a bounded load is byte-comparable
|
|
1078
|
+
# with a full one.
|
|
1079
|
+
result.sort(key=lambda observation: (
|
|
1080
|
+
observation.identity.source_root_key,
|
|
1081
|
+
observation.captured_at,
|
|
1082
|
+
observation.resets_at,
|
|
1083
|
+
observation.source_path,
|
|
1084
|
+
observation.line_offset,
|
|
1085
|
+
))
|
|
512
1086
|
# Window-account continuity fold (#341 spec §2): adopt unidentified
|
|
513
1087
|
# observations into a same-physical-window identified account (exactly
|
|
514
1088
|
# one). Physical signatures above are account-independent (computed from
|
|
@@ -520,12 +1094,11 @@ def load_codex_quota_observations(
|
|
|
520
1094
|
physical_signatures.clear()
|
|
521
1095
|
roots = requested if requested is not None else set(signature_tuples)
|
|
522
1096
|
for root_key in roots:
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
)
|
|
528
|
-
physical_signatures[root_key] = hashlib.sha256(encoded).hexdigest()
|
|
1097
|
+
physical_signatures[root_key] = _ledger.compose_root_signature(
|
|
1098
|
+
(unit, _ledger.group_digest(tuples))
|
|
1099
|
+
for unit, tuples in signature_tuples.get(
|
|
1100
|
+
root_key, {}).items()
|
|
1101
|
+
)
|
|
529
1102
|
if captured_at_or_after is not None or max_rows is not None:
|
|
530
1103
|
return load_codex_quota_observations(
|
|
531
1104
|
source_root_keys=requested,
|
|
@@ -577,27 +1150,141 @@ def _historic_root_keys(conn: sqlite3.Connection) -> set[str]:
|
|
|
577
1150
|
return roots
|
|
578
1151
|
|
|
579
1152
|
|
|
580
|
-
def
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
1153
|
+
def _signature_tuple(observation: QuotaObservation) -> tuple[object, ...]:
|
|
1154
|
+
"""The per-observation tuple both the group digest and the old whole-root
|
|
1155
|
+
signature are taken over. Unchanged, so a group's digest covers exactly the
|
|
1156
|
+
rows it owns."""
|
|
1157
|
+
return (
|
|
1158
|
+
observation.identity.source_root_key,
|
|
1159
|
+
observation.identity.logical_limit_key,
|
|
1160
|
+
_utc_iso(observation.captured_at),
|
|
1161
|
+
observation.source_path,
|
|
1162
|
+
observation.line_offset,
|
|
1163
|
+
observation.used_percent,
|
|
1164
|
+
_utc_iso(observation.resets_at),
|
|
1165
|
+
)
|
|
1166
|
+
|
|
1167
|
+
|
|
1168
|
+
def _observation_unit_text(observation: QuotaObservation) -> str:
|
|
1169
|
+
"""The serialized loading unit one observation belongs to.
|
|
1170
|
+
|
|
1171
|
+
The unit is the reverse map: a ledger entry names it from RAW coordinates
|
|
1172
|
+
while a block stamps it from the INTERPRETED identity, and the two must
|
|
1173
|
+
agree or the scoped sweep looks for a key nothing wrote.
|
|
1174
|
+
"""
|
|
1175
|
+
identity = observation.identity
|
|
1176
|
+
anchor = observation.canonical_resets_at or observation.resets_at
|
|
1177
|
+
return _ledger.physical_group_key_text(_ledger.loading_unit_from_identity(
|
|
1178
|
+
source_root_key=identity.source_root_key,
|
|
1179
|
+
logical_limit_key=identity.logical_limit_key,
|
|
1180
|
+
observed_slot=identity.observed_slot,
|
|
1181
|
+
window_minutes=identity.window_minutes,
|
|
1182
|
+
canonical_reset_iso=_utc_iso(anchor),
|
|
1183
|
+
))
|
|
1184
|
+
|
|
1185
|
+
|
|
1186
|
+
def _block_unit_text(block: QuotaBlock) -> str:
|
|
1187
|
+
identity = block.identity
|
|
1188
|
+
return _ledger.physical_group_key_text(_ledger.loading_unit_from_identity(
|
|
1189
|
+
source_root_key=identity.source_root_key,
|
|
1190
|
+
logical_limit_key=identity.logical_limit_key,
|
|
1191
|
+
observed_slot=identity.observed_slot,
|
|
1192
|
+
window_minutes=identity.window_minutes,
|
|
1193
|
+
# `QuotaBlock.resets_at` IS the canonical anchor (#416 §4.1).
|
|
1194
|
+
canonical_reset_iso=_utc_iso(block.resets_at),
|
|
1195
|
+
))
|
|
1196
|
+
|
|
1197
|
+
|
|
1198
|
+
def _group_digests(
|
|
1199
|
+
observations: Iterable[QuotaObservation],
|
|
1200
|
+
) -> dict[str, str]:
|
|
1201
|
+
"""Digest every loading unit present in ``observations``.
|
|
1202
|
+
|
|
1203
|
+
Only the units the pass LOADED appear, which is exactly right: a bounded
|
|
1204
|
+
pass re-derives the dirty units' digests and leaves every clean unit's
|
|
1205
|
+
stored value alone, and the root signature is then composed from the union.
|
|
1206
|
+
"""
|
|
1207
|
+
by_unit: dict[str, list[tuple[object, ...]]] = {}
|
|
1208
|
+
for observation in observations:
|
|
1209
|
+
by_unit.setdefault(_observation_unit_text(observation), []).append(
|
|
1210
|
+
_signature_tuple(observation))
|
|
1211
|
+
return {
|
|
1212
|
+
unit: _ledger.group_digest(tuples) for unit, tuples in by_unit.items()
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
|
|
1216
|
+
def _signature(
|
|
1217
|
+
observations: Iterable[QuotaObservation], source_root_key: str,
|
|
1218
|
+
) -> str:
|
|
1219
|
+
"""One root's physical signature, computed straight from observations.
|
|
1220
|
+
|
|
1221
|
+
The whole-history spelling of the composition: digest each loading unit the
|
|
1222
|
+
root's observations fall into, then compose the root value from those pairs.
|
|
1223
|
+
For a complete observation set this equals what the projector composes from
|
|
1224
|
+
the STORED per-group digests, which is the property that lets a bounded pass
|
|
1225
|
+
and a whole-history pass agree on the certificate.
|
|
1226
|
+
|
|
1227
|
+
Retained (rather than folded into the projector) because it is the honest
|
|
1228
|
+
way for a caller holding observations — a test, a coherence probe — to ask
|
|
1229
|
+
"what should this root's signature be", without going through the blocks.
|
|
1230
|
+
"""
|
|
1231
|
+
digests = _group_digests(
|
|
1232
|
+
observation for observation in observations
|
|
592
1233
|
if observation.identity.source_root_key == source_root_key
|
|
1234
|
+
)
|
|
1235
|
+
return _ledger.compose_root_signature(digests.items())
|
|
1236
|
+
|
|
1237
|
+
|
|
1238
|
+
def _root_group_pairs(
|
|
1239
|
+
conn: sqlite3.Connection, source_root_key: str,
|
|
1240
|
+
) -> list[tuple[str, str]]:
|
|
1241
|
+
"""The live ``(group key, group digest)`` pairs of one root.
|
|
1242
|
+
|
|
1243
|
+
Read from the blocks rather than from the pass's observations, so a bounded
|
|
1244
|
+
pass composes over the same complete set a whole-history pass does. Orphaned
|
|
1245
|
+
blocks are excluded, which is what makes a swept-to-nothing group drop out
|
|
1246
|
+
of the root's signature with no separate bookkeeping.
|
|
1247
|
+
"""
|
|
1248
|
+
return [
|
|
1249
|
+
(str(row[0]), str(row[1]))
|
|
1250
|
+
for row in conn.execute(
|
|
1251
|
+
"SELECT DISTINCT physical_group_key, physical_group_digest "
|
|
1252
|
+
" FROM quota_window_blocks "
|
|
1253
|
+
" WHERE source='codex' AND source_root_key=? "
|
|
1254
|
+
" AND orphaned_at IS NULL AND physical_group_key IS NOT NULL "
|
|
1255
|
+
" AND physical_group_digest IS NOT NULL",
|
|
1256
|
+
(source_root_key,),
|
|
1257
|
+
)
|
|
593
1258
|
]
|
|
594
|
-
encoded = json.dumps(
|
|
595
|
-
sorted(tuples), ensure_ascii=False, separators=(",", ":"),
|
|
596
|
-
).encode("utf-8")
|
|
597
|
-
return hashlib.sha256(encoded).hexdigest()
|
|
598
1259
|
|
|
599
1260
|
|
|
600
|
-
def
|
|
1261
|
+
def _root_accounts(
|
|
1262
|
+
conn: sqlite3.Connection, source_root_key: str,
|
|
1263
|
+
) -> set[str]:
|
|
1264
|
+
"""The account partitions one root currently projects.
|
|
1265
|
+
|
|
1266
|
+
Derived from the live blocks, not from the pass's observations (#341 +
|
|
1267
|
+
public #5 Task 9): under a bounded pass an account whose only evidence lies
|
|
1268
|
+
in a CLEAN window contributes no observation, so an observation-derived set
|
|
1269
|
+
would drop it and its projection-state row would be left stale. The blocks
|
|
1270
|
+
are the materialized truth, and a retired partition loses its blocks to the
|
|
1271
|
+
sweep, so this both keeps and retires the right rows.
|
|
1272
|
+
"""
|
|
1273
|
+
return {
|
|
1274
|
+
str(row[0])
|
|
1275
|
+
for row in conn.execute(
|
|
1276
|
+
"SELECT DISTINCT account_key FROM quota_window_blocks "
|
|
1277
|
+
" WHERE source='codex' AND source_root_key=? "
|
|
1278
|
+
" AND orphaned_at IS NULL",
|
|
1279
|
+
(source_root_key,),
|
|
1280
|
+
)
|
|
1281
|
+
if row[0] is not None
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
|
|
1285
|
+
def _block_params(
|
|
1286
|
+
block: QuotaBlock, generation: str, unit: str, digest: str,
|
|
1287
|
+
) -> tuple[object, ...]:
|
|
601
1288
|
latest = block.observations[-1]
|
|
602
1289
|
identity = block.identity
|
|
603
1290
|
return (
|
|
@@ -606,7 +1293,7 @@ def _block_params(block: QuotaBlock, generation: str) -> tuple[object, ...]:
|
|
|
606
1293
|
identity.limit_name, _utc_iso(block.resets_at), _utc_iso(block.nominal_start_at),
|
|
607
1294
|
_utc_iso(block.first_observed_at), _utc_iso(block.last_observed_at),
|
|
608
1295
|
block.first_percent, block.current_percent, latest.source_path,
|
|
609
|
-
latest.line_offset, generation, identity.account_key,
|
|
1296
|
+
latest.line_offset, generation, identity.account_key, unit, digest,
|
|
610
1297
|
)
|
|
611
1298
|
|
|
612
1299
|
|
|
@@ -618,8 +1305,9 @@ _BLOCK_UPSERT = """
|
|
|
618
1305
|
(source, source_root_key, logical_limit_key, observed_slot,
|
|
619
1306
|
window_minutes, limit_id, limit_name, resets_at_utc, nominal_start_at_utc,
|
|
620
1307
|
first_observed_at_utc, last_observed_at_utc, first_percent, current_percent,
|
|
621
|
-
last_source_path, last_line_offset, generation, orphaned_at, account_key
|
|
622
|
-
|
|
1308
|
+
last_source_path, last_line_offset, generation, orphaned_at, account_key,
|
|
1309
|
+
physical_group_key, physical_group_digest)
|
|
1310
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,NULL,?,?,?)
|
|
623
1311
|
ON CONFLICT(source, source_root_key, account_key, logical_limit_key,
|
|
624
1312
|
observed_slot, window_minutes, resets_at_utc) DO UPDATE SET
|
|
625
1313
|
limit_id=excluded.limit_id, limit_name=excluded.limit_name,
|
|
@@ -628,7 +1316,9 @@ _BLOCK_UPSERT = """
|
|
|
628
1316
|
last_observed_at_utc=excluded.last_observed_at_utc,
|
|
629
1317
|
first_percent=excluded.first_percent, current_percent=excluded.current_percent,
|
|
630
1318
|
last_source_path=excluded.last_source_path, last_line_offset=excluded.last_line_offset,
|
|
631
|
-
generation=excluded.generation, orphaned_at=NULL
|
|
1319
|
+
generation=excluded.generation, orphaned_at=NULL,
|
|
1320
|
+
physical_group_key=excluded.physical_group_key,
|
|
1321
|
+
physical_group_digest=excluded.physical_group_digest
|
|
632
1322
|
"""
|
|
633
1323
|
|
|
634
1324
|
|
|
@@ -660,7 +1350,55 @@ def _milestone_params(
|
|
|
660
1350
|
)
|
|
661
1351
|
|
|
662
1352
|
|
|
663
|
-
|
|
1353
|
+
#: Restricts a sweep statement to blocks whose loading unit is dirty. Chunked by
|
|
1354
|
+
#: the caller so the ``IN`` list stays inside SQLite's variable budget.
|
|
1355
|
+
_UNIT_SCOPE_BLOCKS = " AND physical_group_key IN ({placeholders})"
|
|
1356
|
+
|
|
1357
|
+
#: The same restriction for the child tables, expressed through the block that
|
|
1358
|
+
#: owns the row. Blocks are ORPHANED, never deleted, so the join still resolves
|
|
1359
|
+
#: for a window whose members all disappeared — which is precisely the case the
|
|
1360
|
+
#: sweep exists to catch.
|
|
1361
|
+
_UNIT_SCOPE_VIA_BLOCK = """
|
|
1362
|
+
AND EXISTS (SELECT 1 FROM quota_window_blocks AS scope
|
|
1363
|
+
WHERE scope.source={alias}.source
|
|
1364
|
+
AND scope.source_root_key={alias}.source_root_key
|
|
1365
|
+
AND scope.account_key={alias}.account_key
|
|
1366
|
+
AND scope.logical_limit_key={alias}.logical_limit_key
|
|
1367
|
+
AND scope.observed_slot={alias}.observed_slot
|
|
1368
|
+
AND scope.window_minutes={alias}.window_minutes
|
|
1369
|
+
AND scope.resets_at_utc={alias}.resets_at_utc
|
|
1370
|
+
AND scope.physical_group_key IN ({placeholders}))
|
|
1371
|
+
"""
|
|
1372
|
+
|
|
1373
|
+
|
|
1374
|
+
def _chunk(values: Sequence[str], size: int):
|
|
1375
|
+
for start in range(0, len(values), size):
|
|
1376
|
+
yield values[start:start + size]
|
|
1377
|
+
|
|
1378
|
+
|
|
1379
|
+
def _orphan_unseen(
|
|
1380
|
+
conn: sqlite3.Connection, roots: set[str], generation: str, now_iso: str,
|
|
1381
|
+
*, units: "frozenset[str] | None" = None,
|
|
1382
|
+
) -> tuple[int, int]:
|
|
1383
|
+
"""Orphan whatever this generation did not re-stamp.
|
|
1384
|
+
|
|
1385
|
+
``units`` is public #5's scoping (spec §2). ``None`` keeps the whole-root
|
|
1386
|
+
sweep, which is what the full path, a rebuild and an interpretation-version
|
|
1387
|
+
bump all want. A non-``None`` set runs the SAME SQL over a bounded root set
|
|
1388
|
+
instead of a different sweep, which is what preserves the child classes a
|
|
1389
|
+
block-only set difference would miss: a milestone threshold that disappears
|
|
1390
|
+
inside a still-present window, an account-specific block variant that
|
|
1391
|
+
vanishes while the physical window remains, and the account-qualified
|
|
1392
|
+
``quota_threshold_events`` orphan/unorphan reconciliation.
|
|
1393
|
+
|
|
1394
|
+
Two cases a scoped sweep structurally cannot see are handled on the full
|
|
1395
|
+
path only, which runs on an interpretation bump, on rebuild and on
|
|
1396
|
+
``force_full``: blocks whose physical group is absent from the cache
|
|
1397
|
+
entirely, and milestones on historic roots no longer active.
|
|
1398
|
+
``_historic_root_keys`` continues to feed that path.
|
|
1399
|
+
"""
|
|
1400
|
+
if units is not None:
|
|
1401
|
+
return _orphan_unseen_scoped(conn, units, generation, now_iso)
|
|
664
1402
|
if not roots:
|
|
665
1403
|
return (0, 0)
|
|
666
1404
|
placeholders = ",".join("?" for _ in roots)
|
|
@@ -704,6 +1442,63 @@ def _orphan_unseen(conn: sqlite3.Connection, roots: set[str], generation: str, n
|
|
|
704
1442
|
return (int(blocks), int(milestones))
|
|
705
1443
|
|
|
706
1444
|
|
|
1445
|
+
def _orphan_unseen_scoped(
|
|
1446
|
+
conn: sqlite3.Connection, units: "frozenset[str]", generation: str,
|
|
1447
|
+
now_iso: str,
|
|
1448
|
+
) -> tuple[int, int]:
|
|
1449
|
+
"""The whole-root sweep, bounded to a set of dirty loading units.
|
|
1450
|
+
|
|
1451
|
+
Same three statements, same semantics, one extra predicate each. The child
|
|
1452
|
+
tables are scoped THROUGH the block that owns them rather than by a column
|
|
1453
|
+
of their own: the block's identity columns are the milestone's and the
|
|
1454
|
+
event's, blocks are orphaned rather than deleted, so the join keeps
|
|
1455
|
+
resolving for a window whose members all disappeared.
|
|
1456
|
+
"""
|
|
1457
|
+
if not units:
|
|
1458
|
+
return (0, 0)
|
|
1459
|
+
keys = sorted(units)
|
|
1460
|
+
blocks = 0
|
|
1461
|
+
milestones = 0
|
|
1462
|
+
for chunk in _chunk(keys, _SWEEP_KEY_CHUNK):
|
|
1463
|
+
placeholders = ",".join("?" for _ in chunk)
|
|
1464
|
+
blocks += conn.execute(
|
|
1465
|
+
"UPDATE quota_window_blocks SET orphaned_at=COALESCE(orphaned_at, ?) "
|
|
1466
|
+
"WHERE source='codex' AND generation<>?"
|
|
1467
|
+
+ _UNIT_SCOPE_BLOCKS.format(placeholders=placeholders),
|
|
1468
|
+
(now_iso, generation, *chunk),
|
|
1469
|
+
).rowcount
|
|
1470
|
+
milestones += conn.execute(
|
|
1471
|
+
"UPDATE quota_percent_milestones AS milestones "
|
|
1472
|
+
"SET orphaned_at=COALESCE(orphaned_at, ?) "
|
|
1473
|
+
"WHERE milestones.source='codex' AND milestones.generation<>?"
|
|
1474
|
+
+ _UNIT_SCOPE_VIA_BLOCK.format(
|
|
1475
|
+
alias="milestones", placeholders=placeholders),
|
|
1476
|
+
(now_iso, generation, *chunk),
|
|
1477
|
+
).rowcount
|
|
1478
|
+
# Terminal evidence: never recreated, only marked. Same CASE as the
|
|
1479
|
+
# whole-root sweep — the account-qualified join included — restricted to
|
|
1480
|
+
# events whose owning block sits in a dirty unit.
|
|
1481
|
+
conn.execute(
|
|
1482
|
+
"UPDATE quota_threshold_events AS events "
|
|
1483
|
+
" SET orphaned_at=CASE WHEN EXISTS ("
|
|
1484
|
+
" SELECT 1 FROM quota_window_blocks AS blocks "
|
|
1485
|
+
" WHERE blocks.source=events.source "
|
|
1486
|
+
" AND blocks.source_root_key=events.source_root_key "
|
|
1487
|
+
" AND blocks.account_key=events.account_key "
|
|
1488
|
+
" AND blocks.logical_limit_key=events.logical_limit_key "
|
|
1489
|
+
" AND blocks.observed_slot=events.observed_slot "
|
|
1490
|
+
" AND blocks.window_minutes=events.window_minutes "
|
|
1491
|
+
" AND blocks.resets_at_utc=events.resets_at_utc "
|
|
1492
|
+
" AND blocks.generation=?"
|
|
1493
|
+
" ) THEN NULL ELSE COALESCE(events.orphaned_at, ?) END "
|
|
1494
|
+
" WHERE events.source='codex'"
|
|
1495
|
+
+ _UNIT_SCOPE_VIA_BLOCK.format(
|
|
1496
|
+
alias="events", placeholders=placeholders),
|
|
1497
|
+
(generation, now_iso, *chunk),
|
|
1498
|
+
)
|
|
1499
|
+
return (int(blocks), int(milestones))
|
|
1500
|
+
|
|
1501
|
+
|
|
707
1502
|
def _quota_alert_config() -> tuple[bool, bool, tuple[QuotaRule, ...], dict]:
|
|
708
1503
|
"""Resolve global + quota gates and exact JSON-shaped overrides once."""
|
|
709
1504
|
c = _cctally()
|
|
@@ -1111,7 +1906,10 @@ def _reanchor_terminal_events(conn: sqlite3.Connection, block) -> None:
|
|
|
1111
1906
|
def _apply_quota_projection_rows(
|
|
1112
1907
|
conn, *, observations, active_roots, now, now_iso,
|
|
1113
1908
|
sink, alert_eligible_roots, journal_emit=None, journal_disarm=None,
|
|
1114
|
-
journal_terminal=None, holder=None,
|
|
1909
|
+
journal_terminal=None, holder=None, dirty_units=None,
|
|
1910
|
+
ledger_watermark=None, alerts_enabled=None,
|
|
1911
|
+
stored_next_evaluation_at=None, stored_last_full_pass_at=None,
|
|
1912
|
+
stored_alerts_enabled=None, consume_alert_axes=True,
|
|
1115
1913
|
):
|
|
1116
1914
|
"""Transaction-neutral quota projection apply (spec §5.3 "projection").
|
|
1117
1915
|
|
|
@@ -1123,23 +1921,59 @@ def _apply_quota_projection_rows(
|
|
|
1123
1921
|
emitter) AND the rebuild re-materialization pass (``sink=None``,
|
|
1124
1922
|
``journal_emit=None``, ``alert_eligible_roots`` empty) so the two never drift.
|
|
1125
1923
|
``holder`` (optional) captures the certificate signatures + result for the
|
|
1126
|
-
live caller; rebuild passes ``None``.
|
|
1924
|
+
live caller; rebuild passes ``None``.
|
|
1925
|
+
|
|
1926
|
+
``dirty_units`` (public #5) is the BOUNDED pass: a set of serialized loading
|
|
1927
|
+
units whose complete current membership ``observations`` carries. ``None`` is
|
|
1928
|
+
the whole-history pass — every other caller, the rebuild, an
|
|
1929
|
+
interpretation-version bump and ``force_full``. It changes exactly two
|
|
1930
|
+
things: the sweep is scoped to those units, and the root signature is
|
|
1931
|
+
composed from the stored per-group digests rather than recomputed from
|
|
1932
|
+
scratch. Everything else runs identically, which is what keeps the two paths
|
|
1933
|
+
from drifting.
|
|
1934
|
+
|
|
1935
|
+
``consume_alert_axes`` says whether this pass may ADVANCE the two
|
|
1936
|
+
non-dirtiness alert axes it stores (``alerts_enabled``,
|
|
1937
|
+
``next_evaluation_at``) or must carry the stored values through untouched,
|
|
1938
|
+
the way ``last_full_pass_at`` is carried through by a bounded pass. False
|
|
1939
|
+
means "this pass did not do the work those axes exist to trigger", and there
|
|
1940
|
+
are two such passes. A REPORTING-ONLY pass (no alert-eligible roots — the
|
|
1941
|
+
``_codex-quota-verify`` worker, the dashboard tick, every ``codex quota``
|
|
1942
|
+
invocation) returns from ``_evaluate_quota_alerts`` at
|
|
1943
|
+
``if not alert_eligible_roots`` before a single threshold is examined, so
|
|
1944
|
+
stamping the gate would retire a delivery-gate ENABLE with no arming row and
|
|
1945
|
+
no ``suppressed_backfill`` written, and the axis could never re-fire because
|
|
1946
|
+
``gate_before`` now reads True. A hook tick that DEFERRED axis 4 is the
|
|
1947
|
+
other. ``stored_alerts_enabled`` / ``stored_next_evaluation_at`` are what it
|
|
1948
|
+
carries through.
|
|
1949
|
+
|
|
1950
|
+
``ledger_watermark`` is stamped INSIDE this transaction. That is deliberate:
|
|
1951
|
+
``run_stats_ingest`` is the sole stats writer, so advancing it after the
|
|
1952
|
+
commit would need a second cycle — and atomicity is the stronger guarantee
|
|
1953
|
+
anyway. A crash replays the range, which is safe because re-materializing a
|
|
1954
|
+
group is idempotent.
|
|
1955
|
+
"""
|
|
1127
1956
|
historic_roots = _historic_root_keys(conn)
|
|
1128
1957
|
roots_to_reconcile = active_roots | historic_roots
|
|
1129
1958
|
if not roots_to_reconcile:
|
|
1130
1959
|
return
|
|
1131
1960
|
generation = secrets.token_hex(16)
|
|
1132
1961
|
blocks = build_blocks(observations)
|
|
1962
|
+
digests = _group_digests(observations)
|
|
1133
1963
|
for block in blocks:
|
|
1134
1964
|
_reanchor_terminal_events(conn, block)
|
|
1135
|
-
|
|
1965
|
+
unit = _block_unit_text(block)
|
|
1966
|
+
conn.execute(
|
|
1967
|
+
_BLOCK_UPSERT,
|
|
1968
|
+
_block_params(block, generation, unit, digests.get(unit, "")),
|
|
1969
|
+
)
|
|
1136
1970
|
for milestone in percent_milestones(block):
|
|
1137
1971
|
conn.execute(
|
|
1138
1972
|
_MILESTONE_UPSERT,
|
|
1139
1973
|
_milestone_params(block, milestone, generation),
|
|
1140
1974
|
)
|
|
1141
1975
|
blocks_orphaned, milestones_orphaned = _orphan_unseen(
|
|
1142
|
-
conn, roots_to_reconcile, generation, now_iso,
|
|
1976
|
+
conn, roots_to_reconcile, generation, now_iso, units=dirty_units,
|
|
1143
1977
|
)
|
|
1144
1978
|
queued = _evaluate_quota_alerts(
|
|
1145
1979
|
conn, observations=observations,
|
|
@@ -1151,22 +1985,28 @@ def _apply_quota_projection_rows(
|
|
|
1151
1985
|
# transaction. A pre-commit failure rolls all projection updates back;
|
|
1152
1986
|
# a retry sees the prior complete generation or rederives it.
|
|
1153
1987
|
#
|
|
1154
|
-
# projection_state is (source_root_key, account_key)-keyed (#341 spec §2).
|
|
1155
|
-
#
|
|
1156
|
-
#
|
|
1157
|
-
#
|
|
1158
|
-
#
|
|
1159
|
-
#
|
|
1160
|
-
#
|
|
1161
|
-
#
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
if root in accounts_by_root:
|
|
1166
|
-
accounts_by_root[root].add(observation.identity.account_key)
|
|
1988
|
+
# projection_state is (source_root_key, account_key)-keyed (#341 spec §2).
|
|
1989
|
+
# The account partitions and the root signature are both derived from the
|
|
1990
|
+
# LIVE BLOCKS rather than from this pass's observations (public #5 Task 9):
|
|
1991
|
+
# under a bounded pass an account whose only evidence lies in a clean window
|
|
1992
|
+
# contributes nothing to `observations`, so an observation-derived set would
|
|
1993
|
+
# drop it and leave its row carrying a stale signature. A retired partition
|
|
1994
|
+
# loses its blocks to the sweep above, so the same read both keeps and
|
|
1995
|
+
# retires the right rows — and the DELETE below removes what it no longer
|
|
1996
|
+
# names. A root with no blocks at all still stamps one `unattributed` row,
|
|
1997
|
+
# byte-stable with the prior behaviour.
|
|
1998
|
+
signatures: dict[str, str] = {}
|
|
1167
1999
|
for root_key in sorted(active_roots):
|
|
1168
|
-
accounts =
|
|
1169
|
-
root_signature =
|
|
2000
|
+
accounts = _root_accounts(conn, root_key) or {_lib_accounts.UNATTRIBUTED}
|
|
2001
|
+
root_signature = _ledger.compose_root_signature(
|
|
2002
|
+
_root_group_pairs(conn, root_key))
|
|
2003
|
+
signatures[root_key] = root_signature
|
|
2004
|
+
placeholders = ",".join("?" for _ in accounts)
|
|
2005
|
+
conn.execute(
|
|
2006
|
+
"DELETE FROM quota_projection_state WHERE source_root_key=? "
|
|
2007
|
+
"AND account_key NOT IN (" + placeholders + ")",
|
|
2008
|
+
(root_key, *sorted(accounts)),
|
|
2009
|
+
)
|
|
1170
2010
|
for account_key in sorted(accounts):
|
|
1171
2011
|
conn.execute(
|
|
1172
2012
|
"""INSERT INTO quota_projection_state
|
|
@@ -1179,14 +2019,59 @@ def _apply_quota_projection_rows(
|
|
|
1179
2019
|
completed_at_utc=excluded.completed_at_utc""",
|
|
1180
2020
|
(root_key, account_key, generation, root_signature, now_iso),
|
|
1181
2021
|
)
|
|
2022
|
+
# The state row is stamped even when ``ledger_watermark`` is ``None`` — a
|
|
2023
|
+
# cache too old to carry the change log, where ``_ledger_max_seq`` cannot
|
|
2024
|
+
# report a sequence. Guarding this whole block on it left such a store with
|
|
2025
|
+
# no row at all, so ``_full_verification_due`` read True forever and it was
|
|
2026
|
+
# permanently "overdue" for a pass it had just run. Harmless in outcome (no
|
|
2027
|
+
# ledger means every pass is full anyway) but dishonest, and it makes the
|
|
2028
|
+
# deadline unusable as a signal. ``0`` is the right watermark there: it is
|
|
2029
|
+
# the only sequence a ledgerless cache can claim to have consumed, and if
|
|
2030
|
+
# the log later appears the pass replays from its first entry, which is
|
|
2031
|
+
# idempotent.
|
|
2032
|
+
#
|
|
2033
|
+
# Axis 4 (spec §3): an observation captured in the FUTURE is skipped as
|
|
2034
|
+
# a threshold qualifier and becomes eligible when wall time passes it,
|
|
2035
|
+
# with no row mutation for the ledger to record. Persisting the earliest
|
|
2036
|
+
# such instant is what turns that into a dirtiness signal.
|
|
2037
|
+
stored_boundary = None
|
|
2038
|
+
if stored_next_evaluation_at:
|
|
2039
|
+
try:
|
|
2040
|
+
stored_boundary = _parse_utc(
|
|
2041
|
+
stored_next_evaluation_at, "next_evaluation_at_utc")
|
|
2042
|
+
except (TypeError, ValueError):
|
|
2043
|
+
stored_boundary = None
|
|
2044
|
+
#
|
|
2045
|
+
# Recording a NEWLY seen future capture is safe from any pass, so that side
|
|
2046
|
+
# stays unconditional; only RETIRING a matured instant is gated, because
|
|
2047
|
+
# that is the half that claims an evaluation happened.
|
|
2048
|
+
boundary = _axes.next_evaluation_boundary(
|
|
2049
|
+
capture_times=[
|
|
2050
|
+
observation.captured_at for observation in observations],
|
|
2051
|
+
now=now, stored=stored_boundary, retain_due=not consume_alert_axes,
|
|
2052
|
+
)
|
|
2053
|
+
_store_ledger_state(
|
|
2054
|
+
conn, watermark=0 if ledger_watermark is None else ledger_watermark,
|
|
2055
|
+
alerts_enabled=(
|
|
2056
|
+
alerts_enabled if consume_alert_axes else stored_alerts_enabled),
|
|
2057
|
+
next_evaluation_at=None if boundary is None else _utc_iso(boundary),
|
|
2058
|
+
# Spec §2: EVERY full pass stamps the verification deadline,
|
|
2059
|
+
# whatever triggered it — the interval itself, a rebuild, an
|
|
2060
|
+
# interpretation bump, `force_full`, or a dirty-unit burst
|
|
2061
|
+
# overflow. `dirty_units is None` is exactly "this pass was
|
|
2062
|
+
# whole-history", so the stamp cannot drift from the thing it
|
|
2063
|
+
# certifies. A bounded pass carries the stored value through
|
|
2064
|
+
# untouched; overwriting it would let an install that never
|
|
2065
|
+
# bursts postpone verification forever.
|
|
2066
|
+
last_full_pass_at=(
|
|
2067
|
+
now_iso if dirty_units is None else stored_last_full_pass_at),
|
|
2068
|
+
)
|
|
1182
2069
|
if sink is not None:
|
|
1183
2070
|
# Set-then-dispatch: all claims committed with the cycle before the
|
|
1184
2071
|
# cycle's post-commit ALERT_DISPATCHER fires them (spec §5.2 step 6).
|
|
1185
2072
|
sink.extend(queued)
|
|
1186
2073
|
if holder is not None:
|
|
1187
|
-
holder["signatures"] =
|
|
1188
|
-
root_key: _signature(observations, root_key) for root_key in active_roots
|
|
1189
|
-
}
|
|
2074
|
+
holder["signatures"] = signatures
|
|
1190
2075
|
holder["result"] = QuotaProjectionResult(
|
|
1191
2076
|
generation=generation,
|
|
1192
2077
|
blocks_upserted=len(blocks),
|
|
@@ -1226,12 +2111,102 @@ def rematerialize_quota_projection_for_rebuild(stats_conn, *, now=None) -> None:
|
|
|
1226
2111
|
observations = load_codex_quota_observations(
|
|
1227
2112
|
source_root_keys=None, cache_conn=cache,
|
|
1228
2113
|
)
|
|
2114
|
+
# A rebuild is a whole-history pass by definition, so it also
|
|
2115
|
+
# initializes the watermark: every ledger entry up to here is already
|
|
2116
|
+
# reflected in what it just materialized, and leaving the watermark at
|
|
2117
|
+
# zero would make the next tick replay the entire ledger for nothing.
|
|
2118
|
+
watermark = _ledger_max_seq(cache)
|
|
1229
2119
|
finally:
|
|
1230
2120
|
cache.close()
|
|
1231
2121
|
_apply_quota_projection_rows(
|
|
1232
2122
|
stats_conn, observations=observations, active_roots=active_roots,
|
|
1233
2123
|
now=now, now_iso=now_iso, sink=None,
|
|
1234
2124
|
alert_eligible_roots=frozenset(), journal_emit=None, holder=None,
|
|
2125
|
+
dirty_units=None, ledger_watermark=watermark,
|
|
2126
|
+
# Reporting-only, but NOT a carry-through: this writes onto a freshly
|
|
2127
|
+
# rebuilt index where there is no stored axis state to preserve, and
|
|
2128
|
+
# recording the boundary from complete evidence is exactly what the
|
|
2129
|
+
# rebuilt row should start life with. A gate of NULL is the fail-safe
|
|
2130
|
+
# value — `gate_before is not True` makes the next alert-eligible pass
|
|
2131
|
+
# widen and do the activation.
|
|
2132
|
+
consume_alert_axes=True,
|
|
2133
|
+
)
|
|
2134
|
+
|
|
2135
|
+
|
|
2136
|
+
def _resolve_pass_scope(
|
|
2137
|
+
cache_conn: sqlite3.Connection, *, force_full: bool,
|
|
2138
|
+
ledger_state: "dict | None", ledger_high: "int | None", watermark: int,
|
|
2139
|
+
active_roots: set[str], stale_reverse_map: bool,
|
|
2140
|
+
verification_due: bool = False,
|
|
2141
|
+
) -> "tuple[dict | None, int | None]":
|
|
2142
|
+
"""Decide between a bounded pass and a whole-history one.
|
|
2143
|
+
|
|
2144
|
+
Returns ``(scope, watermark_target)``. ``scope`` is ``None`` for a full pass
|
|
2145
|
+
or a dict carrying the exact raw groups to load and the loading units to
|
|
2146
|
+
sweep. ``watermark_target`` is the sequence the pass will stamp, or ``None``
|
|
2147
|
+
when there is no ledger to stamp against.
|
|
2148
|
+
|
|
2149
|
+
Every branch that is not provably safe takes the full path. In order:
|
|
2150
|
+
|
|
2151
|
+
* ``force_full`` and a cache with no ledger table are explicit requests.
|
|
2152
|
+
* No stored state at all — a first run, or a rebuilt index — has no consumed
|
|
2153
|
+
range to trust.
|
|
2154
|
+
* A stored interpretation version that is not the current one means the
|
|
2155
|
+
interpreted KEYS may have moved with no row mutation to observe, which is
|
|
2156
|
+
exactly what the ledger cannot see.
|
|
2157
|
+
* ``ledger_high < watermark`` means the ledger was reset under us (a deleted
|
|
2158
|
+
and recreated cache restarts ``AUTOINCREMENT`` at 1), so the stored
|
|
2159
|
+
watermark now points past entries that describe different mutations.
|
|
2160
|
+
* A block with no reverse map cannot be reached by a scoped sweep.
|
|
2161
|
+
* ``verification_due`` is the periodic whole-history pass (spec §2): the
|
|
2162
|
+
scoped sweep cannot see a block whose physical group left the cache
|
|
2163
|
+
entirely, nor a milestone on a historic root, so the interval is what
|
|
2164
|
+
bounds how long either may survive.
|
|
2165
|
+
* More dirty units than ``_MAX_INCREMENTAL_UNITS`` is a burst — a rebuild or
|
|
2166
|
+
a first ingest — where one unbounded scan beats N indexed seeks.
|
|
2167
|
+
"""
|
|
2168
|
+
if force_full or ledger_high is None or verification_due:
|
|
2169
|
+
return None, ledger_high
|
|
2170
|
+
if (
|
|
2171
|
+
ledger_state is None
|
|
2172
|
+
or ledger_state["interpretation_version"]
|
|
2173
|
+
!= _CODEX_QUOTA_INTERPRETATION_VERSION
|
|
2174
|
+
or ledger_high < watermark
|
|
2175
|
+
or stale_reverse_map
|
|
2176
|
+
):
|
|
2177
|
+
return None, ledger_high
|
|
2178
|
+
rows = _ledger_rows_after(cache_conn, watermark, ledger_high)
|
|
2179
|
+
raw_groups = _ledger.expand_dirty_groups(rows)
|
|
2180
|
+
# Spec §2: "Liveness may narrow what the LOADER is asked to fetch, because
|
|
2181
|
+
# an inactive root's shard returns nothing anyway; it must never narrow what
|
|
2182
|
+
# the SWEEP is asked to reconcile." A dirty unit names a group to sweep even
|
|
2183
|
+
# when its root has left `codex_source_roots` — that is precisely the case
|
|
2184
|
+
# where its blocks must be orphaned. Deriving `units` from the FILTERED set
|
|
2185
|
+
# dropped the departed root's ledgered deletions while still pruning their
|
|
2186
|
+
# ledger entries, stranding those blocks permanently in
|
|
2187
|
+
# `_historic_root_keys`, the projection state and the dashboard, which is a
|
|
2188
|
+
# regression against the pre-change whole-root `_orphan_unseen`.
|
|
2189
|
+
units = {
|
|
2190
|
+
_ledger.physical_group_key_text(_ledger.loading_unit_from_raw(group))
|
|
2191
|
+
for group in raw_groups
|
|
2192
|
+
}
|
|
2193
|
+
if len(units) > _MAX_INCREMENTAL_UNITS:
|
|
2194
|
+
return None, ledger_high
|
|
2195
|
+
load_groups = raw_groups
|
|
2196
|
+
if active_roots:
|
|
2197
|
+
load_groups = frozenset(
|
|
2198
|
+
group for group in raw_groups if group[0] in active_roots)
|
|
2199
|
+
return (
|
|
2200
|
+
{
|
|
2201
|
+
# The loader matches RAW stored coordinates, so the request has to
|
|
2202
|
+
# enumerate every spelling that snaps onto a dirty group. One minute
|
|
2203
|
+
# of weekly jitter lives in BOTH the limit key and a column, so two
|
|
2204
|
+
# raw groups can interpret into one window and loading only the
|
|
2205
|
+
# mutated one would hand the fold a PARTIAL population.
|
|
2206
|
+
"raw_groups": _ledger.snap_equivalent_raw_groups(load_groups),
|
|
2207
|
+
"units": units,
|
|
2208
|
+
},
|
|
2209
|
+
ledger_high,
|
|
1235
2210
|
)
|
|
1236
2211
|
|
|
1237
2212
|
|
|
@@ -1240,6 +2215,8 @@ def reconcile_codex_quota_projection(
|
|
|
1240
2215
|
source_root_keys: Iterable[str] | None = None,
|
|
1241
2216
|
alert_eligible_root_keys: Iterable[str] = (),
|
|
1242
2217
|
now: dt.datetime | None = None,
|
|
2218
|
+
force_full: bool = False,
|
|
2219
|
+
full_pass: str = "inline",
|
|
1243
2220
|
_before_stats_commit: Callable[[], None] | None = None,
|
|
1244
2221
|
_after_stats_commit: Callable[[], None] | None = None,
|
|
1245
2222
|
) -> QuotaProjectionResult:
|
|
@@ -1248,7 +2225,65 @@ def reconcile_codex_quota_projection(
|
|
|
1248
2225
|
Reporting reconciles every configured root. Threshold evaluation is limited
|
|
1249
2226
|
to explicitly lifecycle-eligible roots, so read-only quota commands pass an
|
|
1250
2227
|
empty set and never create an alert claim or activation boundary.
|
|
2228
|
+
|
|
2229
|
+
By default the pass is BOUNDED to what the change ledger says moved since
|
|
2230
|
+
the stored watermark (public #5). ``force_full=True`` bypasses the ledger
|
|
2231
|
+
entirely and re-materializes everything, which is what the equivalence
|
|
2232
|
+
oracle compares against and what an operator gets from a rebuild.
|
|
2233
|
+
|
|
2234
|
+
``full_pass`` decides where a WHOLE-HISTORY pass runs. ``"inline"`` is every
|
|
2235
|
+
caller but the hook: the pass happens on this call, which is what makes the
|
|
2236
|
+
verification deadline "satisfied by whichever caller reaches it first".
|
|
2237
|
+
``"defer"`` is the hook's, and the rule there is absolute — the hook path
|
|
2238
|
+
never runs a whole-history pass inline, whatever put it there. Every route
|
|
2239
|
+
``_resolve_pass_scope`` has into one (an absent or rebuilt projector state,
|
|
2240
|
+
an interpretation-version bump, a reset ledger, a block missing its reverse
|
|
2241
|
+
map, a dirty-unit burst, a ledgerless cache, and the periodic interval) is
|
|
2242
|
+
handed to the detached ``_codex-quota-verify`` worker instead.
|
|
2243
|
+
|
|
2244
|
+
Two deliberate carve-outs stay inline. Only the first is unreachable from
|
|
2245
|
+
the hook; the second is reachable and stays inline anyway, which is a
|
|
2246
|
+
different claim and the honest one.
|
|
2247
|
+
|
|
2248
|
+
``force_full`` still runs inline, and no hook caller passes it. It is a
|
|
2249
|
+
programmatic "do it now" — the equivalence oracle and the rebuild path both
|
|
2250
|
+
need it to mean that.
|
|
2251
|
+
|
|
2252
|
+
The spec §3 alert axes also still widen inline, and they ARE reachable here:
|
|
2253
|
+
every hook tick that clears the 15-second lifecycle throttle passes a
|
|
2254
|
+
non-empty ``alert_eligible_root_keys``, so ``alert_scope`` is resolved and
|
|
2255
|
+
the widening branch is live on all of them. That is accepted for axes 2 and
|
|
2256
|
+
3 — a rule change or a delivery-gate ENABLE must write ``suppressed_backfill``
|
|
2257
|
+
terminal rows for already-satisfied blocks rather than dispatch history, and
|
|
2258
|
+
it can only do that by SEEING those blocks; the worker runs reporting-only
|
|
2259
|
+
with no alert-eligible roots, so deferring would break the pass rather than
|
|
2260
|
+
delay it. Both fire on a config change the user just made, so the cost is
|
|
2261
|
+
bounded and attributable.
|
|
2262
|
+
|
|
2263
|
+
Axis 4 is the exception and is DEFERRED under ``"defer"``. It fires on wall
|
|
2264
|
+
clock, not on a config change: a capture stamped in the future (clock skew
|
|
2265
|
+
across a sleep/resume, an NTP correction) sets the boundary, and the first
|
|
2266
|
+
tick after wall time passes it would otherwise run the whole-history load
|
|
2267
|
+
and apply on the blocking path with nothing to have predicted it. A BOUNDED
|
|
2268
|
+
tick carries the stored boundary through untouched instead.
|
|
2269
|
+
|
|
2270
|
+
A tick that widened to whole-history for axis 2 or 3 anyway does retire it,
|
|
2271
|
+
because it did look: at every observation of every active root. Withholding
|
|
2272
|
+
that was an absorbing state rather than a conservative one — the widening
|
|
2273
|
+
routes are exactly the ones that also need to stamp ``alerts_enabled``, so
|
|
2274
|
+
declining both left the same scope standing and repeated the whole-history
|
|
2275
|
+
pass inline on every subsequent tick.
|
|
2276
|
+
|
|
2277
|
+
What is NOT closed: a hook-only install with a steady enabled gate,
|
|
2278
|
+
unchanged rules and a quiet ledger never produces a qualifying pass, so a
|
|
2279
|
+
matured boundary is retained indefinitely. Bounded in cost (the tick stays
|
|
2280
|
+
bounded and fast; the window is re-evaluated the moment it goes
|
|
2281
|
+
ledger-dirty) and under-alerting in direction, but open.
|
|
1251
2282
|
"""
|
|
2283
|
+
if full_pass not in ("inline", "defer"):
|
|
2284
|
+
raise ValueError(
|
|
2285
|
+
"reconcile_codex_quota_projection: full_pass must be "
|
|
2286
|
+
"'inline' or 'defer'")
|
|
1252
2287
|
if now is None:
|
|
1253
2288
|
now = dt.datetime.now(UTC)
|
|
1254
2289
|
if now.tzinfo is None or now.utcoffset() is None:
|
|
@@ -1277,39 +2312,234 @@ def reconcile_codex_quota_projection(
|
|
|
1277
2312
|
)
|
|
1278
2313
|
physical_sequence = codex_physical_mutation_seq(cache)
|
|
1279
2314
|
certificate = load_codex_quota_projection_certificate(cache)
|
|
2315
|
+
ledger_high = _ledger_max_seq(cache)
|
|
1280
2316
|
finally:
|
|
1281
2317
|
cache.commit()
|
|
2318
|
+
# ONE stats read decides the whole shape of the pass: whether the
|
|
2319
|
+
# short-circuit fires, and if not, whether the pass is bounded.
|
|
2320
|
+
ledger_state = None
|
|
2321
|
+
has_arming = False
|
|
2322
|
+
stale_reverse_map = True
|
|
2323
|
+
alert_scope = None
|
|
2324
|
+
stats_conn = _cctally_core.open_db()
|
|
2325
|
+
try:
|
|
2326
|
+
ledger_state = _ledger_state(stats_conn)
|
|
2327
|
+
has_arming = bool(stats_conn.execute(
|
|
2328
|
+
"SELECT 1 FROM quota_alert_arming "
|
|
2329
|
+
"WHERE source='codex' LIMIT 1"
|
|
2330
|
+
).fetchone())
|
|
2331
|
+
stale_reverse_map = _blocks_missing_reverse_map(stats_conn)
|
|
2332
|
+
signatures_match = (
|
|
2333
|
+
certificate is not None
|
|
2334
|
+
and _stats_projection_signatures_match(
|
|
2335
|
+
stats_conn, active_roots, certificate[1])
|
|
2336
|
+
)
|
|
2337
|
+
if alert_eligible_roots:
|
|
2338
|
+
alert_scope = _resolve_alert_scope(
|
|
2339
|
+
stats_conn, ledger_scope=_axes.SCOPE_GROUPS, now=now,
|
|
2340
|
+
ledger_state=ledger_state,
|
|
2341
|
+
global_enabled=global_alerts_enabled,
|
|
2342
|
+
quota_enabled=quota_alerts_enabled,
|
|
2343
|
+
rules=_rules, config=_config,
|
|
2344
|
+
defer_scheduled=(full_pass == "defer"),
|
|
2345
|
+
)
|
|
2346
|
+
finally:
|
|
2347
|
+
stats_conn.close()
|
|
2348
|
+
|
|
2349
|
+
watermark = 0 if ledger_state is None else ledger_state["watermark"]
|
|
1282
2350
|
# Short-circuit: when nothing is alert-eligible and the certificate
|
|
1283
2351
|
# proves the cache physical state is current AND the stats-side
|
|
1284
2352
|
# projection still matches it (F1), the ~2.9 s observation load and the
|
|
1285
2353
|
# whole reconcile are provably a no-op. Any missed concurrent write
|
|
1286
2354
|
# leaves cur_seq != cert_seq (or a stats-signature mismatch) on the next
|
|
1287
2355
|
# call, so the scheme is self-healing.
|
|
1288
|
-
|
|
2356
|
+
#
|
|
2357
|
+
# The unconsumed-ledger clause is the third leg, and it closes a hole
|
|
2358
|
+
# the certificate alone cannot see: a writer that mutates
|
|
2359
|
+
# `quota_window_snapshots` WITHOUT advancing the physical mutation
|
|
2360
|
+
# sequence (migration 028 does exactly this, and the journal cache
|
|
2361
|
+
# applier did too) leaves the certificate reading as current while real
|
|
2362
|
+
# interpretation drift sits unprocessed. The triggers recorded it, so
|
|
2363
|
+
# the ledger knows even when the sequence does not.
|
|
2364
|
+
#
|
|
2365
|
+
# The projector's own state has to be current too. A stale
|
|
2366
|
+
# interpretation version or a block with no reverse map means the next
|
|
2367
|
+
# pass MUST do work, and skipping on the certificate alone would defer
|
|
2368
|
+
# that repair forever — the certificate cannot see either condition.
|
|
2369
|
+
#
|
|
2370
|
+
# The periodic verification joins them for the same reason. A skipped
|
|
2371
|
+
# pass never stamps the deadline, so letting the short-circuit fire
|
|
2372
|
+
# while it is overdue would leave every subsequent tick overdue too —
|
|
2373
|
+
# the interval would never elapse into anything.
|
|
2374
|
+
verification_due = _full_verification_due(ledger_state, now)
|
|
2375
|
+
projector_state_current = (
|
|
2376
|
+
ledger_state is not None
|
|
2377
|
+
and ledger_state["interpretation_version"]
|
|
2378
|
+
== _CODEX_QUOTA_INTERPRETATION_VERSION
|
|
2379
|
+
and not stale_reverse_map
|
|
2380
|
+
)
|
|
2381
|
+
if (
|
|
2382
|
+
verification_due
|
|
2383
|
+
and full_pass == "defer"
|
|
2384
|
+
# The interval alone. When the projector state is otherwise current
|
|
2385
|
+
# this tick can still do its ordinary BOUNDED work after handing the
|
|
2386
|
+
# verification off, which is why this gate is separate from the
|
|
2387
|
+
# catch-all below: deferring here costs nothing, deferring there
|
|
2388
|
+
# costs this tick's incremental progress.
|
|
2389
|
+
and projector_state_current
|
|
2390
|
+
and ledger_high is not None
|
|
2391
|
+
and ledger_high >= watermark
|
|
2392
|
+
):
|
|
2393
|
+
# The bounded ingest leg made every part of the hook tick bounded
|
|
2394
|
+
# except this one. The spec's escape hatch — "whichever caller
|
|
2395
|
+
# reaches the deadline first, a dashboard tick or a `codex quota`
|
|
2396
|
+
# invocation" — does not exist for a hook-only install, which is
|
|
2397
|
+
# precisely the reporter's shape, so once a day the whole-history
|
|
2398
|
+
# reconcile would land on the blocking hook path against Codex's
|
|
2399
|
+
# 30-second timeout and fail acceptance criterion 3 for the very
|
|
2400
|
+
# user who reported the bug. Hand it to a detached worker.
|
|
2401
|
+
#
|
|
2402
|
+
# Skipping it on a failed spawn is deliberate: the deadline is only
|
|
2403
|
+
# stamped by a pass that COMPLETES, so the next tick is still due
|
|
2404
|
+
# and retries (throttled). One missed daily verification is bounded
|
|
2405
|
+
# staleness; a 30-second hook stall is the reported defect.
|
|
2406
|
+
_defer_codex_quota_verification()
|
|
2407
|
+
verification_due = False
|
|
2408
|
+
ledger_state_current = projector_state_current and not verification_due
|
|
2409
|
+
if (
|
|
2410
|
+
not force_full
|
|
2411
|
+
and not alert_eligible_roots
|
|
2412
|
+
and certificate is not None
|
|
2413
|
+
and ledger_high is not None
|
|
2414
|
+
and ledger_high == watermark
|
|
2415
|
+
and ledger_state_current
|
|
2416
|
+
):
|
|
1289
2417
|
cert_seq, cert_sigs = certificate
|
|
1290
2418
|
if physical_sequence == cert_seq and active_roots <= set(cert_sigs):
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
return QuotaProjectionResult(None, 0, 0, 0, 0, 0, 0)
|
|
1305
|
-
finally:
|
|
1306
|
-
stats_conn.close()
|
|
1307
|
-
observations = load_codex_quota_observations(
|
|
1308
|
-
source_root_keys=active_roots, cache_conn=cache,
|
|
2419
|
+
can_skip_delivery = delivery_enabled or not has_arming
|
|
2420
|
+
if can_skip_delivery and signatures_match:
|
|
2421
|
+
return QuotaProjectionResult(None, 0, 0, 0, 0, 0, 0)
|
|
2422
|
+
|
|
2423
|
+
dirty_units, watermark_target = _resolve_pass_scope(
|
|
2424
|
+
cache,
|
|
2425
|
+
force_full=force_full,
|
|
2426
|
+
ledger_state=ledger_state,
|
|
2427
|
+
ledger_high=ledger_high,
|
|
2428
|
+
watermark=watermark,
|
|
2429
|
+
active_roots=active_roots,
|
|
2430
|
+
stale_reverse_map=stale_reverse_map,
|
|
2431
|
+
verification_due=verification_due,
|
|
1309
2432
|
)
|
|
2433
|
+
if dirty_units is None and full_pass == "defer" and not force_full:
|
|
2434
|
+
# The rule, for every OTHER route into a whole-history pass: an
|
|
2435
|
+
# absent or freshly rebuilt projector state, an interpretation-
|
|
2436
|
+
# version bump, a reset ledger, a block missing its reverse map, a
|
|
2437
|
+
# dirty-unit burst, and a cache too old to carry the change log.
|
|
2438
|
+
#
|
|
2439
|
+
# The rebuilt-state route is the one that matters, and it is not
|
|
2440
|
+
# hypothetical: this feature bumps `STATS_INDEX_EPOCH`, so every
|
|
2441
|
+
# upgrading install rebuilds stats.db from the journal on first
|
|
2442
|
+
# open. Measured on a real 211K-observation store that rebuild alone
|
|
2443
|
+
# cost 76.45s of an 82.05s tick. Running the whole-history pass
|
|
2444
|
+
# inline on top of it, on a path Codex kills at 30 seconds, means
|
|
2445
|
+
# `run_stats_ingest` commits nothing, `last_full_pass_at` is never
|
|
2446
|
+
# stamped, and the next tick repeats it — a non-converging
|
|
2447
|
+
# 30-second-per-turn loop, which is the reported defect delivered by
|
|
2448
|
+
# the fix.
|
|
2449
|
+
#
|
|
2450
|
+
# Unlike the interval gate above, this one performs NO projection
|
|
2451
|
+
# work at all: without a trustworthy watermark there is no bounded
|
|
2452
|
+
# pass to fall back to. That leaves the projection transiently
|
|
2453
|
+
# missing rather than merely stale, and it is accepted — the worker
|
|
2454
|
+
# converges it (throttled retry on failure), every non-hook caller
|
|
2455
|
+
# still runs inline, and a 30-second blocking tick is not an option.
|
|
2456
|
+
_defer_codex_quota_verification()
|
|
2457
|
+
return QuotaProjectionResult(None, 0, 0, 0, 0, 0, 0)
|
|
2458
|
+
# Axis 2/3/4 (spec §3): alert state is not a function of window
|
|
2459
|
+
# dirtiness. A rule change and a delivery-gate ENABLE each require a
|
|
2460
|
+
# semantic pass over the affected identities with no observation having
|
|
2461
|
+
# moved — and activation must write `suppressed_backfill` terminal rows
|
|
2462
|
+
# rather than dispatch history, which it can only do if it actually sees
|
|
2463
|
+
# the blocks. Widening a bounded pass is always safe; missing an
|
|
2464
|
+
# identity is not. Both are driven by a configuration change the user
|
|
2465
|
+
# just made, so the widening is bounded and expected, and it stays
|
|
2466
|
+
# inline even under `defer`.
|
|
2467
|
+
#
|
|
2468
|
+
# Axis 4 does NOT, and the honest statement about it is "reachable from
|
|
2469
|
+
# the hook, and therefore deferred" rather than "unreachable". Every
|
|
2470
|
+
# tick that clears the 15s lifecycle throttle carries eligible roots, so
|
|
2471
|
+
# this branch is live on all of them; a future-clocked capture (clock
|
|
2472
|
+
# skew across a sleep/resume, an NTP correction) would then put one
|
|
2473
|
+
# unannounced whole-history load and apply on the blocking path the
|
|
2474
|
+
# first time wall time passed the boundary. `_resolve_alert_scope`
|
|
2475
|
+
# records it as `REASON_SCHEDULED_DEFERRED` instead, and the boundary is
|
|
2476
|
+
# carried through below so the next pass that CAN afford the widening
|
|
2477
|
+
# still sees it.
|
|
2478
|
+
if (
|
|
2479
|
+
dirty_units is not None
|
|
2480
|
+
and alert_scope is not None
|
|
2481
|
+
and alert_scope.widens(_axes.SCOPE_GROUPS)
|
|
2482
|
+
):
|
|
2483
|
+
dirty_units = None
|
|
2484
|
+
if dirty_units is None:
|
|
2485
|
+
observations = load_codex_quota_observations(
|
|
2486
|
+
source_root_keys=active_roots, cache_conn=cache,
|
|
2487
|
+
)
|
|
2488
|
+
else:
|
|
2489
|
+
observations = load_codex_quota_observations(
|
|
2490
|
+
source_root_keys=active_roots, cache_conn=cache,
|
|
2491
|
+
physical_groups=dirty_units["raw_groups"],
|
|
2492
|
+
)
|
|
2493
|
+
# A unit whose members ALL disappeared loads nothing, so the loaded
|
|
2494
|
+
# observations alone would not name it and its blocks would never be
|
|
2495
|
+
# swept. The ledger-derived set is authoritative for the sweep; the
|
|
2496
|
+
# loaded set is unioned in only to cover a stored limit key the raw
|
|
2497
|
+
# snap and the interpreted strip disagree on (reachable by hand
|
|
2498
|
+
# repair, not by ingest).
|
|
2499
|
+
dirty_units = frozenset(
|
|
2500
|
+
dirty_units["units"]
|
|
2501
|
+
| {_observation_unit_text(o) for o in observations}
|
|
2502
|
+
)
|
|
1310
2503
|
finally:
|
|
1311
2504
|
cache.close()
|
|
1312
2505
|
|
|
2506
|
+
# A pass may only advance the two non-dirtiness alert axes it stores if it
|
|
2507
|
+
# actually did the work they exist to trigger.
|
|
2508
|
+
#
|
|
2509
|
+
# A REPORTING-ONLY pass (the `_codex-quota-verify` worker, the dashboard
|
|
2510
|
+
# tick, `codex quota`) never reaches a threshold decision —
|
|
2511
|
+
# `_evaluate_quota_alerts` returns at `if not alert_eligible_roots` — yet it
|
|
2512
|
+
# used to stamp both, so a delivery-gate ENABLE landing on the same tick as
|
|
2513
|
+
# a full pass was consumed with no arming row and no `suppressed_backfill`,
|
|
2514
|
+
# and could not re-fire because `gate_before` then read True. Every
|
|
2515
|
+
# upgrading install is in exactly that state right after the epoch rebuild,
|
|
2516
|
+
# and the daily verification puts installs there routinely. Carrying
|
|
2517
|
+
# eligible roots is what excludes it, and nothing else does.
|
|
2518
|
+
#
|
|
2519
|
+
# The second condition asks whether this pass LOOKED, not whether it
|
|
2520
|
+
# intended to. A bounded tick that deferred axis 4 declined to look, so it
|
|
2521
|
+
# must not retire the boundary. A tick that widened to whole-history for
|
|
2522
|
+
# some OTHER reason did look — at every observation of every active root,
|
|
2523
|
+
# and applied over all of them — so it evaluated the matured instant as
|
|
2524
|
+
# surely as a widening for axis 4 itself would have, and retiring it is
|
|
2525
|
+
# honest. Gating on the deferral alone instead was an absorbing state, not a
|
|
2526
|
+
# conservative one: `alert_dirty_scope` widens whenever `gate_before is not
|
|
2527
|
+
# True` (the NULL a rebuild leaves, the False a disable leaves), the hook is
|
|
2528
|
+
# the only production caller that carries eligible roots AND it always
|
|
2529
|
+
# defers, so a due boundary froze `alerts_enabled` at its stored value and
|
|
2530
|
+
# every following tick re-resolved the identical SCOPE_ALL — the
|
|
2531
|
+
# whole-history reconcile inline on the blocking path, on every turn.
|
|
2532
|
+
#
|
|
2533
|
+
# Reaching this line with `dirty_units is None` under `defer` means
|
|
2534
|
+
# precisely "the alert axes widened it": every other route into a
|
|
2535
|
+
# whole-history pass returned at the catch-all above, and no hook caller
|
|
2536
|
+
# passes `force_full`.
|
|
2537
|
+
consume_alert_axes = bool(alert_eligible_roots & active_roots) and (
|
|
2538
|
+
dirty_units is None
|
|
2539
|
+
or alert_scope is None
|
|
2540
|
+
or _axes.REASON_SCHEDULED_DEFERRED not in alert_scope.reasons
|
|
2541
|
+
)
|
|
2542
|
+
|
|
1313
2543
|
# ── Apply phase (Task 7 Item 3) ─────────────────────────────────────────
|
|
1314
2544
|
# The stats.db writes route through the single-flight ingest cycle instead
|
|
1315
2545
|
# of this function opening its own stats connection + BEGIN IMMEDIATE. The
|
|
@@ -1339,7 +2569,19 @@ def reconcile_codex_quota_projection(
|
|
|
1339
2569
|
alert_eligible_roots=alert_eligible_roots,
|
|
1340
2570
|
journal_emit=journal_emit, journal_disarm=journal_disarm,
|
|
1341
2571
|
journal_terminal=journal_terminal,
|
|
1342
|
-
holder=holder,
|
|
2572
|
+
holder=holder, dirty_units=dirty_units,
|
|
2573
|
+
ledger_watermark=watermark_target,
|
|
2574
|
+
alerts_enabled=delivery_enabled,
|
|
2575
|
+
stored_next_evaluation_at=(
|
|
2576
|
+
None if ledger_state is None
|
|
2577
|
+
else ledger_state["next_evaluation_at"]),
|
|
2578
|
+
stored_last_full_pass_at=(
|
|
2579
|
+
None if ledger_state is None
|
|
2580
|
+
else ledger_state["last_full_pass_at"]),
|
|
2581
|
+
stored_alerts_enabled=(
|
|
2582
|
+
None if ledger_state is None
|
|
2583
|
+
else ledger_state["alerts_enabled"]),
|
|
2584
|
+
consume_alert_axes=consume_alert_axes,
|
|
1343
2585
|
)
|
|
1344
2586
|
|
|
1345
2587
|
import _cctally_journal as _jr
|
|
@@ -1439,10 +2681,14 @@ def reconcile_codex_quota_projection(
|
|
|
1439
2681
|
)
|
|
1440
2682
|
|
|
1441
2683
|
# Finalize: the cache-side certificate (self-healing no-op optimization) is
|
|
1442
|
-
# stored only when the apply actually materialized a projection.
|
|
2684
|
+
# stored only when the apply actually materialized a projection. Ledger
|
|
2685
|
+
# pruning rides the same connection and transaction — it is the only other
|
|
2686
|
+
# cache write this function makes, and folding it in avoids adding a second
|
|
2687
|
+
# unflocked mutator.
|
|
1443
2688
|
if holder["signatures"] is not None:
|
|
1444
2689
|
_store_codex_quota_projection_certificate(
|
|
1445
2690
|
sequence=physical_sequence, signatures=holder["signatures"],
|
|
2691
|
+
prune_ledger_through=watermark_target,
|
|
1446
2692
|
)
|
|
1447
2693
|
return holder["result"]
|
|
1448
2694
|
|