cctally 1.113.0 → 1.115.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +32 -8
- package/bin/_cctally_cache.py +6035 -722
- package/bin/_cctally_completion.py +1287 -0
- package/bin/_cctally_config.py +37 -0
- package/bin/_cctally_core.py +400 -2
- package/bin/_cctally_dashboard.py +98 -20
- package/bin/_cctally_dashboard_conversation.py +292 -23
- package/bin/_cctally_dashboard_envelope.py +16 -3
- package/bin/_cctally_dashboard_perf.py +128 -9
- package/bin/_cctally_dashboard_sources.py +285 -213
- package/bin/_cctally_db.py +1177 -23
- package/bin/_cctally_diagnosis_sources.py +22 -4
- package/bin/_cctally_doctor.py +572 -45
- package/bin/_cctally_five_hour.py +84 -33
- package/bin/_cctally_forecast.py +108 -15
- package/bin/_cctally_journal.py +3950 -116
- package/bin/_cctally_journal_repair.py +101 -12
- package/bin/_cctally_milestone_history.py +82 -9
- package/bin/_cctally_parser.py +42 -0
- package/bin/_cctally_percent_breakdown.py +3 -2
- package/bin/_cctally_project.py +8 -2
- package/bin/_cctally_quarantine.py +652 -0
- package/bin/_cctally_quota.py +227 -37
- package/bin/_cctally_quota_model.py +360 -83
- package/bin/_cctally_record.py +691 -149
- package/bin/_cctally_rederive.py +320 -41
- package/bin/_cctally_refresh.py +84 -13
- package/bin/_cctally_source_analytics.py +83 -17
- package/bin/_cctally_statusline.py +6154 -517
- package/bin/_cctally_store.py +367 -53
- package/bin/_cctally_transcript.py +8 -7
- package/bin/_cctally_tui.py +61 -56
- package/bin/_cctally_weekrefs.py +128 -21
- package/bin/_lib_account_binding.py +152 -0
- package/bin/_lib_account_protocol.py +386 -0
- package/bin/_lib_aggregators.py +4 -0
- package/bin/_lib_cache_coverage.py +13 -1
- package/bin/_lib_cache_report.py +35 -6
- package/bin/_lib_cache_writer_lock.py +19 -3
- package/bin/_lib_clone_copy.py +142 -0
- package/bin/_lib_codex_conversation.py +32 -0
- package/bin/_lib_codex_conversation_query.py +297 -49
- package/bin/_lib_codex_conversation_watch.py +59 -14
- package/bin/_lib_codex_drain_admission.py +214 -0
- package/bin/_lib_codex_metadata.py +18 -1
- package/bin/_lib_codex_publication_barrier.py +277 -0
- package/bin/_lib_codex_rollout_container.py +502 -0
- package/bin/_lib_conversation_dispatch.py +3 -0
- package/bin/_lib_conversation_query.py +106 -59
- package/bin/_lib_conversation_retention.py +1605 -640
- package/bin/_lib_cost_provenance.py +53 -2
- package/bin/_lib_detector_access_registry.py +3817 -0
- package/bin/_lib_doctor.py +714 -69
- package/bin/_lib_five_hour.py +126 -0
- package/bin/_lib_fmt.py +11 -0
- package/bin/_lib_ingest_frontier.py +414 -20
- package/bin/_lib_journal.py +141 -10
- package/bin/_lib_jsonl.py +5 -4
- package/bin/_lib_legacy_spool.py +126 -0
- package/bin/_lib_legacy_writer_evidence.py +136 -0
- package/bin/_lib_merge_gate.py +231 -86
- package/bin/_lib_meter_rate_change.py +10 -0
- package/bin/_lib_outcome_quarantine.py +690 -0
- package/bin/_lib_perf.py +27 -1
- package/bin/_lib_pricing.py +199 -9
- package/bin/_lib_pricing_check.py +20 -4
- package/bin/_lib_quarantine.py +154 -0
- package/bin/_lib_quota_calibration.py +53 -8
- package/bin/_lib_quota_copy.py +2 -1
- package/bin/_lib_quota_model.py +288 -46
- package/bin/_lib_reclaim_planner.py +792 -0
- package/bin/_lib_rederive.py +319 -82
- package/bin/_lib_segment_summary.py +35 -2
- package/bin/_lib_snapshot_cache.py +21 -0
- package/bin/_lib_sqlite_close.py +81 -0
- package/bin/_lib_statusline_candidates.py +292 -23
- package/bin/_lib_subscription_weeks.py +100 -17
- package/bin/_lib_tick_stats.py +217 -17
- package/bin/_lib_wal_checkpoint.py +1077 -0
- package/bin/_lib_write_budget.py +483 -0
- package/bin/_lib_write_io.py +703 -0
- package/bin/cctally +31 -1
- package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
- package/dashboard/static/assets/index-BKITk1CT.css +1 -0
- package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +19 -1
- package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
- package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Per-connection SQLite checkpoint-on-close control (#901 spec §5.4).
|
|
2
|
+
|
|
3
|
+
The compatibility helper that `_cctally_cache._set_cache_no_checkpoint_on_close`
|
|
4
|
+
introduced for cache recovery, generalized to any connection. Python 3.12+
|
|
5
|
+
reaches `SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE` through `Connection.setconfig`;
|
|
6
|
+
CPython 3.11 reaches the same SQLite API through its supported
|
|
7
|
+
`pysqlite_Connection` layout and the stdlib extension's own linked
|
|
8
|
+
`sqlite3_db_config`, so a different SQLite instance is never bound.
|
|
9
|
+
Stdlib-only leaf.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import sqlite3
|
|
14
|
+
import sys
|
|
15
|
+
|
|
16
|
+
#: `SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE` from sqlite3.h.
|
|
17
|
+
SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE = 1006
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def set_no_checkpoint_on_close(conn: sqlite3.Connection, disabled: bool, *,
|
|
21
|
+
purpose: str = "connection") -> None:
|
|
22
|
+
"""Turn SQLite's checkpoint at the last close off (``disabled=True``) or
|
|
23
|
+
back on; raise `sqlite3.NotSupportedError` when it cannot be applied."""
|
|
24
|
+
option = getattr(sqlite3, "SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE", None)
|
|
25
|
+
setconfig = getattr(conn, "setconfig", None)
|
|
26
|
+
if option is not None and setconfig is not None:
|
|
27
|
+
setconfig(option, bool(disabled))
|
|
28
|
+
getconfig = getattr(conn, "getconfig", None)
|
|
29
|
+
if getconfig is not None and bool(getconfig(option)) != bool(disabled):
|
|
30
|
+
raise sqlite3.NotSupportedError(
|
|
31
|
+
f"{purpose} could not configure SQLite close checkpointing")
|
|
32
|
+
return
|
|
33
|
+
set_no_checkpoint_on_close_cpython(conn, disabled, purpose=purpose)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def set_no_checkpoint_on_close_cpython(conn: sqlite3.Connection,
|
|
37
|
+
disabled: bool, *,
|
|
38
|
+
purpose: str = "connection") -> None:
|
|
39
|
+
"""Python 3.11 compatibility adapter for sqlite3_db_config()."""
|
|
40
|
+
if sys.implementation.name != "cpython":
|
|
41
|
+
raise sqlite3.NotSupportedError(
|
|
42
|
+
f"{purpose} requires SQLite no-checkpoint-on-close support")
|
|
43
|
+
|
|
44
|
+
# CPython 3.11's public sqlite3 module does not expose db_config(), but its
|
|
45
|
+
# connection layout begins with PyObject_HEAD followed by ``sqlite3 *db``.
|
|
46
|
+
# Load sqlite3_db_config from the same extension so we never bind a
|
|
47
|
+
# different SQLite instance.
|
|
48
|
+
import _sqlite3
|
|
49
|
+
import ctypes
|
|
50
|
+
|
|
51
|
+
sqlite_lib = ctypes.CDLL(_sqlite3.__file__)
|
|
52
|
+
db_config = sqlite_lib.sqlite3_db_config
|
|
53
|
+
db_config.argtypes = (ctypes.c_void_p, ctypes.c_int)
|
|
54
|
+
db_config.restype = ctypes.c_int
|
|
55
|
+
pointer_size = ctypes.sizeof(ctypes.c_void_p)
|
|
56
|
+
db_pointer = ctypes.c_void_p.from_address(
|
|
57
|
+
id(conn) + (2 * pointer_size)
|
|
58
|
+
).value
|
|
59
|
+
if not db_pointer:
|
|
60
|
+
raise sqlite3.NotSupportedError(
|
|
61
|
+
f"{purpose} could not resolve the SQLite connection handle")
|
|
62
|
+
current = ctypes.c_int()
|
|
63
|
+
rc = db_config(
|
|
64
|
+
ctypes.c_void_p(db_pointer),
|
|
65
|
+
SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE,
|
|
66
|
+
ctypes.c_int(1 if disabled else 0),
|
|
67
|
+
ctypes.byref(current),
|
|
68
|
+
)
|
|
69
|
+
if rc != sqlite3.SQLITE_OK or current.value != int(bool(disabled)):
|
|
70
|
+
raise sqlite3.NotSupportedError(
|
|
71
|
+
f"{purpose} could not configure SQLite close checkpointing")
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def no_checkpoint_on_close_enabled(conn: sqlite3.Connection) -> "bool | None":
|
|
75
|
+
"""The setting as SQLite reports it, or None where it cannot be read
|
|
76
|
+
(CPython 3.11, whose adapter verifies at set time instead)."""
|
|
77
|
+
option = getattr(sqlite3, "SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE", None)
|
|
78
|
+
getconfig = getattr(conn, "getconfig", None)
|
|
79
|
+
if option is None or getconfig is None:
|
|
80
|
+
return None
|
|
81
|
+
return bool(getconfig(option))
|
|
@@ -24,6 +24,9 @@ CONTROL_SCHEMA_VERSION = 2
|
|
|
24
24
|
#: Control documents this binary can read. Version 1 predates the pending
|
|
25
25
|
#: drop's deadline and retained evidence; see `_pending_drop`.
|
|
26
26
|
CONTROL_SCHEMA_VERSIONS_READ = (1, 2)
|
|
27
|
+
#: #825/#832: the account-bound protocol generation. Every current document,
|
|
28
|
+
#: retry namespace and plan names it; the legacy spool never carries it.
|
|
29
|
+
PROTOCOL_GENERATION = "account-bound-v1"
|
|
27
30
|
#: D5: a non-extendable wall clock measured from the first pending drop.
|
|
28
31
|
PENDING_DROP_DEADLINE_SECONDS = 180
|
|
29
32
|
CANDIDATE_DOCUMENT_MAX_BYTES = 4 * 1024
|
|
@@ -53,6 +56,9 @@ class Candidate:
|
|
|
53
56
|
received_at: int = 0
|
|
54
57
|
five_hour: AxisValue | None = None
|
|
55
58
|
seven_day: AxisValue | None = None
|
|
59
|
+
#: The immutable `AccountBinding` captured with this candidate (#825/#832
|
|
60
|
+
#: spec 4.2). The reducer partitions by its account; it is never inferred.
|
|
61
|
+
binding: object | None = None
|
|
56
62
|
|
|
57
63
|
|
|
58
64
|
@dataclasses.dataclass(frozen=True)
|
|
@@ -62,7 +68,11 @@ class AxisProjection:
|
|
|
62
68
|
canonical_key: int
|
|
63
69
|
captured_at: int
|
|
64
70
|
source: str
|
|
65
|
-
|
|
71
|
+
#: The reset generation the projected value belongs to. Weekly: the
|
|
72
|
+
#: reset-aware floor epoch (an int). Five-hour (#832): the LOGICAL
|
|
73
|
+
#: generation string of the exact account's active reset event, or "" for
|
|
74
|
+
#: no event — never a SQLite rowid, so a rebuild cannot change it.
|
|
75
|
+
reset_generation: int | str
|
|
66
76
|
|
|
67
77
|
|
|
68
78
|
@dataclasses.dataclass(frozen=True)
|
|
@@ -73,11 +83,79 @@ class FileFingerprint:
|
|
|
73
83
|
mtime_ns: int
|
|
74
84
|
|
|
75
85
|
|
|
86
|
+
#: The classification evidence of one projected axis (#825/#832 spec 4.3
|
|
87
|
+
#: revision 5, Codex design-fix-round-5 Q1). ``present``: the axis holds a
|
|
88
|
+
#: baseline. ``expired_predecessor``: every admitted five-hour predecessor
|
|
89
|
+
#: left the projection only through expiry at the evaluation's instant (on
|
|
90
|
+
#: the weekly axis: every retained weekly row lies in a week that ends at or
|
|
91
|
+
#: before ``predecessor_key``, fix round K14c). ``proven_empty``: the covered
|
|
92
|
+
#: snapshot proves the account has no admitted baseline on the axis.
|
|
93
|
+
#: ``floor_retired`` (revision 6, Codex design-fix-round-6 Q2): the axis's
|
|
94
|
+
#: group has an effective, admitted floor of the account and no accepted
|
|
95
|
+
#: observation of the axis survives at or after it. ``unknown``: anything
|
|
96
|
+
#: else — never inferred from an absent value.
|
|
97
|
+
EVIDENCE_PRESENT = "present"
|
|
98
|
+
EVIDENCE_EXPIRED_PREDECESSOR = "expired_predecessor"
|
|
99
|
+
EVIDENCE_PROVEN_EMPTY = "proven_empty"
|
|
100
|
+
EVIDENCE_FLOOR_RETIRED = "floor_retired"
|
|
101
|
+
EVIDENCE_UNKNOWN = "unknown"
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@dataclasses.dataclass(frozen=True)
|
|
105
|
+
class AxisEvidence:
|
|
106
|
+
"""How an axis of a projection was classified, computed inside the same
|
|
107
|
+
validated, prefix-covered snapshot as the projection itself. An
|
|
108
|
+
expired predecessor keeps the greatest expired window's account-canonical
|
|
109
|
+
key and raw reset: the evidence a later reading's window is compared
|
|
110
|
+
with, never a value to display or to fill an axis with."""
|
|
111
|
+
|
|
112
|
+
kind: str
|
|
113
|
+
predecessor_key: int | None = None
|
|
114
|
+
predecessor_raw: int | None = None
|
|
115
|
+
#: Fix round K12 (K11R-10): for an unknown axis that wall time alone can
|
|
116
|
+
#: classify — a row beyond the future band — the earliest instant at
|
|
117
|
+
#: which such a row becomes admissible; else None.
|
|
118
|
+
admissible_at: int | None = None
|
|
119
|
+
#: ``floor_retired`` only (spec 4.3 revision 6): the account, the axis,
|
|
120
|
+
#: the retired group's account-canonical key, the floor instant, the
|
|
121
|
+
#: floor's logical identity and the group's anchor raw reset. Never a
|
|
122
|
+
#: retired percentage.
|
|
123
|
+
account_key: str | None = None
|
|
124
|
+
axis: str | None = None
|
|
125
|
+
group_key: int | None = None
|
|
126
|
+
floor_epoch: int | None = None
|
|
127
|
+
floor_identity: str | None = None
|
|
128
|
+
anchor_raw: int | None = None
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
EVIDENCE_UNKNOWN_AXIS = AxisEvidence(EVIDENCE_UNKNOWN)
|
|
132
|
+
|
|
133
|
+
#: How `classify_absent_axis` classifies a reading on an axis that holds no
|
|
134
|
+
#: projected baseline. ``publish``: a first reading or a new window, under
|
|
135
|
+
#: either drop gate. ``stale``: an older window, or a reading proven captured
|
|
136
|
+
#: before the floor of a retired group — the older-window disposition, which
|
|
137
|
+
#: publishes nothing on the axis and opens no consensus. ``floor``: a reading
|
|
138
|
+
#: in a floor-retired group, which publishes only from contributors captured
|
|
139
|
+
#: at or after the floor. ``unknown``: unclassifiable; the caller defers the
|
|
140
|
+
#: whole evaluation.
|
|
141
|
+
CLASS_PUBLISH = "publish"
|
|
142
|
+
CLASS_STALE = "stale"
|
|
143
|
+
CLASS_FLOOR = "floor"
|
|
144
|
+
CLASS_UNKNOWN = "unknown"
|
|
145
|
+
|
|
146
|
+
|
|
76
147
|
@dataclasses.dataclass(frozen=True)
|
|
77
148
|
class DbProjection:
|
|
78
149
|
five_hour: AxisProjection | None
|
|
79
150
|
seven_day: AxisProjection | None
|
|
80
151
|
db_files: Mapping[str, FileFingerprint | None] | None = None
|
|
152
|
+
#: The exact account this projection was read for (#825/#832 spec 4.5).
|
|
153
|
+
account_key: str | None = None
|
|
154
|
+
#: Per-axis `AxisEvidence` (spec 4.3 revision 5), keyed by axis name. Only
|
|
155
|
+
#: the covered projection read carries it; a projection without it — a
|
|
156
|
+
#: persisted control document, a deferred read — classifies an absent
|
|
157
|
+
#: axis as unknown.
|
|
158
|
+
evidence: Mapping[str, AxisEvidence] | None = None
|
|
81
159
|
|
|
82
160
|
|
|
83
161
|
@dataclasses.dataclass(frozen=True)
|
|
@@ -92,7 +170,12 @@ class RetrySignature:
|
|
|
92
170
|
candidate_percent: float
|
|
93
171
|
db_key: int | None
|
|
94
172
|
db_percent: float | None
|
|
95
|
-
db_reset_generation: int | None
|
|
173
|
+
db_reset_generation: int | str | None
|
|
174
|
+
#: The retry namespace (#825/#832 spec 4.3): protocol generation, account
|
|
175
|
+
#: and axis. Window, value and logical generation are the fields above.
|
|
176
|
+
protocol_generation: str | None = None
|
|
177
|
+
account_key: str | None = None
|
|
178
|
+
axis: str | None = None
|
|
96
179
|
|
|
97
180
|
|
|
98
181
|
@dataclasses.dataclass(frozen=True)
|
|
@@ -174,6 +257,12 @@ class ReductionDecision:
|
|
|
174
257
|
action: str
|
|
175
258
|
control: ControlState
|
|
176
259
|
plan: PublicationPlan | None = None
|
|
260
|
+
#: Fix round K13 (review cycle 7 finding P1-2, spec 4.3 *Classification
|
|
261
|
+
#: evidence*): the axes on which a reading met no projected baseline and
|
|
262
|
+
#: no classification evidence. Neither drop gate publishes such an axis;
|
|
263
|
+
#: the caller defers the whole evaluation instead of reading the NOOP
|
|
264
|
+
#: on it as an unchanged reading.
|
|
265
|
+
unknown_axes: frozenset = frozenset()
|
|
177
266
|
|
|
178
267
|
|
|
179
268
|
def _is_int(value: object) -> bool:
|
|
@@ -318,18 +407,32 @@ def _axis_projection(value: object) -> AxisProjection | None:
|
|
|
318
407
|
for key in ("rawResetsAt", "canonicalKey", "capturedAt"):
|
|
319
408
|
if not _is_int(doc[key]):
|
|
320
409
|
raise StateValidationError(f"{key} must be an integer")
|
|
321
|
-
|
|
322
|
-
raise StateValidationError("resetGeneration must be nonnegative")
|
|
410
|
+
generation = _reset_generation(doc["resetGeneration"], "resetGeneration")
|
|
323
411
|
return AxisProjection(
|
|
324
412
|
percent=_percent(doc["percent"]),
|
|
325
413
|
raw_resets_at=doc["rawResetsAt"],
|
|
326
414
|
canonical_key=doc["canonicalKey"],
|
|
327
415
|
captured_at=doc["capturedAt"],
|
|
328
416
|
source=_require_ascii_source(doc["source"]),
|
|
329
|
-
reset_generation=
|
|
417
|
+
reset_generation=generation,
|
|
330
418
|
)
|
|
331
419
|
|
|
332
420
|
|
|
421
|
+
#: A logical five-hour generation is `<effective>|<detected>|<journal id>`.
|
|
422
|
+
_RESET_GENERATION_MAX_CHARS = 512
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _reset_generation(value: object, label: str) -> int | str:
|
|
426
|
+
"""A weekly floor epoch (nonnegative int) or a five-hour logical
|
|
427
|
+
generation (bounded ASCII string; "" means no event)."""
|
|
428
|
+
if _is_nonnegative_int(value):
|
|
429
|
+
return value
|
|
430
|
+
if (isinstance(value, str) and len(value) <= _RESET_GENERATION_MAX_CHARS
|
|
431
|
+
and value.isascii()):
|
|
432
|
+
return value
|
|
433
|
+
raise StateValidationError(f"{label} must be a nonnegative int or a logical generation")
|
|
434
|
+
|
|
435
|
+
|
|
333
436
|
def _fingerprint(value: object) -> FileFingerprint | None:
|
|
334
437
|
if value is None:
|
|
335
438
|
return None
|
|
@@ -360,21 +463,30 @@ def _retry_signature(value: object) -> RetrySignature | None:
|
|
|
360
463
|
if value is None:
|
|
361
464
|
return None
|
|
362
465
|
doc = _require_object(value, "retrySignature")
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
466
|
+
base = {"candidateKey", "candidatePercent", "dbKey", "dbPercent", "dbResetGeneration"}
|
|
467
|
+
scoped = base | {"protocolGeneration", "accountKey", "axis"}
|
|
468
|
+
if set(doc) not in (base, scoped):
|
|
469
|
+
raise StateValidationError("unexpected retrySignature keys")
|
|
367
470
|
if not _is_int(doc["candidateKey"]):
|
|
368
471
|
raise StateValidationError("candidateKey must be an integer")
|
|
369
472
|
db_generation = doc["dbResetGeneration"]
|
|
370
|
-
if db_generation is not None
|
|
371
|
-
|
|
473
|
+
if db_generation is not None:
|
|
474
|
+
db_generation = _reset_generation(db_generation, "dbResetGeneration")
|
|
475
|
+
scope = (None, None, None)
|
|
476
|
+
if set(doc) == scoped:
|
|
477
|
+
scope = (doc["protocolGeneration"], doc["accountKey"], doc["axis"])
|
|
478
|
+
if (not isinstance(scope[0], str) or not isinstance(scope[1], str)
|
|
479
|
+
or not scope[1] or scope[2] not in AXES):
|
|
480
|
+
raise StateValidationError("invalid retry namespace")
|
|
372
481
|
return RetrySignature(
|
|
373
482
|
candidate_key=doc["candidateKey"],
|
|
374
483
|
candidate_percent=_percent(doc["candidatePercent"]),
|
|
375
484
|
db_key=_nullable_int(doc["dbKey"], "dbKey"),
|
|
376
485
|
db_percent=_nullable_percent(doc["dbPercent"], "dbPercent"),
|
|
377
486
|
db_reset_generation=db_generation,
|
|
487
|
+
protocol_generation=scope[0],
|
|
488
|
+
account_key=scope[1],
|
|
489
|
+
axis=scope[2],
|
|
378
490
|
)
|
|
379
491
|
|
|
380
492
|
|
|
@@ -611,6 +723,9 @@ class _AxisDecision:
|
|
|
611
723
|
action: str
|
|
612
724
|
candidate: AxisValue | None
|
|
613
725
|
pending: PendingDrop | None
|
|
726
|
+
#: A reading on an axis with no baseline and no classification evidence
|
|
727
|
+
#: (fix round K13): never published, whatever the drop gate.
|
|
728
|
+
unknown: bool = False
|
|
614
729
|
|
|
615
730
|
|
|
616
731
|
def _candidate_axis(candidate: Candidate, axis: str) -> AxisValue | None:
|
|
@@ -621,6 +736,62 @@ def _projection_axis(projection: DbProjection, axis: str) -> AxisProjection | No
|
|
|
621
736
|
return projection.five_hour if axis == "fiveHour" else projection.seven_day
|
|
622
737
|
|
|
623
738
|
|
|
739
|
+
def axis_evidence(projection: DbProjection, axis: str) -> AxisEvidence:
|
|
740
|
+
"""The classification evidence of ``axis``: ``present`` whenever the
|
|
741
|
+
axis holds a baseline, else what the covered read recorded, else
|
|
742
|
+
unknown (an absent value alone proves nothing)."""
|
|
743
|
+
if _projection_axis(projection, axis) is not None:
|
|
744
|
+
return AxisEvidence(EVIDENCE_PRESENT)
|
|
745
|
+
recorded = (projection.evidence or {}).get(axis)
|
|
746
|
+
return recorded if isinstance(recorded, AxisEvidence) else EVIDENCE_UNKNOWN_AXIS
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
def classify_absent_axis(evidence: AxisEvidence | None, key: int, *,
|
|
750
|
+
axis: str) -> str:
|
|
751
|
+
"""How a reading in window ``key`` on ``axis``, which holds no projected
|
|
752
|
+
baseline, is classified (spec 4.3 revisions 5 and 6): one of the
|
|
753
|
+
``CLASS_*`` answers.
|
|
754
|
+
|
|
755
|
+
- Proven empty, or a predecessor whose window ``key`` strictly follows:
|
|
756
|
+
a first reading or a new window, so it publishes under a closed drop
|
|
757
|
+
gate too.
|
|
758
|
+
- An expired five-hour predecessor whose key ``key`` does not strictly
|
|
759
|
+
exceed (revision 6, *Stale expired window*, Codex design-fix-round-6
|
|
760
|
+
Q3): stale. On the weekly axis the predecessor's key is only a bound,
|
|
761
|
+
so such a reading stays unknown.
|
|
762
|
+
- A floor-retired group (revision 6, *Floor retired*, Q2): a strictly
|
|
763
|
+
newer group takes the new-window rule, an older group is stale, and a
|
|
764
|
+
reading in the group itself depends on its contributors' capture
|
|
765
|
+
instants (``CLASS_FLOOR``).
|
|
766
|
+
- Anything else is unknown classification."""
|
|
767
|
+
if evidence is None:
|
|
768
|
+
return CLASS_UNKNOWN
|
|
769
|
+
if evidence.kind == EVIDENCE_PROVEN_EMPTY:
|
|
770
|
+
return CLASS_PUBLISH
|
|
771
|
+
if (evidence.kind == EVIDENCE_EXPIRED_PREDECESSOR
|
|
772
|
+
and evidence.predecessor_key is not None):
|
|
773
|
+
if int(key) > int(evidence.predecessor_key):
|
|
774
|
+
return CLASS_PUBLISH
|
|
775
|
+
return CLASS_STALE if axis == "fiveHour" else CLASS_UNKNOWN
|
|
776
|
+
if (evidence.kind == EVIDENCE_FLOOR_RETIRED
|
|
777
|
+
and evidence.group_key is not None
|
|
778
|
+
and evidence.floor_epoch is not None):
|
|
779
|
+
if int(key) > int(evidence.group_key):
|
|
780
|
+
return CLASS_PUBLISH
|
|
781
|
+
if int(key) < int(evidence.group_key):
|
|
782
|
+
return CLASS_STALE
|
|
783
|
+
return CLASS_FLOOR
|
|
784
|
+
return CLASS_UNKNOWN
|
|
785
|
+
|
|
786
|
+
|
|
787
|
+
def classifies_first_reading(evidence: AxisEvidence | None, key: int) -> bool:
|
|
788
|
+
"""Whether a reading in window ``key`` on an axis with no projected
|
|
789
|
+
baseline is a first reading or a new window whatever its contributors
|
|
790
|
+
(`classify_absent_axis` answers ``publish``). Such a reading is never a
|
|
791
|
+
drop, so it publishes under a closed drop gate too."""
|
|
792
|
+
return classify_absent_axis(evidence, key, axis="sevenDay") == CLASS_PUBLISH
|
|
793
|
+
|
|
794
|
+
|
|
624
795
|
def _axis_value_key(value: AxisValue) -> int:
|
|
625
796
|
return value.canonical_key if value.canonical_key is not None else value.raw_resets_at
|
|
626
797
|
|
|
@@ -793,13 +964,22 @@ def _axis_control_changed(before: PendingDrop | None, after: PendingDrop | None)
|
|
|
793
964
|
return before != after
|
|
794
965
|
|
|
795
966
|
|
|
796
|
-
def _build_retry_signature(
|
|
967
|
+
def _build_retry_signature(
|
|
968
|
+
reduced: AxisValue, db_axis: AxisProjection | None, *, scope=None,
|
|
969
|
+
) -> RetrySignature:
|
|
970
|
+
"""``scope`` is ``(protocol_generation, account_key, axis)`` for a bound
|
|
971
|
+
partition (#825/#832 spec 4.3), so two accounts never share a spent
|
|
972
|
+
attempt even with identical values."""
|
|
973
|
+
protocol, account, axis = scope if scope is not None else (None, None, None)
|
|
797
974
|
return RetrySignature(
|
|
798
975
|
candidate_key=_axis_value_key(reduced),
|
|
799
976
|
candidate_percent=reduced.percent,
|
|
800
977
|
db_key=None if db_axis is None else db_axis.canonical_key,
|
|
801
978
|
db_percent=None if db_axis is None else db_axis.percent,
|
|
802
979
|
db_reset_generation=None if db_axis is None else db_axis.reset_generation,
|
|
980
|
+
protocol_generation=protocol,
|
|
981
|
+
account_key=account,
|
|
982
|
+
axis=axis,
|
|
803
983
|
)
|
|
804
984
|
|
|
805
985
|
|
|
@@ -807,6 +987,8 @@ def _pending_kernel_attempt(
|
|
|
807
987
|
pending: PendingDrop,
|
|
808
988
|
reduced: AxisValue,
|
|
809
989
|
db_axis: AxisProjection | None,
|
|
990
|
+
*,
|
|
991
|
+
scope=None,
|
|
810
992
|
) -> PendingDrop | None:
|
|
811
993
|
"""Return the persisted state for one bounded kernel attempt.
|
|
812
994
|
|
|
@@ -815,7 +997,7 @@ def _pending_kernel_attempt(
|
|
|
815
997
|
plan. A successful post-record re-reduction clears it from DB truth; an
|
|
816
998
|
unchanged projection retains this bounded retry state for the next tick.
|
|
817
999
|
"""
|
|
818
|
-
signature = _build_retry_signature(reduced, db_axis)
|
|
1000
|
+
signature = _build_retry_signature(reduced, db_axis, scope=scope)
|
|
819
1001
|
if pending.kernel_stage == "suppressed":
|
|
820
1002
|
if pending.retry_signature == signature:
|
|
821
1003
|
return None
|
|
@@ -888,7 +1070,8 @@ def _supporters_retracted(
|
|
|
888
1070
|
|
|
889
1071
|
|
|
890
1072
|
def _hold_pending(
|
|
891
|
-
pending: PendingDrop | None, db_axis: AxisProjection | None, now_epoch: int
|
|
1073
|
+
pending: PendingDrop | None, db_axis: AxisProjection | None, now_epoch: int,
|
|
1074
|
+
*, scope=None, drop_allowed: bool = True,
|
|
892
1075
|
) -> _AxisDecision:
|
|
893
1076
|
"""Decide a pending drop with no eligible candidate on its axis.
|
|
894
1077
|
|
|
@@ -910,10 +1093,13 @@ def _hold_pending(
|
|
|
910
1093
|
if db_axis.percent <= pending.retained.percent:
|
|
911
1094
|
# The database already holds the drop, so the record has done its work.
|
|
912
1095
|
return _AxisDecision("WRITE_CONTROL", None, None)
|
|
913
|
-
if now_epoch < pending.deadline_at:
|
|
1096
|
+
if now_epoch < pending.deadline_at or not drop_allowed:
|
|
1097
|
+
# #825/#832 spec 5.6 item 7: while the account lacks either recovery
|
|
1098
|
+
# certificate a drop below the database baseline is not published at
|
|
1099
|
+
# its deadline either. It stays pending, with no attempt spent.
|
|
914
1100
|
return _AxisDecision("NOOP", None, pending)
|
|
915
1101
|
publish = _retained_axis_value(pending.retained, db_axis)
|
|
916
|
-
attempted = _pending_kernel_attempt(pending, publish, db_axis)
|
|
1102
|
+
attempted = _pending_kernel_attempt(pending, publish, db_axis, scope=scope)
|
|
917
1103
|
if attempted is not None:
|
|
918
1104
|
return _AxisDecision("PUBLISH_DB", publish, attempted)
|
|
919
1105
|
# The bounded attempts are spent and no contributor remains that could
|
|
@@ -928,22 +1114,71 @@ def _reduce_axis(
|
|
|
928
1114
|
pending: PendingDrop | None,
|
|
929
1115
|
tombstone: Tombstone | None,
|
|
930
1116
|
now_epoch: int,
|
|
1117
|
+
*,
|
|
1118
|
+
scope=None,
|
|
1119
|
+
drop_allowed: bool = True,
|
|
1120
|
+
evidence: AxisEvidence | None = None,
|
|
931
1121
|
) -> _AxisDecision:
|
|
932
1122
|
reduced_data = _reduced_candidate(
|
|
933
1123
|
candidates, axis=axis, tombstone=tombstone, db_axis=db_axis
|
|
934
1124
|
)
|
|
935
1125
|
if reduced_data is None:
|
|
936
|
-
return _hold_pending(pending, db_axis, now_epoch
|
|
1126
|
+
return _hold_pending(pending, db_axis, now_epoch, scope=scope,
|
|
1127
|
+
drop_allowed=drop_allowed)
|
|
937
1128
|
reduced, in_window = reduced_data
|
|
938
1129
|
key = _axis_value_key(reduced)
|
|
939
1130
|
|
|
940
1131
|
if db_axis is None:
|
|
1132
|
+
verdict = classify_absent_axis(evidence, key, axis=axis)
|
|
1133
|
+
if verdict == CLASS_FLOOR:
|
|
1134
|
+
# Spec 4.3 revision 6 (*Floor retired*, Codex design-fix-round-6
|
|
1135
|
+
# Q2): the group's admitted floor retired every accepted reading.
|
|
1136
|
+
# Only contributors captured at or after the floor are the
|
|
1137
|
+
# ordinary first post-floor reading; one captured before it is
|
|
1138
|
+
# stale and is never relabelled as post-floor. The reading opens
|
|
1139
|
+
# no consensus and spends no attempt, under either gate.
|
|
1140
|
+
post = tuple((candidate, value) for candidate, value in in_window
|
|
1141
|
+
if candidate.received_at >= int(evidence.floor_epoch))
|
|
1142
|
+
if not post:
|
|
1143
|
+
verdict = CLASS_STALE
|
|
1144
|
+
else:
|
|
1145
|
+
maximum = max(value.percent for _, value in post)
|
|
1146
|
+
reduced = AxisValue(
|
|
1147
|
+
maximum,
|
|
1148
|
+
min(value.raw_resets_at for _, value in post
|
|
1149
|
+
if value.percent == maximum),
|
|
1150
|
+
canonical_key=key)
|
|
1151
|
+
verdict = CLASS_PUBLISH
|
|
1152
|
+
if verdict == CLASS_UNKNOWN:
|
|
1153
|
+
# #825/#832 review finding P1-2 (Codex fix-round-4 Q2): a missing
|
|
1154
|
+
# projection alone proves no empty account — it cannot tell a
|
|
1155
|
+
# first reading from a drop below a baseline it does not show.
|
|
1156
|
+
# Only classification evidence does (spec 4.3 revision 5,
|
|
1157
|
+
# design-fix-round-5 Q1): a proven-empty baseline, or an expired
|
|
1158
|
+
# five-hour predecessor whose window this reading's strictly
|
|
1159
|
+
# follows. Without it the reading is unknown: never published,
|
|
1160
|
+
# and a pending drop's evidence is kept. Fix round K13 (review
|
|
1161
|
+
# cycle 7 P1-2): whatever the drop gate — an open gate is no
|
|
1162
|
+
# classification evidence either.
|
|
1163
|
+
return _AxisDecision("NOOP", None, pending, unknown=True)
|
|
1164
|
+
if verdict == CLASS_STALE:
|
|
1165
|
+
# Spec 4.3 revision 6 (*Stale expired window*, *Floor retired*):
|
|
1166
|
+
# the existing older-window disposition — nothing published on
|
|
1167
|
+
# this axis, no consensus opened, the pending drop kept.
|
|
1168
|
+
return _hold_pending(pending, None, now_epoch, scope=scope,
|
|
1169
|
+
drop_allowed=drop_allowed)
|
|
1170
|
+
if (evidence.kind == EVIDENCE_FLOOR_RETIRED and pending is not None
|
|
1171
|
+
and pending.canonical_key > int(evidence.group_key)):
|
|
1172
|
+
# Pending state retires only where the floor proves it obsolete:
|
|
1173
|
+
# a pending drop of a newer window is not the retired group's.
|
|
1174
|
+
return _AxisDecision("PUBLISH_DB", reduced, pending)
|
|
941
1175
|
return _AxisDecision("PUBLISH_DB", reduced, None)
|
|
942
1176
|
if key > db_axis.canonical_key:
|
|
943
1177
|
return _AxisDecision("PUBLISH_DB", reduced, None)
|
|
944
1178
|
if key < db_axis.canonical_key:
|
|
945
1179
|
# A stale older window says nothing about the pending newer one.
|
|
946
|
-
return _hold_pending(pending, db_axis, now_epoch
|
|
1180
|
+
return _hold_pending(pending, db_axis, now_epoch, scope=scope,
|
|
1181
|
+
drop_allowed=drop_allowed)
|
|
947
1182
|
# `reduced` is the MAXIMUM across active contributors, so it can equal or
|
|
948
1183
|
# exceed the database value while a lower contributor is reporting a real
|
|
949
1184
|
# drop. The maximum decides what is displayed; it may not erase the
|
|
@@ -1017,8 +1252,13 @@ def _reduce_axis(
|
|
|
1017
1252
|
and bool(updated.contributors)
|
|
1018
1253
|
and all(item.satisfied for item in updated.contributors.values())
|
|
1019
1254
|
)
|
|
1020
|
-
if
|
|
1021
|
-
|
|
1255
|
+
if not drop_allowed:
|
|
1256
|
+
# #825/#832 spec 5.6 item 7: withheld — neither consensus nor the
|
|
1257
|
+
# deadline publishes a drop below the baseline, and no bounded attempt
|
|
1258
|
+
# is spent; the record stays pending with its evidence.
|
|
1259
|
+
pass
|
|
1260
|
+
elif unanimous:
|
|
1261
|
+
attempted = _pending_kernel_attempt(updated, reduced, db_axis, scope=scope)
|
|
1022
1262
|
if attempted is not None:
|
|
1023
1263
|
return _AxisDecision("PUBLISH_DB", reduced, attempted)
|
|
1024
1264
|
elif now_epoch >= updated.deadline_at:
|
|
@@ -1027,7 +1267,7 @@ def _reduce_axis(
|
|
|
1027
1267
|
# all-contributor maximum, never the old high, and never a fresh
|
|
1028
1268
|
# generation over the same drop.
|
|
1029
1269
|
publish = _retained_axis_value(updated.retained, db_axis)
|
|
1030
|
-
attempted = _pending_kernel_attempt(updated, publish, db_axis)
|
|
1270
|
+
attempted = _pending_kernel_attempt(updated, publish, db_axis, scope=scope)
|
|
1031
1271
|
if attempted is not None:
|
|
1032
1272
|
return _AxisDecision("PUBLISH_DB", publish, attempted)
|
|
1033
1273
|
# The bounded attempts are spent against an unchanged signature, so this
|
|
@@ -1056,11 +1296,29 @@ def reduce_candidates(
|
|
|
1056
1296
|
control: ControlState,
|
|
1057
1297
|
tombstones: Mapping[str, Tombstone | None],
|
|
1058
1298
|
now_epoch: int,
|
|
1299
|
+
account_key: str | None = None,
|
|
1300
|
+
drop_consensus_allowed: bool = True,
|
|
1059
1301
|
) -> ReductionDecision:
|
|
1060
1302
|
"""Reduce active candidates independently for 5h and 7d axes.
|
|
1061
1303
|
|
|
1062
1304
|
The output describes only a pure state transition. The caller owns
|
|
1063
1305
|
filesystem persistence, DB reconciliation, and invoking the record kernel.
|
|
1306
|
+
|
|
1307
|
+
``account_key`` names the bound partition (#825/#832 spec 4.3): every
|
|
1308
|
+
retry signature minted here then carries the namespace
|
|
1309
|
+
``(PROTOCOL_GENERATION, account_key, axis)``. The caller passes only that
|
|
1310
|
+
partition's candidates, control and tombstones.
|
|
1311
|
+
|
|
1312
|
+
``drop_consensus_allowed`` is the outcome-quarantine control gate
|
|
1313
|
+
(#825/#832 spec 5.6 item 7). False while the account lacks either recovery
|
|
1314
|
+
certificate: a pending drop below the database baseline is then published
|
|
1315
|
+
neither by consensus nor at its deadline, and stays pending; rises and new
|
|
1316
|
+
windows publish as always. An axis with no projected baseline publishes
|
|
1317
|
+
under either gate only when ``db`` carries classification evidence for
|
|
1318
|
+
it (`classify_absent_axis`: a first reading, a new window or the first
|
|
1319
|
+
reading after a retiring floor); a stale reading takes the older-window
|
|
1320
|
+
disposition, and an unclassifiable one is named in ``unknown_axes`` (fix
|
|
1321
|
+
rounds K13 and K14c).
|
|
1064
1322
|
"""
|
|
1065
1323
|
active = tuple(
|
|
1066
1324
|
candidate
|
|
@@ -1074,6 +1332,9 @@ def reduce_candidates(
|
|
|
1074
1332
|
control.pending_drops.get("fiveHour"),
|
|
1075
1333
|
tombstones.get("fiveHour"),
|
|
1076
1334
|
now_epoch,
|
|
1335
|
+
scope=None if account_key is None else (PROTOCOL_GENERATION, account_key, "fiveHour"),
|
|
1336
|
+
drop_allowed=drop_consensus_allowed,
|
|
1337
|
+
evidence=axis_evidence(db, "fiveHour"),
|
|
1077
1338
|
)
|
|
1078
1339
|
seven = _reduce_axis(
|
|
1079
1340
|
"sevenDay",
|
|
@@ -1082,17 +1343,25 @@ def reduce_candidates(
|
|
|
1082
1343
|
control.pending_drops.get("sevenDay"),
|
|
1083
1344
|
tombstones.get("sevenDay"),
|
|
1084
1345
|
now_epoch,
|
|
1346
|
+
scope=None if account_key is None else (PROTOCOL_GENERATION, account_key, "sevenDay"),
|
|
1347
|
+
drop_allowed=drop_consensus_allowed,
|
|
1348
|
+
evidence=axis_evidence(db, "sevenDay"),
|
|
1085
1349
|
)
|
|
1086
1350
|
next_control = ControlState(
|
|
1087
1351
|
db_projection=db,
|
|
1088
1352
|
pending_drops={"fiveHour": five.pending, "sevenDay": seven.pending},
|
|
1089
1353
|
)
|
|
1354
|
+
unknown = frozenset(
|
|
1355
|
+
axis for axis, decided in (("fiveHour", five), ("sevenDay", seven))
|
|
1356
|
+
if decided.unknown)
|
|
1090
1357
|
if five.action == "PUBLISH_DB" or seven.action == "PUBLISH_DB":
|
|
1091
1358
|
return ReductionDecision(
|
|
1092
1359
|
"PUBLISH_DB",
|
|
1093
1360
|
next_control,
|
|
1094
1361
|
PublicationPlan(seven_day=seven.candidate, five_hour=five.candidate),
|
|
1362
|
+
unknown_axes=unknown,
|
|
1095
1363
|
)
|
|
1096
1364
|
if five.action == "WRITE_CONTROL" or seven.action == "WRITE_CONTROL":
|
|
1097
|
-
return ReductionDecision("WRITE_CONTROL", next_control
|
|
1098
|
-
|
|
1365
|
+
return ReductionDecision("WRITE_CONTROL", next_control,
|
|
1366
|
+
unknown_axes=unknown)
|
|
1367
|
+
return ReductionDecision("NOOP", next_control, unknown_axes=unknown)
|