cctally 1.72.0 → 1.73.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/bin/_cctally_cache.py +245 -50
- package/bin/_cctally_core.py +20 -40
- package/bin/_cctally_dashboard_conversation.py +731 -94
- package/bin/_cctally_doctor.py +59 -0
- package/bin/_cctally_parser.py +17 -4
- package/bin/_cctally_record.py +162 -33
- package/bin/_cctally_refresh.py +58 -36
- package/bin/_cctally_statusline.py +811 -82
- package/bin/_cctally_transcript.py +223 -7
- package/bin/_lib_codex_conversation_export.py +124 -0
- package/bin/_lib_codex_conversation_query.py +500 -28
- package/bin/_lib_codex_conversation_watch.py +323 -0
- package/bin/_lib_conversation_dispatch.py +269 -7
- package/bin/_lib_conversation_query.py +88 -0
- package/bin/_lib_doctor.py +62 -0
- package/bin/_lib_statusline_candidates.py +734 -0
- package/bin/cctally +31 -2
- package/package.json +4 -1
|
@@ -0,0 +1,734 @@
|
|
|
1
|
+
"""Pure candidate-arbitration schemas and reducer primitives for #318.
|
|
2
|
+
|
|
3
|
+
This module deliberately has no filesystem, database, clock, process, or
|
|
4
|
+
``cctally`` namespace dependency. Callers supply all mutable state and time.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import dataclasses
|
|
9
|
+
import json
|
|
10
|
+
import math
|
|
11
|
+
import re
|
|
12
|
+
from typing import Callable, Mapping
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
SCHEMA_VERSION = 1
|
|
16
|
+
CANDIDATE_DOCUMENT_MAX_BYTES = 4 * 1024
|
|
17
|
+
CONTROL_DOCUMENT_MAX_BYTES = 1024 * 1024
|
|
18
|
+
TOMBSTONE_DOCUMENT_MAX_BYTES = 1024
|
|
19
|
+
CANDIDATE_TOKEN_RE = re.compile(r"[0-9a-f]{64}\Z")
|
|
20
|
+
AXES = ("fiveHour", "sevenDay")
|
|
21
|
+
_SIGNED_I64_MIN = -(2**63)
|
|
22
|
+
_SIGNED_I64_MAX = 2**63 - 1
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class StateValidationError(ValueError):
|
|
26
|
+
"""A persisted arbitration artifact is malformed or outside its contract."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclasses.dataclass(frozen=True)
|
|
30
|
+
class AxisValue:
|
|
31
|
+
percent: float
|
|
32
|
+
raw_resets_at: int
|
|
33
|
+
canonical_key: int | None = None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclasses.dataclass(frozen=True)
|
|
37
|
+
class Candidate:
|
|
38
|
+
token: str = ""
|
|
39
|
+
received_at: int = 0
|
|
40
|
+
five_hour: AxisValue | None = None
|
|
41
|
+
seven_day: AxisValue | None = None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclasses.dataclass(frozen=True)
|
|
45
|
+
class AxisProjection:
|
|
46
|
+
percent: float
|
|
47
|
+
raw_resets_at: int
|
|
48
|
+
canonical_key: int
|
|
49
|
+
captured_at: int
|
|
50
|
+
source: str
|
|
51
|
+
reset_generation: int
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclasses.dataclass(frozen=True)
|
|
55
|
+
class FileFingerprint:
|
|
56
|
+
device: int
|
|
57
|
+
inode: int
|
|
58
|
+
size: int
|
|
59
|
+
mtime_ns: int
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclasses.dataclass(frozen=True)
|
|
63
|
+
class DbProjection:
|
|
64
|
+
five_hour: AxisProjection | None
|
|
65
|
+
seven_day: AxisProjection | None
|
|
66
|
+
db_files: Mapping[str, FileFingerprint | None] | None = None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclasses.dataclass(frozen=True)
|
|
70
|
+
class Contributor:
|
|
71
|
+
baseline_received_at: int
|
|
72
|
+
satisfied: bool
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclasses.dataclass(frozen=True)
|
|
76
|
+
class RetrySignature:
|
|
77
|
+
candidate_key: int
|
|
78
|
+
candidate_percent: float
|
|
79
|
+
db_key: int | None
|
|
80
|
+
db_percent: float | None
|
|
81
|
+
db_reset_generation: int | None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclasses.dataclass(frozen=True)
|
|
85
|
+
class PendingDrop:
|
|
86
|
+
canonical_key: int
|
|
87
|
+
reduced_percent: float
|
|
88
|
+
first_seen_at: int
|
|
89
|
+
kernel_stage: str
|
|
90
|
+
attempts: int
|
|
91
|
+
contributors: Mapping[str, Contributor]
|
|
92
|
+
retry_signature: RetrySignature | None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@dataclasses.dataclass(frozen=True)
|
|
96
|
+
class ControlState:
|
|
97
|
+
db_projection: DbProjection
|
|
98
|
+
pending_drops: Mapping[str, PendingDrop | None]
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@dataclasses.dataclass(frozen=True)
|
|
102
|
+
class Tombstone:
|
|
103
|
+
axis: str
|
|
104
|
+
state: str
|
|
105
|
+
started_at: int | None = None
|
|
106
|
+
prior_block_received_at_through: int | None = None
|
|
107
|
+
block_received_at_through: int | None = None
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@dataclasses.dataclass(frozen=True)
|
|
111
|
+
class PublicationPlan:
|
|
112
|
+
seven_day: AxisValue | None
|
|
113
|
+
five_hour: AxisValue | None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@dataclasses.dataclass(frozen=True)
|
|
117
|
+
class ReductionDecision:
|
|
118
|
+
action: str
|
|
119
|
+
control: ControlState
|
|
120
|
+
plan: PublicationPlan | None = None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _is_int(value: object) -> bool:
|
|
124
|
+
return (
|
|
125
|
+
isinstance(value, int)
|
|
126
|
+
and not isinstance(value, bool)
|
|
127
|
+
and _SIGNED_I64_MIN <= value <= _SIGNED_I64_MAX
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _is_nonnegative_int(value: object) -> bool:
|
|
132
|
+
return _is_int(value) and value >= 0
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _percent(value: object) -> float:
|
|
136
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
137
|
+
raise StateValidationError("percent must be numeric")
|
|
138
|
+
result = float(value)
|
|
139
|
+
if not math.isfinite(result) or not 0.0 <= result <= 100.0:
|
|
140
|
+
raise StateValidationError("percent outside [0,100]")
|
|
141
|
+
return result
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _require_object(value: object, label: str) -> dict:
|
|
145
|
+
if not isinstance(value, dict):
|
|
146
|
+
raise StateValidationError(f"{label} must be an object")
|
|
147
|
+
return value
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _require_exact_keys(doc: dict, required: set[str]) -> None:
|
|
151
|
+
if set(doc) != required:
|
|
152
|
+
raise StateValidationError("unexpected document keys")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _require_ascii_source(value: object) -> str:
|
|
156
|
+
if not isinstance(value, str) or not (1 <= len(value) <= 64):
|
|
157
|
+
raise StateValidationError("source must be 1..64 ASCII characters")
|
|
158
|
+
try:
|
|
159
|
+
value.encode("ascii")
|
|
160
|
+
except UnicodeEncodeError as exc:
|
|
161
|
+
raise StateValidationError("source must be ASCII") from exc
|
|
162
|
+
return value
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _axis_value(
|
|
166
|
+
value: object,
|
|
167
|
+
*,
|
|
168
|
+
axis: str,
|
|
169
|
+
reset_is_plausible: Callable[[str, int], bool],
|
|
170
|
+
) -> AxisValue:
|
|
171
|
+
doc = _require_object(value, axis)
|
|
172
|
+
_require_exact_keys(doc, {"percent", "resetsAt"})
|
|
173
|
+
resets_at = doc["resetsAt"]
|
|
174
|
+
if not _is_int(resets_at) or not reset_is_plausible(axis, resets_at):
|
|
175
|
+
raise StateValidationError("invalid resetsAt")
|
|
176
|
+
return AxisValue(percent=_percent(doc["percent"]), raw_resets_at=resets_at)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _reject_json_constant(value: str) -> None:
|
|
180
|
+
raise StateValidationError(f"non-standard JSON constant {value}")
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _load_json_document(raw: str | bytes, *, maximum_bytes: int) -> object:
|
|
184
|
+
if isinstance(raw, bytes):
|
|
185
|
+
data = raw
|
|
186
|
+
try:
|
|
187
|
+
raw_text = raw.decode("utf-8")
|
|
188
|
+
except UnicodeDecodeError as exc:
|
|
189
|
+
raise StateValidationError("document is not UTF-8") from exc
|
|
190
|
+
elif isinstance(raw, str):
|
|
191
|
+
raw_text = raw
|
|
192
|
+
data = raw.encode("utf-8")
|
|
193
|
+
else:
|
|
194
|
+
raise StateValidationError("document must be text")
|
|
195
|
+
if len(data) > maximum_bytes:
|
|
196
|
+
raise StateValidationError("document exceeds size limit")
|
|
197
|
+
try:
|
|
198
|
+
return json.loads(raw_text, parse_constant=_reject_json_constant)
|
|
199
|
+
except StateValidationError:
|
|
200
|
+
raise
|
|
201
|
+
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
202
|
+
raise StateValidationError("invalid JSON document") from exc
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def validate_candidate_document(
|
|
206
|
+
document: object,
|
|
207
|
+
*,
|
|
208
|
+
now_epoch: int,
|
|
209
|
+
reset_is_plausible: Callable[[str, int], bool],
|
|
210
|
+
token: str = "",
|
|
211
|
+
) -> Candidate:
|
|
212
|
+
"""Validate one persisted candidate document without reading filesystem state."""
|
|
213
|
+
doc = _require_object(document, "candidate")
|
|
214
|
+
allowed = {"schemaVersion", "receivedAt", "fiveHour", "sevenDay"}
|
|
215
|
+
if (set(doc) - allowed or not {"schemaVersion", "receivedAt"} <= set(doc)
|
|
216
|
+
or not ({"fiveHour", "sevenDay"} & set(doc))):
|
|
217
|
+
raise StateValidationError("unexpected document keys")
|
|
218
|
+
if not _is_int(doc["schemaVersion"]) or doc["schemaVersion"] != SCHEMA_VERSION:
|
|
219
|
+
raise StateValidationError("unsupported schemaVersion")
|
|
220
|
+
received_at = doc["receivedAt"]
|
|
221
|
+
if not _is_int(received_at) or not 0 <= received_at <= now_epoch + 5:
|
|
222
|
+
raise StateValidationError("invalid receivedAt")
|
|
223
|
+
if token and not CANDIDATE_TOKEN_RE.fullmatch(token):
|
|
224
|
+
raise StateValidationError("invalid candidate token")
|
|
225
|
+
return Candidate(
|
|
226
|
+
token=token,
|
|
227
|
+
received_at=received_at,
|
|
228
|
+
five_hour=(
|
|
229
|
+
_axis_value(doc["fiveHour"], axis="fiveHour", reset_is_plausible=reset_is_plausible)
|
|
230
|
+
if "fiveHour" in doc else None
|
|
231
|
+
),
|
|
232
|
+
seven_day=(
|
|
233
|
+
_axis_value(doc["sevenDay"], axis="sevenDay", reset_is_plausible=reset_is_plausible)
|
|
234
|
+
if "sevenDay" in doc else None
|
|
235
|
+
),
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def load_candidate_document(
|
|
240
|
+
raw: str | bytes,
|
|
241
|
+
*,
|
|
242
|
+
now_epoch: int,
|
|
243
|
+
reset_is_plausible: Callable[[str, int], bool],
|
|
244
|
+
token: str = "",
|
|
245
|
+
) -> Candidate:
|
|
246
|
+
return validate_candidate_document(
|
|
247
|
+
_load_json_document(raw, maximum_bytes=CANDIDATE_DOCUMENT_MAX_BYTES),
|
|
248
|
+
now_epoch=now_epoch,
|
|
249
|
+
reset_is_plausible=reset_is_plausible,
|
|
250
|
+
token=token,
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _axis_projection(value: object) -> AxisProjection | None:
|
|
255
|
+
if value is None:
|
|
256
|
+
return None
|
|
257
|
+
doc = _require_object(value, "axis projection")
|
|
258
|
+
_require_exact_keys(
|
|
259
|
+
doc,
|
|
260
|
+
{"percent", "rawResetsAt", "canonicalKey", "capturedAt", "source", "resetGeneration"},
|
|
261
|
+
)
|
|
262
|
+
for key in ("rawResetsAt", "canonicalKey", "capturedAt"):
|
|
263
|
+
if not _is_int(doc[key]):
|
|
264
|
+
raise StateValidationError(f"{key} must be an integer")
|
|
265
|
+
if not _is_nonnegative_int(doc["resetGeneration"]):
|
|
266
|
+
raise StateValidationError("resetGeneration must be nonnegative")
|
|
267
|
+
return AxisProjection(
|
|
268
|
+
percent=_percent(doc["percent"]),
|
|
269
|
+
raw_resets_at=doc["rawResetsAt"],
|
|
270
|
+
canonical_key=doc["canonicalKey"],
|
|
271
|
+
captured_at=doc["capturedAt"],
|
|
272
|
+
source=_require_ascii_source(doc["source"]),
|
|
273
|
+
reset_generation=doc["resetGeneration"],
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _fingerprint(value: object) -> FileFingerprint | None:
|
|
278
|
+
if value is None:
|
|
279
|
+
return None
|
|
280
|
+
doc = _require_object(value, "fingerprint")
|
|
281
|
+
_require_exact_keys(doc, {"device", "inode", "size", "mtimeNs"})
|
|
282
|
+
if not all(_is_nonnegative_int(doc[k]) for k in doc):
|
|
283
|
+
raise StateValidationError("invalid fingerprint")
|
|
284
|
+
return FileFingerprint(
|
|
285
|
+
device=doc["device"], inode=doc["inode"], size=doc["size"], mtime_ns=doc["mtimeNs"]
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _nullable_int(value: object, label: str) -> int | None:
|
|
290
|
+
if value is None:
|
|
291
|
+
return None
|
|
292
|
+
if not _is_int(value):
|
|
293
|
+
raise StateValidationError(f"{label} must be an integer or null")
|
|
294
|
+
return value
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _nullable_percent(value: object, label: str) -> float | None:
|
|
298
|
+
if value is None:
|
|
299
|
+
return None
|
|
300
|
+
return _percent(value)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _retry_signature(value: object) -> RetrySignature | None:
|
|
304
|
+
if value is None:
|
|
305
|
+
return None
|
|
306
|
+
doc = _require_object(value, "retrySignature")
|
|
307
|
+
_require_exact_keys(
|
|
308
|
+
doc,
|
|
309
|
+
{"candidateKey", "candidatePercent", "dbKey", "dbPercent", "dbResetGeneration"},
|
|
310
|
+
)
|
|
311
|
+
if not _is_int(doc["candidateKey"]):
|
|
312
|
+
raise StateValidationError("candidateKey must be an integer")
|
|
313
|
+
db_generation = doc["dbResetGeneration"]
|
|
314
|
+
if db_generation is not None and not _is_nonnegative_int(db_generation):
|
|
315
|
+
raise StateValidationError("dbResetGeneration must be nonnegative or null")
|
|
316
|
+
return RetrySignature(
|
|
317
|
+
candidate_key=doc["candidateKey"],
|
|
318
|
+
candidate_percent=_percent(doc["candidatePercent"]),
|
|
319
|
+
db_key=_nullable_int(doc["dbKey"], "dbKey"),
|
|
320
|
+
db_percent=_nullable_percent(doc["dbPercent"], "dbPercent"),
|
|
321
|
+
db_reset_generation=db_generation,
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _pending_drop(value: object, *, now_epoch: int) -> PendingDrop | None:
|
|
326
|
+
if value is None:
|
|
327
|
+
return None
|
|
328
|
+
doc = _require_object(value, "pending drop")
|
|
329
|
+
_require_exact_keys(
|
|
330
|
+
doc,
|
|
331
|
+
{"canonicalKey", "reducedPercent", "firstSeenAt", "kernelStage", "attempts", "contributors", "retrySignature"},
|
|
332
|
+
)
|
|
333
|
+
if not _is_int(doc["canonicalKey"]):
|
|
334
|
+
raise StateValidationError("canonicalKey must be an integer")
|
|
335
|
+
if not _is_int(doc["firstSeenAt"]) or not 0 <= doc["firstSeenAt"] <= now_epoch + 5:
|
|
336
|
+
raise StateValidationError("invalid firstSeenAt")
|
|
337
|
+
if doc["kernelStage"] not in {"settling", "ready", "zero_armed", "suppressed"}:
|
|
338
|
+
raise StateValidationError("invalid kernelStage")
|
|
339
|
+
if not _is_int(doc["attempts"]) or not 0 <= doc["attempts"] <= 2:
|
|
340
|
+
raise StateValidationError("invalid attempts")
|
|
341
|
+
contributors_doc = _require_object(doc["contributors"], "contributors")
|
|
342
|
+
if len(contributors_doc) > 4096:
|
|
343
|
+
raise StateValidationError("too many contributors")
|
|
344
|
+
contributors: dict[str, Contributor] = {}
|
|
345
|
+
for token, contributor in contributors_doc.items():
|
|
346
|
+
if not isinstance(token, str) or not CANDIDATE_TOKEN_RE.fullmatch(token):
|
|
347
|
+
raise StateValidationError("invalid contributor token")
|
|
348
|
+
contributor_doc = _require_object(contributor, "contributor")
|
|
349
|
+
_require_exact_keys(contributor_doc, {"baselineReceivedAt", "satisfied"})
|
|
350
|
+
baseline = contributor_doc["baselineReceivedAt"]
|
|
351
|
+
if not _is_int(baseline) or not 0 <= baseline <= now_epoch + 5:
|
|
352
|
+
raise StateValidationError("invalid baselineReceivedAt")
|
|
353
|
+
if not isinstance(contributor_doc["satisfied"], bool):
|
|
354
|
+
raise StateValidationError("satisfied must be boolean")
|
|
355
|
+
contributors[token] = Contributor(
|
|
356
|
+
baseline_received_at=baseline, satisfied=contributor_doc["satisfied"]
|
|
357
|
+
)
|
|
358
|
+
return PendingDrop(
|
|
359
|
+
canonical_key=doc["canonicalKey"],
|
|
360
|
+
reduced_percent=_percent(doc["reducedPercent"]),
|
|
361
|
+
first_seen_at=doc["firstSeenAt"],
|
|
362
|
+
kernel_stage=doc["kernelStage"],
|
|
363
|
+
attempts=doc["attempts"],
|
|
364
|
+
contributors=contributors,
|
|
365
|
+
retry_signature=_retry_signature(doc["retrySignature"]),
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def validate_control_document(document: object, *, now_epoch: int) -> ControlState:
|
|
370
|
+
doc = _require_object(document, "control state")
|
|
371
|
+
_require_exact_keys(doc, {"schemaVersion", "dbProjection", "dbFiles", "pendingDrops"})
|
|
372
|
+
if not _is_int(doc["schemaVersion"]) or doc["schemaVersion"] != SCHEMA_VERSION:
|
|
373
|
+
raise StateValidationError("unsupported schemaVersion")
|
|
374
|
+
projection_doc = _require_object(doc["dbProjection"], "dbProjection")
|
|
375
|
+
_require_exact_keys(projection_doc, {"fiveHour", "sevenDay"})
|
|
376
|
+
files_doc = _require_object(doc["dbFiles"], "dbFiles")
|
|
377
|
+
_require_exact_keys(files_doc, {"main", "wal"})
|
|
378
|
+
main = _fingerprint(files_doc["main"])
|
|
379
|
+
if main is None:
|
|
380
|
+
raise StateValidationError("dbFiles.main is required")
|
|
381
|
+
pending_doc = _require_object(doc["pendingDrops"], "pendingDrops")
|
|
382
|
+
_require_exact_keys(pending_doc, {"fiveHour", "sevenDay"})
|
|
383
|
+
return ControlState(
|
|
384
|
+
db_projection=DbProjection(
|
|
385
|
+
five_hour=_axis_projection(projection_doc["fiveHour"]),
|
|
386
|
+
seven_day=_axis_projection(projection_doc["sevenDay"]),
|
|
387
|
+
db_files={"main": main, "wal": _fingerprint(files_doc["wal"])},
|
|
388
|
+
),
|
|
389
|
+
pending_drops={
|
|
390
|
+
"fiveHour": _pending_drop(pending_doc["fiveHour"], now_epoch=now_epoch),
|
|
391
|
+
"sevenDay": _pending_drop(pending_doc["sevenDay"], now_epoch=now_epoch),
|
|
392
|
+
},
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def load_control_document(raw: str | bytes, *, now_epoch: int) -> ControlState:
|
|
397
|
+
return validate_control_document(
|
|
398
|
+
_load_json_document(raw, maximum_bytes=CONTROL_DOCUMENT_MAX_BYTES), now_epoch=now_epoch
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def validate_tombstone_document(
|
|
403
|
+
document: object, *, expected_axis: str, now_epoch: int
|
|
404
|
+
) -> Tombstone:
|
|
405
|
+
doc = _require_object(document, "tombstone")
|
|
406
|
+
if expected_axis not in AXES:
|
|
407
|
+
raise StateValidationError("invalid expected axis")
|
|
408
|
+
if not _is_int(doc.get("schemaVersion")) or doc["schemaVersion"] != SCHEMA_VERSION:
|
|
409
|
+
raise StateValidationError("unsupported schemaVersion")
|
|
410
|
+
if doc.get("axis") != expected_axis:
|
|
411
|
+
raise StateValidationError("wrong tombstone axis")
|
|
412
|
+
state = doc.get("state")
|
|
413
|
+
if state == "inflight":
|
|
414
|
+
_require_exact_keys(
|
|
415
|
+
doc,
|
|
416
|
+
{"schemaVersion", "axis", "state", "startedAt", "priorBlockReceivedAtThrough"},
|
|
417
|
+
)
|
|
418
|
+
started = doc["startedAt"]
|
|
419
|
+
prior = doc["priorBlockReceivedAtThrough"]
|
|
420
|
+
if not _is_int(started) or not 0 <= started <= now_epoch + 5:
|
|
421
|
+
raise StateValidationError("invalid startedAt")
|
|
422
|
+
if prior is not None and (not _is_int(prior) or not 0 <= prior <= now_epoch + 5):
|
|
423
|
+
raise StateValidationError("invalid priorBlockReceivedAtThrough")
|
|
424
|
+
return Tombstone(
|
|
425
|
+
axis=expected_axis,
|
|
426
|
+
state=state,
|
|
427
|
+
started_at=started,
|
|
428
|
+
prior_block_received_at_through=prior,
|
|
429
|
+
)
|
|
430
|
+
if state == "committed":
|
|
431
|
+
_require_exact_keys(
|
|
432
|
+
doc,
|
|
433
|
+
{"schemaVersion", "axis", "state", "blockReceivedAtThrough"},
|
|
434
|
+
)
|
|
435
|
+
cutoff = doc["blockReceivedAtThrough"]
|
|
436
|
+
if not _is_int(cutoff) or not 0 <= cutoff <= now_epoch + 5:
|
|
437
|
+
raise StateValidationError("invalid blockReceivedAtThrough")
|
|
438
|
+
return Tombstone(axis=expected_axis, state=state, block_received_at_through=cutoff)
|
|
439
|
+
raise StateValidationError("invalid tombstone state")
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def load_tombstone_document(
|
|
443
|
+
raw: str | bytes, *, expected_axis: str, now_epoch: int
|
|
444
|
+
) -> Tombstone:
|
|
445
|
+
return validate_tombstone_document(
|
|
446
|
+
_load_json_document(raw, maximum_bytes=TOMBSTONE_DOCUMENT_MAX_BYTES),
|
|
447
|
+
expected_axis=expected_axis,
|
|
448
|
+
now_epoch=now_epoch,
|
|
449
|
+
)
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def canonicalize_five_hour_axes(
|
|
453
|
+
values: list[AxisValue] | tuple[AxisValue, ...],
|
|
454
|
+
*,
|
|
455
|
+
db_anchor: tuple[int, int] | None,
|
|
456
|
+
canonicalize: Callable[[int, tuple[int, int] | None], int],
|
|
457
|
+
) -> tuple[AxisValue, ...]:
|
|
458
|
+
"""Canonicalize sorted raw 5h resets with a rolling physical-window anchor.
|
|
459
|
+
|
|
460
|
+
A distant old DB anchor must not make two adjacent new raw resets form two
|
|
461
|
+
artificial windows. Only a newly established key advances the rolling
|
|
462
|
+
anchor; a reused key retains the first raw reset of that cluster.
|
|
463
|
+
"""
|
|
464
|
+
anchor = db_anchor
|
|
465
|
+
result: list[AxisValue] = []
|
|
466
|
+
for value in sorted(values, key=lambda item: item.raw_resets_at):
|
|
467
|
+
key = canonicalize(value.raw_resets_at, anchor)
|
|
468
|
+
if anchor is None or key != anchor[1]:
|
|
469
|
+
anchor = (value.raw_resets_at, key)
|
|
470
|
+
result.append(dataclasses.replace(value, canonical_key=key))
|
|
471
|
+
return tuple(result)
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
@dataclasses.dataclass(frozen=True)
|
|
475
|
+
class _AxisDecision:
|
|
476
|
+
action: str
|
|
477
|
+
candidate: AxisValue | None
|
|
478
|
+
pending: PendingDrop | None
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _candidate_axis(candidate: Candidate, axis: str) -> AxisValue | None:
|
|
482
|
+
return candidate.five_hour if axis == "fiveHour" else candidate.seven_day
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _projection_axis(projection: DbProjection, axis: str) -> AxisProjection | None:
|
|
486
|
+
return projection.five_hour if axis == "fiveHour" else projection.seven_day
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _axis_value_key(value: AxisValue) -> int:
|
|
490
|
+
return value.canonical_key if value.canonical_key is not None else value.raw_resets_at
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def _eligible_candidate_axis(
|
|
494
|
+
candidate: Candidate,
|
|
495
|
+
axis: str,
|
|
496
|
+
tombstone: Tombstone | None,
|
|
497
|
+
) -> AxisValue | None:
|
|
498
|
+
value = _candidate_axis(candidate, axis)
|
|
499
|
+
if value is None:
|
|
500
|
+
return None
|
|
501
|
+
if tombstone is None:
|
|
502
|
+
return value
|
|
503
|
+
if tombstone.state == "inflight":
|
|
504
|
+
return None
|
|
505
|
+
if (tombstone.state == "committed"
|
|
506
|
+
and tombstone.block_received_at_through is not None
|
|
507
|
+
and candidate.received_at <= tombstone.block_received_at_through):
|
|
508
|
+
return None
|
|
509
|
+
return value
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _reduced_candidate(
|
|
513
|
+
candidates: tuple[Candidate, ...],
|
|
514
|
+
*,
|
|
515
|
+
axis: str,
|
|
516
|
+
tombstone: Tombstone | None,
|
|
517
|
+
db_axis: AxisProjection | None,
|
|
518
|
+
) -> tuple[AxisValue, tuple[tuple[Candidate, AxisValue], ...]] | None:
|
|
519
|
+
eligible = tuple(
|
|
520
|
+
(candidate, value)
|
|
521
|
+
for candidate in candidates
|
|
522
|
+
if (value := _eligible_candidate_axis(candidate, axis, tombstone)) is not None
|
|
523
|
+
)
|
|
524
|
+
if not eligible:
|
|
525
|
+
return None
|
|
526
|
+
newest_key = max(_axis_value_key(value) for _, value in eligible)
|
|
527
|
+
in_window = tuple(
|
|
528
|
+
(candidate, value)
|
|
529
|
+
for candidate, value in eligible if _axis_value_key(value) == newest_key
|
|
530
|
+
)
|
|
531
|
+
maximum = max(value.percent for _, value in in_window)
|
|
532
|
+
maxima = tuple((candidate, value) for candidate, value in in_window if value.percent == maximum)
|
|
533
|
+
if db_axis is not None and db_axis.canonical_key == newest_key:
|
|
534
|
+
raw = db_axis.raw_resets_at
|
|
535
|
+
else:
|
|
536
|
+
raw = min(value.raw_resets_at for _, value in maxima)
|
|
537
|
+
return AxisValue(maximum, raw, canonical_key=newest_key), in_window
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def _new_pending(
|
|
541
|
+
reduced: AxisValue, contributors: tuple[tuple[Candidate, AxisValue], ...], now_epoch: int
|
|
542
|
+
) -> PendingDrop:
|
|
543
|
+
return PendingDrop(
|
|
544
|
+
canonical_key=_axis_value_key(reduced),
|
|
545
|
+
reduced_percent=reduced.percent,
|
|
546
|
+
first_seen_at=now_epoch,
|
|
547
|
+
kernel_stage="settling",
|
|
548
|
+
attempts=0,
|
|
549
|
+
contributors={
|
|
550
|
+
candidate.token: Contributor(candidate.received_at, False)
|
|
551
|
+
for candidate, _ in contributors
|
|
552
|
+
},
|
|
553
|
+
retry_signature=None,
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def _reconcile_pending(
|
|
558
|
+
pending: PendingDrop,
|
|
559
|
+
contributors: tuple[tuple[Candidate, AxisValue], ...],
|
|
560
|
+
) -> PendingDrop:
|
|
561
|
+
current = {candidate.token: candidate for candidate, _ in contributors}
|
|
562
|
+
merged: dict[str, Contributor] = {}
|
|
563
|
+
for token, candidate in current.items():
|
|
564
|
+
old = pending.contributors.get(token)
|
|
565
|
+
if old is None:
|
|
566
|
+
merged[token] = Contributor(candidate.received_at, False)
|
|
567
|
+
else:
|
|
568
|
+
merged[token] = Contributor(
|
|
569
|
+
baseline_received_at=old.baseline_received_at,
|
|
570
|
+
satisfied=(old.satisfied or candidate.received_at > old.baseline_received_at),
|
|
571
|
+
)
|
|
572
|
+
return dataclasses.replace(pending, contributors=merged)
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def _axis_control_changed(before: PendingDrop | None, after: PendingDrop | None) -> bool:
|
|
576
|
+
return before != after
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def _build_retry_signature(reduced: AxisValue, db_axis: AxisProjection | None) -> RetrySignature:
|
|
580
|
+
return RetrySignature(
|
|
581
|
+
candidate_key=_axis_value_key(reduced),
|
|
582
|
+
candidate_percent=reduced.percent,
|
|
583
|
+
db_key=None if db_axis is None else db_axis.canonical_key,
|
|
584
|
+
db_percent=None if db_axis is None else db_axis.percent,
|
|
585
|
+
db_reset_generation=None if db_axis is None else db_axis.reset_generation,
|
|
586
|
+
)
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def _pending_kernel_attempt(
|
|
590
|
+
pending: PendingDrop,
|
|
591
|
+
reduced: AxisValue,
|
|
592
|
+
db_axis: AxisProjection | None,
|
|
593
|
+
) -> PendingDrop | None:
|
|
594
|
+
"""Return the persisted state for one bounded kernel attempt.
|
|
595
|
+
|
|
596
|
+
The reducer cannot know whether the external record kernel will mutate the
|
|
597
|
+
DB, so it advances the per-axis retry state *before* returning a publication
|
|
598
|
+
plan. A successful post-record re-reduction clears it from DB truth; an
|
|
599
|
+
unchanged projection retains this bounded retry state for the next tick.
|
|
600
|
+
"""
|
|
601
|
+
signature = _build_retry_signature(reduced, db_axis)
|
|
602
|
+
if pending.kernel_stage == "suppressed":
|
|
603
|
+
if pending.retry_signature == signature:
|
|
604
|
+
return None
|
|
605
|
+
return dataclasses.replace(
|
|
606
|
+
pending,
|
|
607
|
+
kernel_stage="zero_armed" if reduced.percent == 0.0 else "ready",
|
|
608
|
+
attempts=1,
|
|
609
|
+
retry_signature=signature,
|
|
610
|
+
)
|
|
611
|
+
if pending.kernel_stage == "settling" or pending.retry_signature != signature:
|
|
612
|
+
return dataclasses.replace(
|
|
613
|
+
pending,
|
|
614
|
+
kernel_stage="zero_armed" if reduced.percent == 0.0 else "ready",
|
|
615
|
+
attempts=1,
|
|
616
|
+
retry_signature=signature,
|
|
617
|
+
)
|
|
618
|
+
if pending.kernel_stage in {"ready", "zero_armed"}:
|
|
619
|
+
# The second attempt is the final expensive retry. For zero this is
|
|
620
|
+
# also the revalidated confirmation pass after cmd_record_usage armed
|
|
621
|
+
# its existing debounce marker on the first attempt.
|
|
622
|
+
return dataclasses.replace(
|
|
623
|
+
pending,
|
|
624
|
+
kernel_stage="suppressed",
|
|
625
|
+
attempts=2,
|
|
626
|
+
retry_signature=signature,
|
|
627
|
+
)
|
|
628
|
+
return None
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def _reduce_axis(
|
|
632
|
+
axis: str,
|
|
633
|
+
candidates: tuple[Candidate, ...],
|
|
634
|
+
db_axis: AxisProjection | None,
|
|
635
|
+
pending: PendingDrop | None,
|
|
636
|
+
tombstone: Tombstone | None,
|
|
637
|
+
now_epoch: int,
|
|
638
|
+
) -> _AxisDecision:
|
|
639
|
+
reduced_data = _reduced_candidate(
|
|
640
|
+
candidates, axis=axis, tombstone=tombstone, db_axis=db_axis
|
|
641
|
+
)
|
|
642
|
+
if reduced_data is None:
|
|
643
|
+
return _AxisDecision("WRITE_CONTROL" if pending is not None else "NOOP", None, None)
|
|
644
|
+
reduced, in_window = reduced_data
|
|
645
|
+
key = _axis_value_key(reduced)
|
|
646
|
+
|
|
647
|
+
if db_axis is None:
|
|
648
|
+
return _AxisDecision("PUBLISH_DB", reduced, None)
|
|
649
|
+
if key > db_axis.canonical_key:
|
|
650
|
+
return _AxisDecision("PUBLISH_DB", reduced, None)
|
|
651
|
+
if key < db_axis.canonical_key:
|
|
652
|
+
return _AxisDecision("NOOP", None, pending)
|
|
653
|
+
if reduced.percent > db_axis.percent:
|
|
654
|
+
return _AxisDecision("PUBLISH_DB", reduced, None)
|
|
655
|
+
if reduced.percent == db_axis.percent:
|
|
656
|
+
return _AxisDecision(
|
|
657
|
+
"WRITE_CONTROL" if pending is not None else "NOOP",
|
|
658
|
+
None,
|
|
659
|
+
None,
|
|
660
|
+
)
|
|
661
|
+
|
|
662
|
+
# The current max is lower than the current DB value. Because `reduced`
|
|
663
|
+
# is that max, every active contributor in this canonical window is lower.
|
|
664
|
+
if pending is None or pending.canonical_key != key:
|
|
665
|
+
return _AxisDecision("WRITE_CONTROL", None, _new_pending(reduced, in_window, now_epoch))
|
|
666
|
+
if reduced.percent > pending.reduced_percent:
|
|
667
|
+
# A lower-only upward correction starts a distinct consensus generation;
|
|
668
|
+
# old baselines must never be mutated into this new target.
|
|
669
|
+
return _AxisDecision("WRITE_CONTROL", None, _new_pending(reduced, in_window, now_epoch))
|
|
670
|
+
|
|
671
|
+
reconciled = _reconcile_pending(pending, in_window)
|
|
672
|
+
if reconciled.contributors and all(item.satisfied for item in reconciled.contributors.values()):
|
|
673
|
+
attempted = _pending_kernel_attempt(reconciled, reduced, db_axis)
|
|
674
|
+
if attempted is not None:
|
|
675
|
+
return _AxisDecision("PUBLISH_DB", reduced, attempted)
|
|
676
|
+
return _AxisDecision(
|
|
677
|
+
"WRITE_CONTROL" if _axis_control_changed(pending, reconciled) else "NOOP",
|
|
678
|
+
None,
|
|
679
|
+
reconciled,
|
|
680
|
+
)
|
|
681
|
+
return _AxisDecision(
|
|
682
|
+
"WRITE_CONTROL" if _axis_control_changed(pending, reconciled) else "NOOP",
|
|
683
|
+
None,
|
|
684
|
+
reconciled,
|
|
685
|
+
)
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def reduce_candidates(
|
|
689
|
+
candidates: tuple[Candidate, ...] | list[Candidate],
|
|
690
|
+
*,
|
|
691
|
+
db: DbProjection,
|
|
692
|
+
control: ControlState,
|
|
693
|
+
tombstones: Mapping[str, Tombstone | None],
|
|
694
|
+
now_epoch: int,
|
|
695
|
+
) -> ReductionDecision:
|
|
696
|
+
"""Reduce active candidates independently for 5h and 7d axes.
|
|
697
|
+
|
|
698
|
+
The output describes only a pure state transition. The caller owns
|
|
699
|
+
filesystem persistence, DB reconciliation, and invoking the record kernel.
|
|
700
|
+
"""
|
|
701
|
+
active = tuple(
|
|
702
|
+
candidate
|
|
703
|
+
for candidate in candidates
|
|
704
|
+
if -5 <= now_epoch - candidate.received_at < 90
|
|
705
|
+
)
|
|
706
|
+
five = _reduce_axis(
|
|
707
|
+
"fiveHour",
|
|
708
|
+
active,
|
|
709
|
+
db.five_hour,
|
|
710
|
+
control.pending_drops.get("fiveHour"),
|
|
711
|
+
tombstones.get("fiveHour"),
|
|
712
|
+
now_epoch,
|
|
713
|
+
)
|
|
714
|
+
seven = _reduce_axis(
|
|
715
|
+
"sevenDay",
|
|
716
|
+
active,
|
|
717
|
+
db.seven_day,
|
|
718
|
+
control.pending_drops.get("sevenDay"),
|
|
719
|
+
tombstones.get("sevenDay"),
|
|
720
|
+
now_epoch,
|
|
721
|
+
)
|
|
722
|
+
next_control = ControlState(
|
|
723
|
+
db_projection=db,
|
|
724
|
+
pending_drops={"fiveHour": five.pending, "sevenDay": seven.pending},
|
|
725
|
+
)
|
|
726
|
+
if five.action == "PUBLISH_DB" or seven.action == "PUBLISH_DB":
|
|
727
|
+
return ReductionDecision(
|
|
728
|
+
"PUBLISH_DB",
|
|
729
|
+
next_control,
|
|
730
|
+
PublicationPlan(seven_day=seven.candidate, five_hour=five.candidate),
|
|
731
|
+
)
|
|
732
|
+
if five.action == "WRITE_CONTROL" or seven.action == "WRITE_CONTROL":
|
|
733
|
+
return ReductionDecision("WRITE_CONTROL", next_control)
|
|
734
|
+
return ReductionDecision("NOOP", next_control)
|