cctally 1.114.0 → 1.115.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/bin/_cctally_account.py +32 -8
- package/bin/_cctally_cache.py +4394 -279
- package/bin/_cctally_completion.py +1287 -0
- package/bin/_cctally_config.py +37 -0
- package/bin/_cctally_core.py +178 -2
- package/bin/_cctally_dashboard.py +2 -2
- package/bin/_cctally_dashboard_conversation.py +292 -23
- package/bin/_cctally_dashboard_envelope.py +16 -3
- package/bin/_cctally_dashboard_sources.py +18 -9
- package/bin/_cctally_db.py +316 -5
- package/bin/_cctally_doctor.py +436 -27
- package/bin/_cctally_five_hour.py +82 -31
- package/bin/_cctally_forecast.py +81 -9
- package/bin/_cctally_journal.py +3884 -112
- package/bin/_cctally_journal_repair.py +101 -12
- package/bin/_cctally_parser.py +42 -0
- package/bin/_cctally_project.py +8 -2
- package/bin/_cctally_quarantine.py +652 -0
- package/bin/_cctally_quota.py +26 -4
- package/bin/_cctally_quota_model.py +360 -83
- package/bin/_cctally_record.py +677 -149
- package/bin/_cctally_rederive.py +297 -32
- package/bin/_cctally_refresh.py +84 -13
- package/bin/_cctally_source_analytics.py +23 -2
- package/bin/_cctally_statusline.py +6137 -517
- package/bin/_cctally_store.py +287 -48
- package/bin/_cctally_transcript.py +8 -7
- package/bin/_cctally_tui.py +57 -53
- package/bin/_lib_account_binding.py +152 -0
- package/bin/_lib_account_protocol.py +386 -0
- package/bin/_lib_aggregators.py +4 -0
- package/bin/_lib_cache_coverage.py +13 -1
- package/bin/_lib_cache_writer_lock.py +19 -3
- package/bin/_lib_clone_copy.py +142 -0
- package/bin/_lib_codex_conversation_query.py +133 -19
- package/bin/_lib_codex_conversation_watch.py +59 -14
- package/bin/_lib_codex_drain_admission.py +214 -0
- package/bin/_lib_codex_metadata.py +18 -1
- package/bin/_lib_codex_publication_barrier.py +277 -0
- package/bin/_lib_codex_rollout_container.py +502 -0
- package/bin/_lib_conversation_dispatch.py +3 -0
- package/bin/_lib_conversation_query.py +11 -3
- package/bin/_lib_detector_access_registry.py +3817 -0
- package/bin/_lib_doctor.py +452 -1
- package/bin/_lib_five_hour.py +126 -0
- package/bin/_lib_ingest_frontier.py +414 -20
- package/bin/_lib_journal.py +141 -10
- package/bin/_lib_jsonl.py +5 -4
- package/bin/_lib_legacy_spool.py +126 -0
- package/bin/_lib_legacy_writer_evidence.py +136 -0
- package/bin/_lib_merge_gate.py +29 -302
- package/bin/_lib_meter_rate_change.py +10 -0
- package/bin/_lib_outcome_quarantine.py +690 -0
- package/bin/_lib_pricing.py +7 -2
- package/bin/_lib_quarantine.py +154 -0
- package/bin/_lib_quota_calibration.py +53 -8
- package/bin/_lib_quota_copy.py +2 -1
- package/bin/_lib_quota_model.py +288 -46
- package/bin/_lib_rederive.py +94 -5
- package/bin/_lib_segment_summary.py +35 -2
- package/bin/_lib_statusline_candidates.py +292 -23
- package/bin/cctally +23 -1
- package/package.json +14 -1
|
@@ -0,0 +1,690 @@
|
|
|
1
|
+
"""Pure outcome quarantine and certified account recovery (#825/#832, spec 5.6).
|
|
2
|
+
|
|
3
|
+
After the account-bound cutover, an older cctally process can still write.
|
|
4
|
+
Its records are excluded from current accounting (spec 5.3), but excluding a
|
|
5
|
+
reading changes what the detectors see: even an excluded reading that would
|
|
6
|
+
not itself have produced an outcome can change a LATER detection. So every
|
|
7
|
+
excluded record and every legacy spool change is an uncertainty TRIGGER that
|
|
8
|
+
quarantines every Claude account: automatic in-place credits, five-hour
|
|
9
|
+
credits, provider-moved week-end transitions and drop consensus are withheld
|
|
10
|
+
until the account holds a recovery certificate.
|
|
11
|
+
|
|
12
|
+
This module is the pure half:
|
|
13
|
+
|
|
14
|
+
* `Reading` — one Claude rate-limit observation as the detectors see it.
|
|
15
|
+
* `Envelope` — monotone running bounds over the WHOLE journal prefix (never
|
|
16
|
+
truncated at a trigger or a revocation): the greatest week-start date key,
|
|
17
|
+
five-hour key, raw five-hour reset, weekly end, stored capture ordering and
|
|
18
|
+
effective instant any reading or effect in that prefix could have stored.
|
|
19
|
+
* `certify` — the seven clauses of spec 5.6 item 4, every one evaluated, so a
|
|
20
|
+
refusal names ALL failed clauses.
|
|
21
|
+
* `Certificate` / `continuing_validity` — a certificate is revocable: backward
|
|
22
|
+
input, a clock or ordering regression, or a fallback read that reaches the
|
|
23
|
+
uncertain prefix through a cross-period access not registered inert revokes
|
|
24
|
+
it before any automatic outcome.
|
|
25
|
+
* `QuarantineState` — the deterministic fold over journal order that live
|
|
26
|
+
ingest, `db rebuild --db stats` and the `db rederive` scratch replay all run,
|
|
27
|
+
so triggers, envelopes and certificates are a function of the journal alone.
|
|
28
|
+
* The unresolved spool marker (spec revision 3, section 5.6 item 1(b); Codex
|
|
29
|
+
design-fix-round-4 Q6). A `legacy_spool_activity` op records only digests,
|
|
30
|
+
never a legacy value, so the candidate it stands for contributes no bound to
|
|
31
|
+
any envelope, and a fresh-looking reading cannot certify against the bounds
|
|
32
|
+
the envelope does hold. Every such trigger therefore persists a marker keyed
|
|
33
|
+
by its journal coordinate and `at`, and while any marker exists no account —
|
|
34
|
+
one first seen later included — holds an accounting certificate. Nothing
|
|
35
|
+
clears a marker in this outcome (automatic recovery is follow-up #948);
|
|
36
|
+
the control certificate still reconciles on its own.
|
|
37
|
+
|
|
38
|
+
Deliberately absent (the design consultation rejected each): a taint map, a
|
|
39
|
+
timer, and an irrevocable clean flag. Diagnostic quiet time never decides
|
|
40
|
+
recovery.
|
|
41
|
+
|
|
42
|
+
Pure: no I/O, no clock.
|
|
43
|
+
"""
|
|
44
|
+
from __future__ import annotations
|
|
45
|
+
|
|
46
|
+
import dataclasses
|
|
47
|
+
import json
|
|
48
|
+
|
|
49
|
+
#: `bin/_cctally_record.py` `_derive_5h_window_key`: the nearest
|
|
50
|
+
#: `five_hour_blocks` raw reset within 1,800 s anchors the window.
|
|
51
|
+
TOLERANCE_BLOCK_SEARCH_S = 1800
|
|
52
|
+
#: `bin/_lib_five_hour.py` `_FIVE_HOUR_JITTER_FLOOR_SECONDS`: the rolling
|
|
53
|
+
#: canonicalizer's jitter tolerance.
|
|
54
|
+
TOLERANCE_CANONICAL_S = 600
|
|
55
|
+
|
|
56
|
+
#: The envelope scope whose bounds apply to EVERY account: excluded readings
|
|
57
|
+
#: and effects under every reference interpretation (absent, or any Claude
|
|
58
|
+
#: account including `unattributed` and accounts first seen later), plus
|
|
59
|
+
#: admitted historical readings that carry no account at all.
|
|
60
|
+
EXCLUDED_SCOPE = "excluded"
|
|
61
|
+
|
|
62
|
+
INELIGIBLE_REASONS = (
|
|
63
|
+
"projection-fill", "retained-low", "reviewed-hold", "manual-synthetic",
|
|
64
|
+
)
|
|
65
|
+
#: A witness that is not eligible at all (spec 5.6 item 4 preamble).
|
|
66
|
+
INELIGIBLE_SINGLE_AXIS = "single-axis"
|
|
67
|
+
INELIGIBLE_UNBOUND = "unbound"
|
|
68
|
+
|
|
69
|
+
REVOCATION_REASONS = (
|
|
70
|
+
"backward-key", "clock-regression", "order-regression",
|
|
71
|
+
"non-inert-fallback",
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
CLAUSES = (1, 2, 3, 4, 5, 6, 7)
|
|
75
|
+
|
|
76
|
+
#: The trigger kind of a current `legacy_spool_activity` op. Each one leaves
|
|
77
|
+
#: an unresolved spool marker (spec revision 3, section 5.6 item 1(b)).
|
|
78
|
+
TRIGGER_KIND_LEGACY_SPOOL = "legacy-spool-activity"
|
|
79
|
+
#: The gate reason while a spool marker is unresolved.
|
|
80
|
+
GATE_REASON_SPOOL_MARKER = "unresolved-spool-marker"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _max(a, b):
|
|
84
|
+
if a is None:
|
|
85
|
+
return b
|
|
86
|
+
if b is None:
|
|
87
|
+
return a
|
|
88
|
+
return a if a >= b else b
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclasses.dataclass(frozen=True)
|
|
92
|
+
class Reading:
|
|
93
|
+
"""One observation as the detector sees it.
|
|
94
|
+
|
|
95
|
+
``captured_at_utc`` is the value the snapshot row stores in
|
|
96
|
+
``captured_at_utc`` (the payload's capture stamp verbatim), because the
|
|
97
|
+
cross-period reads order on that TEXT column lexically.
|
|
98
|
+
``detection_epoch`` is the record's ``at`` (the detection clock).
|
|
99
|
+
``five_hour_key`` is the planned account-local key a current producer
|
|
100
|
+
journaled, else the pure canonical floor of the raw reset."""
|
|
101
|
+
|
|
102
|
+
observation_id: str
|
|
103
|
+
account_key: "str | None"
|
|
104
|
+
captured_at_utc: str
|
|
105
|
+
capture_epoch: int
|
|
106
|
+
detection_epoch: int
|
|
107
|
+
week_start_date: "str | None"
|
|
108
|
+
week_start_at_epoch: "int | None"
|
|
109
|
+
weekly_end_epoch: "int | None"
|
|
110
|
+
five_hour_key: "int | None"
|
|
111
|
+
raw_five_hour_reset: "int | None"
|
|
112
|
+
two_axis: bool
|
|
113
|
+
ineligible_reason: "str | None" = None
|
|
114
|
+
stamped: bool = True
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@dataclasses.dataclass(frozen=True)
|
|
118
|
+
class Envelope:
|
|
119
|
+
"""Monotone running bounds over the WHOLE prefix (never truncated)."""
|
|
120
|
+
|
|
121
|
+
max_week_start_date: "str | None" = None
|
|
122
|
+
max_five_hour_key: "int | None" = None
|
|
123
|
+
max_raw_five_hour_reset: "int | None" = None
|
|
124
|
+
max_weekly_end_epoch: "int | None" = None
|
|
125
|
+
#: Lexical, as stored in `captured_at_utc`.
|
|
126
|
+
max_capture_at_utc: "str | None" = None
|
|
127
|
+
#: Reset/credit effective instants AND every capture and detection
|
|
128
|
+
#: instant: an automatic credit's effective instant is at or before its
|
|
129
|
+
#: arming tick, so absorbing every reading's instants bounds them
|
|
130
|
+
#: conservatively, including effects no surviving row records.
|
|
131
|
+
max_effective_instant_epoch: "int | None" = None
|
|
132
|
+
|
|
133
|
+
def absorb_reading(self, r: Reading) -> "Envelope":
|
|
134
|
+
return Envelope(
|
|
135
|
+
max_week_start_date=_max(self.max_week_start_date, r.week_start_date),
|
|
136
|
+
max_five_hour_key=_max(self.max_five_hour_key, r.five_hour_key),
|
|
137
|
+
max_raw_five_hour_reset=_max(
|
|
138
|
+
self.max_raw_five_hour_reset, r.raw_five_hour_reset),
|
|
139
|
+
max_weekly_end_epoch=_max(self.max_weekly_end_epoch, r.weekly_end_epoch),
|
|
140
|
+
max_capture_at_utc=_max(self.max_capture_at_utc, r.captured_at_utc),
|
|
141
|
+
max_effective_instant_epoch=_max(
|
|
142
|
+
self.max_effective_instant_epoch,
|
|
143
|
+
_max(r.capture_epoch, r.detection_epoch)),
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
def absorb_effect(self, *, effective_epoch: "int | None",
|
|
147
|
+
week_start_date: "str | None" = None,
|
|
148
|
+
five_hour_key: "int | None" = None,
|
|
149
|
+
captured_at_utc: "str | None" = None,
|
|
150
|
+
weekly_end_epoch: "int | None" = None,
|
|
151
|
+
raw_five_hour_reset: "int | None" = None) -> "Envelope":
|
|
152
|
+
"""An effect any reference could hold: an event's effective instant, a
|
|
153
|
+
manual floor's date key, a synthetic or exported snapshot's keys."""
|
|
154
|
+
return Envelope(
|
|
155
|
+
max_week_start_date=_max(self.max_week_start_date, week_start_date),
|
|
156
|
+
max_five_hour_key=_max(self.max_five_hour_key, five_hour_key),
|
|
157
|
+
max_raw_five_hour_reset=_max(
|
|
158
|
+
self.max_raw_five_hour_reset, raw_five_hour_reset),
|
|
159
|
+
max_weekly_end_epoch=_max(self.max_weekly_end_epoch, weekly_end_epoch),
|
|
160
|
+
max_capture_at_utc=_max(self.max_capture_at_utc, captured_at_utc),
|
|
161
|
+
max_effective_instant_epoch=_max(
|
|
162
|
+
self.max_effective_instant_epoch, effective_epoch),
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
@staticmethod
|
|
166
|
+
def merge(a: "Envelope", b: "Envelope") -> "Envelope":
|
|
167
|
+
return Envelope(
|
|
168
|
+
max_week_start_date=_max(a.max_week_start_date, b.max_week_start_date),
|
|
169
|
+
max_five_hour_key=_max(a.max_five_hour_key, b.max_five_hour_key),
|
|
170
|
+
max_raw_five_hour_reset=_max(
|
|
171
|
+
a.max_raw_five_hour_reset, b.max_raw_five_hour_reset),
|
|
172
|
+
max_weekly_end_epoch=_max(a.max_weekly_end_epoch, b.max_weekly_end_epoch),
|
|
173
|
+
max_capture_at_utc=_max(a.max_capture_at_utc, b.max_capture_at_utc),
|
|
174
|
+
max_effective_instant_epoch=_max(
|
|
175
|
+
a.max_effective_instant_epoch, b.max_effective_instant_epoch),
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
def to_document(self) -> dict:
|
|
179
|
+
return {
|
|
180
|
+
"maxWeekStartDate": self.max_week_start_date,
|
|
181
|
+
"maxFiveHourKey": self.max_five_hour_key,
|
|
182
|
+
"maxRawFiveHourReset": self.max_raw_five_hour_reset,
|
|
183
|
+
"maxWeeklyEndEpoch": self.max_weekly_end_epoch,
|
|
184
|
+
"maxCaptureAtUtc": self.max_capture_at_utc,
|
|
185
|
+
"maxEffectiveInstantEpoch": self.max_effective_instant_epoch,
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
@staticmethod
|
|
189
|
+
def from_document(doc) -> "Envelope | None":
|
|
190
|
+
"""The envelope a document describes, or None when it is not a valid
|
|
191
|
+
envelope document. An unverifiable envelope must never read as an
|
|
192
|
+
empty (permissive) one, so callers treat None as quarantined."""
|
|
193
|
+
if not isinstance(doc, dict):
|
|
194
|
+
return None
|
|
195
|
+
keys = {"maxWeekStartDate", "maxFiveHourKey", "maxRawFiveHourReset",
|
|
196
|
+
"maxWeeklyEndEpoch", "maxCaptureAtUtc",
|
|
197
|
+
"maxEffectiveInstantEpoch"}
|
|
198
|
+
if set(doc) != keys:
|
|
199
|
+
return None
|
|
200
|
+
for key in ("maxWeekStartDate", "maxCaptureAtUtc"):
|
|
201
|
+
if doc[key] is not None and not isinstance(doc[key], str):
|
|
202
|
+
return None
|
|
203
|
+
for key in ("maxFiveHourKey", "maxRawFiveHourReset",
|
|
204
|
+
"maxWeeklyEndEpoch", "maxEffectiveInstantEpoch"):
|
|
205
|
+
if doc[key] is not None and type(doc[key]) is not int:
|
|
206
|
+
return None
|
|
207
|
+
return Envelope(
|
|
208
|
+
max_week_start_date=doc["maxWeekStartDate"],
|
|
209
|
+
max_five_hour_key=doc["maxFiveHourKey"],
|
|
210
|
+
max_raw_five_hour_reset=doc["maxRawFiveHourReset"],
|
|
211
|
+
max_weekly_end_epoch=doc["maxWeeklyEndEpoch"],
|
|
212
|
+
max_capture_at_utc=doc["maxCaptureAtUtc"],
|
|
213
|
+
max_effective_instant_epoch=doc["maxEffectiveInstantEpoch"],
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
@dataclasses.dataclass(frozen=True)
|
|
218
|
+
class CertificateDecision:
|
|
219
|
+
certified: bool
|
|
220
|
+
#: Subset of 1..7, empty when certified or when the witness is ineligible.
|
|
221
|
+
failed_clauses: "tuple[int, ...]"
|
|
222
|
+
ineligible: "str | None" = None
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def witness_ineligibility(witness: Reading) -> "str | None":
|
|
226
|
+
"""Why ``witness`` can never certify, or None (spec 5.6 item 4)."""
|
|
227
|
+
if witness.ineligible_reason is not None:
|
|
228
|
+
return witness.ineligible_reason
|
|
229
|
+
if (not witness.stamped or witness.account_key is None):
|
|
230
|
+
return INELIGIBLE_UNBOUND
|
|
231
|
+
if (not witness.two_axis or witness.week_start_date is None
|
|
232
|
+
or witness.five_hour_key is None
|
|
233
|
+
or witness.raw_five_hour_reset is None
|
|
234
|
+
or witness.week_start_at_epoch is None):
|
|
235
|
+
return INELIGIBLE_SINGLE_AXIS
|
|
236
|
+
return None
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _debounce_could_match(witness: Reading, envelope: Envelope) -> bool:
|
|
240
|
+
"""Clause 6, by the comparisons the detector performs.
|
|
241
|
+
|
|
242
|
+
The weekly debounce state is ONE row per account; the detector treats it
|
|
243
|
+
as armed for a reading iff the stored `week_start_date` equals the
|
|
244
|
+
reading's and the stored end equals its end (`detect_reset_and_credit`).
|
|
245
|
+
Every state the prefix could have armed carries a date key the prefix
|
|
246
|
+
produced, so it is bounded by `max_week_start_date`; a witness date beyond
|
|
247
|
+
that bound can match none of them, whatever its end. The five-hour
|
|
248
|
+
confirmation state is keyed `(account, window key, source)`, so a witness
|
|
249
|
+
key beyond `max_five_hour_key` reads no prefix row."""
|
|
250
|
+
if (envelope.max_week_start_date is not None
|
|
251
|
+
and witness.week_start_date <= envelope.max_week_start_date):
|
|
252
|
+
return True
|
|
253
|
+
if (envelope.max_five_hour_key is not None
|
|
254
|
+
and witness.five_hour_key <= envelope.max_five_hour_key):
|
|
255
|
+
return True
|
|
256
|
+
return False
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def certify(witness: Reading, envelope: Envelope, *,
|
|
260
|
+
seen_identities=frozenset()) -> CertificateDecision:
|
|
261
|
+
"""Evaluate every clause of spec 5.6 item 4 against ``envelope`` (the
|
|
262
|
+
merge of the account's and the excluded scope's bounds) and return ALL
|
|
263
|
+
failed clauses, never just the first."""
|
|
264
|
+
ineligible = witness_ineligibility(witness)
|
|
265
|
+
if ineligible is not None:
|
|
266
|
+
return CertificateDecision(False, (), ineligible)
|
|
267
|
+
failed = []
|
|
268
|
+
env = envelope
|
|
269
|
+
# 1. Period-keyed freshness: the date key and the five-hour key lie
|
|
270
|
+
# beyond every possible prefix key. Comparing normalized ends is not
|
|
271
|
+
# enough, because distinct ends share one date key.
|
|
272
|
+
if ((env.max_week_start_date is not None
|
|
273
|
+
and witness.week_start_date <= env.max_week_start_date)
|
|
274
|
+
or (env.max_five_hour_key is not None
|
|
275
|
+
and witness.five_hour_key <= env.max_five_hour_key)):
|
|
276
|
+
failed.append(1)
|
|
277
|
+
# 2. Anchor selection: no prefix raw reset inside either tolerance of the
|
|
278
|
+
# witness's, so every possible prefix anchor yields the witness's own
|
|
279
|
+
# fresh key (conservatively the SUM of both tolerances).
|
|
280
|
+
if (env.max_raw_five_hour_reset is not None
|
|
281
|
+
and witness.raw_five_hour_reset - env.max_raw_five_hour_reset
|
|
282
|
+
<= TOLERANCE_BLOCK_SEARCH_S + TOLERANCE_CANONICAL_S):
|
|
283
|
+
failed.append(2)
|
|
284
|
+
# 3. Range-keyed reads: no prefix effective instant inside the witness's
|
|
285
|
+
# weekly interval (the sufficient check: its start exceeds them all).
|
|
286
|
+
if (env.max_effective_instant_epoch is not None
|
|
287
|
+
and witness.week_start_at_epoch <= env.max_effective_instant_epoch):
|
|
288
|
+
failed.append(3)
|
|
289
|
+
# 4. Cross-period ordering under the actual SQL ordering
|
|
290
|
+
# (`captured_at_utc DESC, id DESC`, lexical TEXT comparison).
|
|
291
|
+
if (env.max_capture_at_utc is not None
|
|
292
|
+
and witness.captured_at_utc <= env.max_capture_at_utc):
|
|
293
|
+
failed.append(4)
|
|
294
|
+
# 5. Detection clock later than every predecessor's weekly end and raw
|
|
295
|
+
# five-hour reset — separate from capture ordering.
|
|
296
|
+
latest_end = _max(env.max_weekly_end_epoch, env.max_raw_five_hour_reset)
|
|
297
|
+
if latest_end is not None and witness.detection_epoch <= latest_end:
|
|
298
|
+
failed.append(5)
|
|
299
|
+
# 6. Singletons: the weekly debounce row and the five-hour confirmation
|
|
300
|
+
# state cannot match the witness.
|
|
301
|
+
if _debounce_could_match(witness, env):
|
|
302
|
+
failed.append(6)
|
|
303
|
+
# 7. Logical identity: the witness is not a replay of a seen observation.
|
|
304
|
+
if witness.observation_id in seen_identities:
|
|
305
|
+
failed.append(7)
|
|
306
|
+
return CertificateDecision(not failed, tuple(failed), None)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
@dataclasses.dataclass(frozen=True)
|
|
310
|
+
class Certificate:
|
|
311
|
+
witness_observation_id: str
|
|
312
|
+
witness_week_start_date: str
|
|
313
|
+
witness_five_hour_key: int
|
|
314
|
+
witness_captured_at_utc: str
|
|
315
|
+
last_capture_at_utc: str
|
|
316
|
+
last_detection_epoch: int
|
|
317
|
+
|
|
318
|
+
@staticmethod
|
|
319
|
+
def from_witness(r: Reading) -> "Certificate":
|
|
320
|
+
return Certificate(
|
|
321
|
+
witness_observation_id=r.observation_id,
|
|
322
|
+
witness_week_start_date=r.week_start_date,
|
|
323
|
+
witness_five_hour_key=r.five_hour_key,
|
|
324
|
+
witness_captured_at_utc=r.captured_at_utc,
|
|
325
|
+
last_capture_at_utc=r.captured_at_utc,
|
|
326
|
+
last_detection_epoch=r.detection_epoch,
|
|
327
|
+
)
|
|
328
|
+
|
|
329
|
+
def advance(self, r: Reading) -> "Certificate":
|
|
330
|
+
return dataclasses.replace(
|
|
331
|
+
self,
|
|
332
|
+
last_capture_at_utc=_max(self.last_capture_at_utc, r.captured_at_utc),
|
|
333
|
+
last_detection_epoch=_max(self.last_detection_epoch, r.detection_epoch),
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
def to_document(self) -> dict:
|
|
337
|
+
return {
|
|
338
|
+
"witnessObservationId": self.witness_observation_id,
|
|
339
|
+
"witnessWeekStartDate": self.witness_week_start_date,
|
|
340
|
+
"witnessFiveHourKey": self.witness_five_hour_key,
|
|
341
|
+
"witnessCapturedAtUtc": self.witness_captured_at_utc,
|
|
342
|
+
"lastCaptureAtUtc": self.last_capture_at_utc,
|
|
343
|
+
"lastDetectionEpoch": self.last_detection_epoch,
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
@staticmethod
|
|
347
|
+
def from_document(doc) -> "Certificate | None":
|
|
348
|
+
if not isinstance(doc, dict) or set(doc) != {
|
|
349
|
+
"witnessObservationId", "witnessWeekStartDate",
|
|
350
|
+
"witnessFiveHourKey", "witnessCapturedAtUtc",
|
|
351
|
+
"lastCaptureAtUtc", "lastDetectionEpoch"}:
|
|
352
|
+
return None
|
|
353
|
+
for key in ("witnessObservationId", "witnessWeekStartDate",
|
|
354
|
+
"witnessCapturedAtUtc", "lastCaptureAtUtc"):
|
|
355
|
+
if not isinstance(doc[key], str) or not doc[key]:
|
|
356
|
+
return None
|
|
357
|
+
for key in ("witnessFiveHourKey", "lastDetectionEpoch"):
|
|
358
|
+
if type(doc[key]) is not int:
|
|
359
|
+
return None
|
|
360
|
+
return Certificate(
|
|
361
|
+
witness_observation_id=doc["witnessObservationId"],
|
|
362
|
+
witness_week_start_date=doc["witnessWeekStartDate"],
|
|
363
|
+
witness_five_hour_key=doc["witnessFiveHourKey"],
|
|
364
|
+
witness_captured_at_utc=doc["witnessCapturedAtUtc"],
|
|
365
|
+
last_capture_at_utc=doc["lastCaptureAtUtc"],
|
|
366
|
+
last_detection_epoch=doc["lastDetectionEpoch"],
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def continuing_validity(cert: Certificate, r: Reading, *,
|
|
371
|
+
fallback_rows_captured_at=()) -> "str | None":
|
|
372
|
+
"""None while ``r`` stays inside the certified forward domain, else the
|
|
373
|
+
revocation reason (spec 5.6 item 5).
|
|
374
|
+
|
|
375
|
+
``fallback_rows_captured_at`` are the capture instants of the rows the
|
|
376
|
+
decision's NON-INERT cross-period reads returned (the access registry
|
|
377
|
+
decides which reads are inert); a row from before the witness means the
|
|
378
|
+
read fell back into the uncertain prefix."""
|
|
379
|
+
if r.week_start_date is None or r.week_start_date < cert.witness_week_start_date:
|
|
380
|
+
return "backward-key"
|
|
381
|
+
if r.five_hour_key is not None and r.five_hour_key < cert.witness_five_hour_key:
|
|
382
|
+
return "backward-key"
|
|
383
|
+
if r.detection_epoch < cert.last_detection_epoch:
|
|
384
|
+
return "clock-regression"
|
|
385
|
+
if r.captured_at_utc < cert.last_capture_at_utc:
|
|
386
|
+
return "order-regression"
|
|
387
|
+
for captured in fallback_rows_captured_at:
|
|
388
|
+
if captured is not None and captured < cert.witness_captured_at_utc:
|
|
389
|
+
return "non-inert-fallback"
|
|
390
|
+
return None
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
@dataclasses.dataclass(frozen=True)
|
|
394
|
+
class ControlCertificate:
|
|
395
|
+
"""A bound two-axis authoritative OAuth publication for one account
|
|
396
|
+
(spec 5.6 item 7): the journal observation that witnesses it."""
|
|
397
|
+
|
|
398
|
+
observation_id: str
|
|
399
|
+
captured_at_utc: str
|
|
400
|
+
|
|
401
|
+
def to_document(self) -> dict:
|
|
402
|
+
return {"observationId": self.observation_id,
|
|
403
|
+
"capturedAtUtc": self.captured_at_utc}
|
|
404
|
+
|
|
405
|
+
@staticmethod
|
|
406
|
+
def from_document(doc) -> "ControlCertificate | None":
|
|
407
|
+
if (not isinstance(doc, dict)
|
|
408
|
+
or set(doc) != {"observationId", "capturedAtUtc"}
|
|
409
|
+
or not isinstance(doc["observationId"], str)
|
|
410
|
+
or not isinstance(doc["capturedAtUtc"], str)):
|
|
411
|
+
return None
|
|
412
|
+
return ControlCertificate(doc["observationId"], doc["capturedAtUtc"])
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
@dataclasses.dataclass
|
|
416
|
+
class AccountRecovery:
|
|
417
|
+
"""One account's certificates and the trigger sequence each was granted
|
|
418
|
+
against. A certificate granted against an older sequence is void."""
|
|
419
|
+
|
|
420
|
+
accounting: "Certificate | None" = None
|
|
421
|
+
accounting_trigger_seq: "int | None" = None
|
|
422
|
+
control: "ControlCertificate | None" = None
|
|
423
|
+
control_trigger_seq: "int | None" = None
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
@dataclasses.dataclass(frozen=True)
|
|
427
|
+
class Trigger:
|
|
428
|
+
seq: int
|
|
429
|
+
segment: "str | None"
|
|
430
|
+
offset: "int | None"
|
|
431
|
+
kind: str # "excluded" | "legacy-spool-activity"
|
|
432
|
+
reason: str # the excluded family, or the activity reason
|
|
433
|
+
at: "str | None"
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def spool_marker(trigger: Trigger) -> dict:
|
|
437
|
+
"""The unresolved structural-envelope marker a spool trigger leaves: its
|
|
438
|
+
journal coordinate and `at` (spec revision 3), and its reason — spec
|
|
439
|
+
revision 5 words the marker of an incomplete inventory differently."""
|
|
440
|
+
return {"seq": trigger.seq, "segment": trigger.segment,
|
|
441
|
+
"offset": trigger.offset, "at": trigger.at,
|
|
442
|
+
"reason": trigger.reason}
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
@dataclasses.dataclass(frozen=True)
|
|
446
|
+
class GateDecision:
|
|
447
|
+
"""Whether automatic outcomes may be minted for one reading."""
|
|
448
|
+
|
|
449
|
+
allowed: bool
|
|
450
|
+
reason: str
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
GATE_NO_TRIGGER = GateDecision(True, "no-trigger")
|
|
454
|
+
GATE_CERTIFIED = GateDecision(True, "certified")
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
class QuarantineState:
|
|
458
|
+
"""The deterministic quarantine fold over journal order.
|
|
459
|
+
|
|
460
|
+
Every consumer — live ingest, rebuild, the rederive scratch replay — feeds
|
|
461
|
+
the same events in journal order, so the persisted state is a function of
|
|
462
|
+
the journal alone. Methods mutate in place and record which rows changed
|
|
463
|
+
so a caller persists only those."""
|
|
464
|
+
|
|
465
|
+
def __init__(self, *, latest_trigger_seq: int = 0, envelopes=None,
|
|
466
|
+
accounts=None, non_inert_cross_period: bool = False,
|
|
467
|
+
spool_markers=()):
|
|
468
|
+
self.latest_trigger_seq = int(latest_trigger_seq)
|
|
469
|
+
self.envelopes: dict = dict(envelopes or {})
|
|
470
|
+
self.accounts: dict = dict(accounts or {})
|
|
471
|
+
self.new_triggers: list = []
|
|
472
|
+
self.dirty_envelopes: set = set()
|
|
473
|
+
self.dirty_accounts: set = set()
|
|
474
|
+
#: Every spool trigger's unresolved marker, in journal order. None is
|
|
475
|
+
#: ever removed: a later trigger adds a generation and keeps the
|
|
476
|
+
#: earlier uncertainty (spec revision 3).
|
|
477
|
+
self.spool_markers: list = [dict(m) for m in spool_markers]
|
|
478
|
+
if self.spool_markers:
|
|
479
|
+
# A certificate persisted while a marker exists was issued
|
|
480
|
+
# without the marker check (the rule before spec revision 3), so
|
|
481
|
+
# it is invalidated here, never honored or advanced; a rebuild
|
|
482
|
+
# likewise grants none while the retained triggers hold a marker.
|
|
483
|
+
self.revoke_all_accounting()
|
|
484
|
+
#: Session-local seen observation identities per scope (clause 7).
|
|
485
|
+
#: Not persisted: a replayed identity always fails clause 1 as well,
|
|
486
|
+
#: because its own first copy is inside the envelope.
|
|
487
|
+
self.seen: dict = {}
|
|
488
|
+
#: True when some registered cross-period access has no inert rule;
|
|
489
|
+
#: then a deletion outcome revokes the account at its journal position.
|
|
490
|
+
self.non_inert_cross_period = bool(non_inert_cross_period)
|
|
491
|
+
|
|
492
|
+
# -- queries ---------------------------------------------------------
|
|
493
|
+
def has_trigger(self) -> bool:
|
|
494
|
+
return self.latest_trigger_seq > 0
|
|
495
|
+
|
|
496
|
+
def has_unresolved_spool_marker(self) -> bool:
|
|
497
|
+
return bool(self.spool_markers)
|
|
498
|
+
|
|
499
|
+
def accounting_certified(self, account_key: str) -> bool:
|
|
500
|
+
"""Whether ``account_key`` holds a current accounting certificate.
|
|
501
|
+
Never while a spool marker is unresolved, whatever a persisted row
|
|
502
|
+
says (spec revision 3)."""
|
|
503
|
+
if self.spool_markers:
|
|
504
|
+
return False
|
|
505
|
+
rec = self.accounts.get(account_key)
|
|
506
|
+
return (rec is not None and rec.accounting is not None
|
|
507
|
+
and rec.accounting_trigger_seq == self.latest_trigger_seq)
|
|
508
|
+
|
|
509
|
+
def control_certified(self, account_key: str) -> bool:
|
|
510
|
+
rec = self.accounts.get(account_key)
|
|
511
|
+
return (rec is not None and rec.control is not None
|
|
512
|
+
and rec.control_trigger_seq == self.latest_trigger_seq)
|
|
513
|
+
|
|
514
|
+
def accounting_quarantined(self, account_key: str) -> bool:
|
|
515
|
+
return self.has_trigger() and not self.accounting_certified(account_key)
|
|
516
|
+
|
|
517
|
+
def control_quarantined(self, account_key: str) -> bool:
|
|
518
|
+
return self.has_trigger() and not self.control_certified(account_key)
|
|
519
|
+
|
|
520
|
+
def envelope(self, scope: str) -> Envelope:
|
|
521
|
+
return self.envelopes.get(scope) or Envelope()
|
|
522
|
+
|
|
523
|
+
# -- transitions -----------------------------------------------------
|
|
524
|
+
def _set_envelope(self, scope: str, value: Envelope) -> None:
|
|
525
|
+
if self.envelopes.get(scope) != value:
|
|
526
|
+
self.envelopes[scope] = value
|
|
527
|
+
self.dirty_envelopes.add(scope)
|
|
528
|
+
|
|
529
|
+
def _account(self, account_key: str) -> AccountRecovery:
|
|
530
|
+
rec = self.accounts.get(account_key)
|
|
531
|
+
if rec is None:
|
|
532
|
+
rec = AccountRecovery()
|
|
533
|
+
self.accounts[account_key] = rec
|
|
534
|
+
return rec
|
|
535
|
+
|
|
536
|
+
def on_trigger(self, *, kind: str, reason: str, segment=None, offset=None,
|
|
537
|
+
at=None) -> Trigger:
|
|
538
|
+
"""A trigger quarantines EVERY account and voids every certificate
|
|
539
|
+
(its sequence moves past them). The account a record names never
|
|
540
|
+
narrows it."""
|
|
541
|
+
self.latest_trigger_seq += 1
|
|
542
|
+
trig = Trigger(self.latest_trigger_seq, segment, offset, kind, reason, at)
|
|
543
|
+
self.new_triggers.append(trig)
|
|
544
|
+
if kind == TRIGGER_KIND_LEGACY_SPOOL:
|
|
545
|
+
self.spool_markers.append(spool_marker(trig))
|
|
546
|
+
return trig
|
|
547
|
+
|
|
548
|
+
def absorb_excluded_reading(self, r: Reading) -> None:
|
|
549
|
+
self._set_envelope(
|
|
550
|
+
EXCLUDED_SCOPE, self.envelope(EXCLUDED_SCOPE).absorb_reading(r))
|
|
551
|
+
self.seen.setdefault(EXCLUDED_SCOPE, set()).add(r.observation_id)
|
|
552
|
+
|
|
553
|
+
def absorb_effect(self, scope: str, **effect) -> None:
|
|
554
|
+
self._set_envelope(scope, self.envelope(scope).absorb_effect(**effect))
|
|
555
|
+
|
|
556
|
+
def revoke_accounting(self, account_key: str) -> None:
|
|
557
|
+
"""Drop the accounting certificate. The sequence it was granted
|
|
558
|
+
against is kept, so `doctor` can say since when the account has been
|
|
559
|
+
paused; without a certificate it never certifies anything."""
|
|
560
|
+
rec = self.accounts.get(account_key)
|
|
561
|
+
if rec is not None and rec.accounting is not None:
|
|
562
|
+
rec.accounting = None
|
|
563
|
+
self.dirty_accounts.add(account_key)
|
|
564
|
+
|
|
565
|
+
def revoke_control(self, account_key: str) -> None:
|
|
566
|
+
rec = self.accounts.get(account_key)
|
|
567
|
+
if rec is not None and rec.control is not None:
|
|
568
|
+
rec.control = None
|
|
569
|
+
self.dirty_accounts.add(account_key)
|
|
570
|
+
|
|
571
|
+
def revoke_all_accounting(self) -> None:
|
|
572
|
+
for account_key in sorted(self.accounts):
|
|
573
|
+
self.revoke_accounting(account_key)
|
|
574
|
+
|
|
575
|
+
def on_reading(self, account_key: "str | None", r: Reading, *,
|
|
576
|
+
fallback_rows_captured_at=()) -> GateDecision:
|
|
577
|
+
"""Fold one admitted reading. Certification is evaluated BEFORE the
|
|
578
|
+
reading is absorbed, against the whole prefix; continuing validity is
|
|
579
|
+
checked before any outcome; the reading is then absorbed into its
|
|
580
|
+
account's envelope. Returns the gate for the reading's outcomes."""
|
|
581
|
+
if account_key is None:
|
|
582
|
+
# An admitted historical reading with no account: its bounds
|
|
583
|
+
# apply to every account.
|
|
584
|
+
self._set_envelope(
|
|
585
|
+
EXCLUDED_SCOPE, self.envelope(EXCLUDED_SCOPE).absorb_reading(r))
|
|
586
|
+
self.seen.setdefault(EXCLUDED_SCOPE, set()).add(r.observation_id)
|
|
587
|
+
return GATE_NO_TRIGGER if not self.has_trigger() else GateDecision(
|
|
588
|
+
False, "quarantined")
|
|
589
|
+
gate = GATE_NO_TRIGGER
|
|
590
|
+
if self.has_trigger():
|
|
591
|
+
if self.accounting_certified(account_key):
|
|
592
|
+
rec = self.accounts[account_key]
|
|
593
|
+
reason = continuing_validity(
|
|
594
|
+
rec.accounting, r,
|
|
595
|
+
fallback_rows_captured_at=fallback_rows_captured_at)
|
|
596
|
+
if reason is not None:
|
|
597
|
+
self.revoke_accounting(account_key)
|
|
598
|
+
gate = GateDecision(False, f"revoked:{reason}")
|
|
599
|
+
else:
|
|
600
|
+
advanced = rec.accounting.advance(r)
|
|
601
|
+
if advanced != rec.accounting:
|
|
602
|
+
rec.accounting = advanced
|
|
603
|
+
self.dirty_accounts.add(account_key)
|
|
604
|
+
gate = GATE_CERTIFIED
|
|
605
|
+
elif self.spool_markers:
|
|
606
|
+
# The omitted spool contributor bounds nothing, so no reading
|
|
607
|
+
# certifies against the envelope while a marker is unresolved.
|
|
608
|
+
gate = GateDecision(False, GATE_REASON_SPOOL_MARKER)
|
|
609
|
+
else:
|
|
610
|
+
merged = Envelope.merge(self.envelope(account_key),
|
|
611
|
+
self.envelope(EXCLUDED_SCOPE))
|
|
612
|
+
seen = (self.seen.get(account_key, set())
|
|
613
|
+
| self.seen.get(EXCLUDED_SCOPE, set()))
|
|
614
|
+
decision = certify(r, merged, seen_identities=seen)
|
|
615
|
+
if decision.certified:
|
|
616
|
+
rec = self._account(account_key)
|
|
617
|
+
rec.accounting = Certificate.from_witness(r)
|
|
618
|
+
rec.accounting_trigger_seq = self.latest_trigger_seq
|
|
619
|
+
self.dirty_accounts.add(account_key)
|
|
620
|
+
gate = GateDecision(True, "witness")
|
|
621
|
+
else:
|
|
622
|
+
gate = GateDecision(False, "quarantined")
|
|
623
|
+
self._set_envelope(account_key, self.envelope(account_key).absorb_reading(r))
|
|
624
|
+
self.seen.setdefault(account_key, set()).add(r.observation_id)
|
|
625
|
+
return gate
|
|
626
|
+
|
|
627
|
+
def on_control_witness(self, account_key: str, *, observation_id: str,
|
|
628
|
+
captured_at_utc: str) -> None:
|
|
629
|
+
"""A bound two-axis authoritative OAuth publication for one account,
|
|
630
|
+
positioned after the latest trigger. Its control transition must ALSO
|
|
631
|
+
record the commit before drop consensus resumes (checked by the
|
|
632
|
+
statusline reducer)."""
|
|
633
|
+
if not self.has_trigger():
|
|
634
|
+
return
|
|
635
|
+
rec = self._account(account_key)
|
|
636
|
+
cert = ControlCertificate(observation_id, captured_at_utc)
|
|
637
|
+
if rec.control != cert or rec.control_trigger_seq != self.latest_trigger_seq:
|
|
638
|
+
rec.control = cert
|
|
639
|
+
rec.control_trigger_seq = self.latest_trigger_seq
|
|
640
|
+
self.dirty_accounts.add(account_key)
|
|
641
|
+
|
|
642
|
+
def on_deletion_outcome(self, account_key: "str | None") -> None:
|
|
643
|
+
"""A journaled automatic outcome that deleted snapshot rows. While
|
|
644
|
+
every registered cross-period fallback read is inert this revokes
|
|
645
|
+
nothing (spec 5.6 item 5: the certificate survives where the fallback
|
|
646
|
+
read is registered inert); otherwise the affected account — every
|
|
647
|
+
account when the event names none — is revoked at the event's journal
|
|
648
|
+
position."""
|
|
649
|
+
if not self.non_inert_cross_period:
|
|
650
|
+
return
|
|
651
|
+
if account_key is None:
|
|
652
|
+
self.revoke_all_accounting()
|
|
653
|
+
else:
|
|
654
|
+
self.revoke_accounting(account_key)
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def certificate_row(rec: AccountRecovery) -> tuple:
|
|
658
|
+
"""The persisted columns of one account's recovery state."""
|
|
659
|
+
return (
|
|
660
|
+
None if rec.accounting is None else json.dumps(
|
|
661
|
+
rec.accounting.to_document(), sort_keys=True, separators=(",", ":")),
|
|
662
|
+
rec.accounting_trigger_seq,
|
|
663
|
+
None if rec.control is None else json.dumps(
|
|
664
|
+
rec.control.to_document(), sort_keys=True, separators=(",", ":")),
|
|
665
|
+
rec.control_trigger_seq,
|
|
666
|
+
)
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def account_from_row(accounting_json, accounting_seq, control_json,
|
|
670
|
+
control_seq) -> AccountRecovery:
|
|
671
|
+
"""The recovery state a persisted row describes. Missing or unverifiable
|
|
672
|
+
certificate evidence reads as NO certificate (quarantined)."""
|
|
673
|
+
def _load(text, loader):
|
|
674
|
+
if not text:
|
|
675
|
+
return None
|
|
676
|
+
try:
|
|
677
|
+
return loader(json.loads(text))
|
|
678
|
+
except (TypeError, ValueError):
|
|
679
|
+
return None
|
|
680
|
+
|
|
681
|
+
accounting = _load(accounting_json, Certificate.from_document)
|
|
682
|
+
control = _load(control_json, ControlCertificate.from_document)
|
|
683
|
+
return AccountRecovery(
|
|
684
|
+
accounting=accounting,
|
|
685
|
+
accounting_trigger_seq=(
|
|
686
|
+
accounting_seq if type(accounting_seq) is int else None),
|
|
687
|
+
control=control,
|
|
688
|
+
control_trigger_seq=(
|
|
689
|
+
control_seq if type(control_seq) is int else None),
|
|
690
|
+
)
|