cctally 1.115.0 → 1.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/bin/_cctally_cache.py +85 -4
- package/bin/_cctally_completion.py +80 -12
- package/bin/_cctally_core.py +106 -9
- package/bin/_cctally_dashboard.py +6 -9
- package/bin/_cctally_dashboard_envelope.py +26 -32
- package/bin/_cctally_db.py +85 -4
- package/bin/_cctally_doctor.py +38 -5
- package/bin/_cctally_five_hour.py +83 -1017
- package/bin/_cctally_journal.py +1307 -180
- package/bin/_cctally_journal_repair.py +9 -7
- package/bin/_cctally_oauth_samples.py +280 -0
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_quarantine.py +14 -4
- package/bin/_cctally_quota.py +70 -8
- package/bin/_cctally_quota_model.py +35 -4
- package/bin/_cctally_record.py +414 -123
- package/bin/_cctally_rederive.py +632 -235
- package/bin/_cctally_refresh.py +29 -0
- package/bin/_cctally_statusline.py +984 -47
- package/bin/_cctally_store.py +972 -58
- package/bin/_cctally_tui.py +62 -7
- package/bin/_cctally_update.py +25 -0
- package/bin/_lib_account_protocol.py +201 -18
- package/bin/_lib_clone_copy.py +46 -11
- package/bin/_lib_dashboard_sources.py +57 -15
- package/bin/_lib_detector_access_registry.py +273 -52
- package/bin/_lib_doctor.py +302 -1
- package/bin/_lib_journal.py +145 -25
- package/bin/_lib_journal_router.py +166 -48
- package/bin/_lib_oauth_samples.py +238 -0
- package/bin/_lib_pricing_check.py +0 -12
- package/bin/_lib_provider_activity.py +770 -0
- package/bin/_lib_quarantine.py +3 -2
- package/bin/_lib_quota_model.py +96 -6
- package/bin/_lib_rebuild_progress.py +146 -0
- package/bin/_lib_record.py +22 -0
- package/bin/_lib_rederive.py +299 -3
- package/bin/_lib_snapshot_cache.py +43 -1
- package/bin/_lib_stale_proof.py +708 -0
- package/bin/_lib_statusline_candidates.py +65 -5
- package/bin/_lib_worker_launches.py +315 -0
- package/bin/cctally +6 -11
- package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
- package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
- package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
- package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
- package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
- package/dashboard/static/dashboard.html +1 -1
- package/package.json +7 -2
- package/bin/_lib_merge_gate.py +0 -452
- package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
|
@@ -321,7 +321,7 @@ def cmd_statusline(args: argparse.Namespace) -> int:
|
|
|
321
321
|
# =========================================================================
|
|
322
322
|
#
|
|
323
323
|
# Every current statusline coordination file lives under ONE sibling root,
|
|
324
|
-
# `APP_DIR/statusline-account-bound-
|
|
324
|
+
# `APP_DIR/statusline-account-bound-v2/`, partitioned by the exact account:
|
|
325
325
|
#
|
|
326
326
|
# selected.lock the selected-state / persist flock
|
|
327
327
|
# transport.last render-path transport marker
|
|
@@ -646,15 +646,39 @@ def _candidate_from_input(parsed, *, received_at: int) -> "_candidates.Candidate
|
|
|
646
646
|
binding = _capture_binding()
|
|
647
647
|
if binding is None:
|
|
648
648
|
return None
|
|
649
|
+
token = _candidate_identity_token(parsed)
|
|
649
650
|
return _candidates.Candidate(
|
|
650
|
-
token=
|
|
651
|
+
token=token,
|
|
651
652
|
received_at=received_at,
|
|
652
653
|
five_hour=five,
|
|
653
654
|
seven_day=seven,
|
|
654
655
|
binding=binding,
|
|
656
|
+
activity=_capture_activity(parsed, token),
|
|
655
657
|
)
|
|
656
658
|
|
|
657
659
|
|
|
660
|
+
def _activity_cache_path(token: str) -> pathlib.Path:
|
|
661
|
+
"""The contributor's provider-activity scan cache in the v2 spool (#797
|
|
662
|
+
spec 4.1, D10a): one accelerator document per contributor, never read by
|
|
663
|
+
acknowledgement, detached reduction or frozen-plan recovery."""
|
|
664
|
+
if _candidates.CANDIDATE_TOKEN_RE.fullmatch(token) is None:
|
|
665
|
+
raise ValueError("invalid candidate token")
|
|
666
|
+
return _bound_root() / "activity-cache" / f"{token}.json"
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def _capture_activity(parsed, token: str):
|
|
670
|
+
"""The arrival's frozen provider-activity witness (#797 spec 4.1, 4.2),
|
|
671
|
+
captured once here: in the render process, after Claude Code composed the
|
|
672
|
+
input and before the intake commit, outside the intake critical section
|
|
673
|
+
(its leaf lock forbids further locks beneath it). The transcript path is
|
|
674
|
+
read here and goes nowhere else."""
|
|
675
|
+
import _lib_provider_activity
|
|
676
|
+
|
|
677
|
+
path = parsed.transcript_path if isinstance(parsed.transcript_path, str) else None
|
|
678
|
+
return _lib_provider_activity.capture_activity(
|
|
679
|
+
path, cache_path=_activity_cache_path(token))
|
|
680
|
+
|
|
681
|
+
|
|
658
682
|
#: Completion markers of the account-bound root's persistence barriers
|
|
659
683
|
#: (#825/#832 review finding CD4-D01, cycle 4 verification): one empty file
|
|
660
684
|
#: per directory or intake segment whose entry has been made durable, kept in
|
|
@@ -801,13 +825,20 @@ def _atomic_write_json(path: pathlib.Path, document: dict, *,
|
|
|
801
825
|
def _candidate_document(candidate: "_candidates.Candidate") -> dict:
|
|
802
826
|
"""The current candidate document: the protocol generation and the
|
|
803
827
|
immutable binding travel with the raw per-axis readings (spec 4.2)."""
|
|
828
|
+
import _lib_provider_activity
|
|
829
|
+
|
|
804
830
|
if candidate.binding is None:
|
|
805
831
|
raise ValueError("a current candidate requires an account binding")
|
|
832
|
+
if not isinstance(candidate.activity, _lib_provider_activity.ActivityWitness):
|
|
833
|
+
raise ValueError("a current candidate requires its activity witness")
|
|
806
834
|
document = {
|
|
807
835
|
"schemaVersion": _candidates.SCHEMA_VERSION,
|
|
808
836
|
"protocolGeneration": _candidates.PROTOCOL_GENERATION,
|
|
809
837
|
"binding": candidate.binding.document(),
|
|
810
838
|
"receivedAt": candidate.received_at,
|
|
839
|
+
# #797 spec 4.2: the frozen witness travels inside the document, so
|
|
840
|
+
# the intake identity hashes it and every reader revalidates it.
|
|
841
|
+
"activity": _lib_provider_activity.witness_to_json(candidate.activity),
|
|
811
842
|
}
|
|
812
843
|
if candidate.five_hour is not None:
|
|
813
844
|
document["fiveHour"] = {
|
|
@@ -855,6 +886,9 @@ def _parse_candidate(raw: bytes, *, account_key: str, token: str, now_epoch: int
|
|
|
855
886
|
binding = _lib_account_binding.load(doc.pop("binding", None))
|
|
856
887
|
if binding.account_key != account_key:
|
|
857
888
|
raise _candidates.StateValidationError("cross-account candidate")
|
|
889
|
+
if "activity" not in doc:
|
|
890
|
+
raise _candidates.StateValidationError("a current candidate requires its activity witness")
|
|
891
|
+
activity = _candidates.validate_activity(doc.pop("activity"))
|
|
858
892
|
candidate = _candidates.validate_candidate_document(
|
|
859
893
|
doc,
|
|
860
894
|
now_epoch=now_epoch,
|
|
@@ -862,7 +896,7 @@ def _parse_candidate(raw: bytes, *, account_key: str, token: str, now_epoch: int
|
|
|
862
896
|
axis, epoch, now_epoch),
|
|
863
897
|
token=token,
|
|
864
898
|
)
|
|
865
|
-
return dataclasses.replace(candidate, binding=binding)
|
|
899
|
+
return dataclasses.replace(candidate, binding=binding, activity=activity)
|
|
866
900
|
|
|
867
901
|
|
|
868
902
|
def _read_account_candidates(account_key: str, *, now_epoch: int, prune: bool):
|
|
@@ -987,7 +1021,8 @@ def _five_hour_reset_admitted(raw: int, instant: int, now_epoch: int) -> bool:
|
|
|
987
1021
|
<= max(instant, now_epoch) + _FIVE_HOUR_FUTURE_BAND_SECONDS)
|
|
988
1022
|
|
|
989
1023
|
|
|
990
|
-
def _read_db_projection_at(*, account_key: str, at: "int | None"
|
|
1024
|
+
def _read_db_projection_at(*, account_key: str, at: "int | None",
|
|
1025
|
+
stale: "list | None" = None):
|
|
991
1026
|
"""`_read_db_projection_once` plus the inclusive range of evaluation
|
|
992
1027
|
instants for which the same read yields the same projection: only the
|
|
993
1028
|
five-hour expiry grace depends on the instant, so the projection is
|
|
@@ -1000,12 +1035,13 @@ def _read_db_projection_at(*, account_key: str, at: "int | None"):
|
|
|
1000
1035
|
admission moves with the wall clock, when it is the capped range."""
|
|
1001
1036
|
validity: list = []
|
|
1002
1037
|
projection = _read_db_projection_once(
|
|
1003
|
-
account_key=account_key, at=at, validity=validity)
|
|
1038
|
+
account_key=account_key, at=at, validity=validity, stale=stale)
|
|
1004
1039
|
return projection, validity[0], validity[1]
|
|
1005
1040
|
|
|
1006
1041
|
|
|
1007
1042
|
def _read_db_projection_once(*, account_key: str, at: "int | None" = None,
|
|
1008
1043
|
validity: "list | None" = None,
|
|
1044
|
+
stale: "list | None" = None,
|
|
1009
1045
|
) -> "_candidates.DbProjection":
|
|
1010
1046
|
"""Project the selected weekly and five-hour values of EXACTLY one account
|
|
1011
1047
|
(#825/#832 spec 4.5), as seen by an evaluation at ``at`` (the wall clock
|
|
@@ -1017,7 +1053,12 @@ def _read_db_projection_once(*, account_key: str, at: "int | None" = None,
|
|
|
1017
1053
|
SELECTs, the reset-aware weekly floor and the five-hour reset generation are
|
|
1018
1054
|
scoped to ``account_key``, so another account's rows or reset events can
|
|
1019
1055
|
neither become this account's projection nor filter its rows. The returned
|
|
1020
|
-
projection names the account it was read for.
|
|
1056
|
+
projection names the account it was read for.
|
|
1057
|
+
|
|
1058
|
+
``stale``, when given, receives the account's `_StaleInputs` (#797 spec
|
|
1059
|
+
4.4) read in the SAME snapshot: the weekly window's provider samples,
|
|
1060
|
+
credits, accounted value and index identity the refusal facts and the
|
|
1061
|
+
receipts are resolved from."""
|
|
1021
1062
|
if not isinstance(account_key, str) or not account_key:
|
|
1022
1063
|
raise ValueError("the statusline projection requires an exact account")
|
|
1023
1064
|
import _cctally_journal
|
|
@@ -1074,6 +1115,7 @@ def _read_db_projection_once(*, account_key: str, at: "int | None" = None,
|
|
|
1074
1115
|
|
|
1075
1116
|
weekly_projection = None
|
|
1076
1117
|
weekly_retired = None
|
|
1118
|
+
weekly_selected = weekly_floor = reference = None
|
|
1077
1119
|
if weekly_groups:
|
|
1078
1120
|
canonical = max(weekly_groups)
|
|
1079
1121
|
rows = weekly_groups[canonical]
|
|
@@ -1106,6 +1148,7 @@ def _read_db_projection_once(*, account_key: str, at: "int | None" = None,
|
|
|
1106
1148
|
eligible,
|
|
1107
1149
|
key=lambda row: (_epoch_from_iso(row["captured_at_utc"]), int(row["id"])),
|
|
1108
1150
|
)
|
|
1151
|
+
weekly_selected = weekly
|
|
1109
1152
|
raw = _epoch_from_iso(weekly["week_end_at"])
|
|
1110
1153
|
weekly_projection = _candidates.AxisProjection(
|
|
1111
1154
|
float(weekly["weekly_percent"]), raw, canonical,
|
|
@@ -1215,6 +1258,12 @@ def _read_db_projection_once(*, account_key: str, at: "int | None" = None,
|
|
|
1215
1258
|
five_admissible_at=five_admissible_at,
|
|
1216
1259
|
weekly_retired=weekly_retired, five_retired=five_retired,
|
|
1217
1260
|
weekly_bound=weekly_bound, instant=instant)
|
|
1261
|
+
if stale is not None:
|
|
1262
|
+
stale.append(_read_stale_inputs(
|
|
1263
|
+
conn, account_key, groups=weekly_groups, selected=weekly_selected,
|
|
1264
|
+
projection=weekly_projection, floor=weekly_floor,
|
|
1265
|
+
reference=reference,
|
|
1266
|
+
five_window=max(five_groups) if five_groups else None))
|
|
1218
1267
|
finally:
|
|
1219
1268
|
conn.close()
|
|
1220
1269
|
if validity is not None:
|
|
@@ -1233,6 +1282,302 @@ def _read_db_projection_once(*, account_key: str, at: "int | None" = None,
|
|
|
1233
1282
|
evidence=evidence)
|
|
1234
1283
|
|
|
1235
1284
|
|
|
1285
|
+
@dataclasses.dataclass(frozen=True)
|
|
1286
|
+
class _StaleInputs:
|
|
1287
|
+
"""What #797 spec 4.4 resolves for one account from a classification's
|
|
1288
|
+
snapshot: the latest weekly window, its accounted value
|
|
1289
|
+
(`_lib_stale_proof.Accounted`, None without a window), its credits, its
|
|
1290
|
+
admitted provider samples requested after ``samples_after`` (None: not
|
|
1291
|
+
read, because no member or receipt needed them), whether the sample
|
|
1292
|
+
projection covers the account, the stats index identity receipts are
|
|
1293
|
+
bound to, and the live support identities.
|
|
1294
|
+
|
|
1295
|
+
The five-hour fields (Task 5) are the latest five-hour window's canonical
|
|
1296
|
+
key, its accounted value (the `statusline` source's detector baseline
|
|
1297
|
+
under the current credit generation, D11; None without one), its
|
|
1298
|
+
credits resolved through their causal facts, and its admitted provider
|
|
1299
|
+
samples requested after ``samples_after``."""
|
|
1300
|
+
|
|
1301
|
+
window: "int | None"
|
|
1302
|
+
accounted: object
|
|
1303
|
+
credits: tuple
|
|
1304
|
+
samples: tuple
|
|
1305
|
+
samples_after: "str | None"
|
|
1306
|
+
samples_covered: bool
|
|
1307
|
+
index_identity: str
|
|
1308
|
+
supports: frozenset
|
|
1309
|
+
five_window: "int | None" = None
|
|
1310
|
+
five_accounted: object = None
|
|
1311
|
+
five_credits: tuple = ()
|
|
1312
|
+
five_samples: tuple = ()
|
|
1313
|
+
|
|
1314
|
+
|
|
1315
|
+
#: #797 spec 4.4: ``(samples_after, receipts_held)`` for the partition the
|
|
1316
|
+
#: detached reducer is working on — the earliest provider-activity instant
|
|
1317
|
+
#: any of its members or receipts can need a sample after, and whether it
|
|
1318
|
+
#: holds receipts (a refusal is then possible, so an uncovered sample
|
|
1319
|
+
#: projection makes classification unavailable, never "no proof needed").
|
|
1320
|
+
_STALE_SCOPE = contextvars.ContextVar("statusline_stale_scope", default=(None, False))
|
|
1321
|
+
|
|
1322
|
+
|
|
1323
|
+
def _stale_index_identity(conn) -> str:
|
|
1324
|
+
"""The stats index identity receipts are bound to: its epoch and its
|
|
1325
|
+
publication stamp, which every rebuild publication rewrites (an index
|
|
1326
|
+
never rebuilt has none). A rebuild, re-upgrade or epoch change gives a
|
|
1327
|
+
different identity and so invalidates every receipt."""
|
|
1328
|
+
epoch = int(conn.execute("PRAGMA user_version").fetchone()[0])
|
|
1329
|
+
try:
|
|
1330
|
+
stamp = sorted([str(v) for v in row] for row in conn.execute(
|
|
1331
|
+
"SELECT record_path, started_at_utc, stamped_at_utc "
|
|
1332
|
+
"FROM stats_publication_stamp"))
|
|
1333
|
+
except sqlite3.Error:
|
|
1334
|
+
stamp = None
|
|
1335
|
+
return hashlib.sha256(json.dumps(
|
|
1336
|
+
[epoch, stamp], separators=(",", ":")).encode("utf-8")).hexdigest()
|
|
1337
|
+
|
|
1338
|
+
|
|
1339
|
+
def _weekly_window_of(value: "str | None") -> "int | None":
|
|
1340
|
+
try:
|
|
1341
|
+
raw = _epoch_from_iso(value)
|
|
1342
|
+
except (TypeError, ValueError):
|
|
1343
|
+
return None
|
|
1344
|
+
return int(_cctally_core._normalize_week_boundary_dt(
|
|
1345
|
+
dt.datetime.fromtimestamp(raw, tz=dt.timezone.utc)).timestamp())
|
|
1346
|
+
|
|
1347
|
+
|
|
1348
|
+
def _credit_cause(conn, account_key: str, origin: "str | None") -> tuple:
|
|
1349
|
+
"""A weekly credit's causal low (spec 4.4): its originating (dropping or
|
|
1350
|
+
first-zero) observation. A provider sample carries its request instant;
|
|
1351
|
+
an api reading without one is unordered; a status-line reading survives
|
|
1352
|
+
receipts (R15); a missing origin is legacy; anything else unknown."""
|
|
1353
|
+
if origin is None:
|
|
1354
|
+
return "legacy", None
|
|
1355
|
+
try:
|
|
1356
|
+
row = conn.execute(
|
|
1357
|
+
"SELECT requested_at_utc FROM claude_oauth_samples "
|
|
1358
|
+
"WHERE account_key = ? AND observation_id = ?",
|
|
1359
|
+
(account_key, origin)).fetchone()
|
|
1360
|
+
except sqlite3.Error:
|
|
1361
|
+
row = None
|
|
1362
|
+
if row is not None:
|
|
1363
|
+
return "provider", str(row[0])
|
|
1364
|
+
# A journalled snapshot's identity is ``"sa:" + <raw observation id>``.
|
|
1365
|
+
row = conn.execute(
|
|
1366
|
+
"SELECT source FROM weekly_usage_snapshots "
|
|
1367
|
+
"WHERE account_key = ? AND journal_id = ?",
|
|
1368
|
+
(account_key, "sa:" + origin)).fetchone()
|
|
1369
|
+
if row is None:
|
|
1370
|
+
return "unknown", None
|
|
1371
|
+
if row[0] == "statusline":
|
|
1372
|
+
return "statusline", None
|
|
1373
|
+
if row[0] == "api":
|
|
1374
|
+
return "provider", None
|
|
1375
|
+
return "unknown", None
|
|
1376
|
+
|
|
1377
|
+
|
|
1378
|
+
def _window_credits(conn, account_key: str, window: int, reference) -> tuple:
|
|
1379
|
+
"""Every credit of the account's weekly ``window``, resolved through its
|
|
1380
|
+
causal observation: the window's in-place reset events and the manual
|
|
1381
|
+
credit floors of its week."""
|
|
1382
|
+
import _lib_journal
|
|
1383
|
+
import _lib_stale_proof
|
|
1384
|
+
|
|
1385
|
+
credits = []
|
|
1386
|
+
for row in conn.execute(
|
|
1387
|
+
"SELECT detected_at_utc, old_week_end_at, new_week_end_at, "
|
|
1388
|
+
"effective_reset_at_utc, observed_pre_credit_pct, origin_observation_id "
|
|
1389
|
+
"FROM week_reset_events WHERE account_key = ?", (account_key,)):
|
|
1390
|
+
if _weekly_window_of(row[2]) != window:
|
|
1391
|
+
continue
|
|
1392
|
+
kind, requested = _credit_cause(conn, account_key, row[5])
|
|
1393
|
+
credits.append(_lib_stale_proof.CreditTransition(
|
|
1394
|
+
axis="sevenDay",
|
|
1395
|
+
event_id=_lib_journal.evt_id("wr", *_lib_journal.week_reset_identity_parts(
|
|
1396
|
+
account_key, row[1], row[2], row[5])),
|
|
1397
|
+
event_revision=hashlib.sha256(json.dumps(
|
|
1398
|
+
[None if v is None else str(v) for v in row],
|
|
1399
|
+
separators=(",", ":")).encode("utf-8")).hexdigest(),
|
|
1400
|
+
causal_kind=kind, requested_at_utc=requested))
|
|
1401
|
+
if reference is not None:
|
|
1402
|
+
week = str(reference["week_start_date"])
|
|
1403
|
+
for row in conn.execute(
|
|
1404
|
+
"SELECT effective_at_utc, observed_pre_credit_pct, applied_at_utc "
|
|
1405
|
+
"FROM weekly_credit_floors WHERE account_key = ? AND week_start_date = ?",
|
|
1406
|
+
(account_key, week)):
|
|
1407
|
+
credits.append(_lib_stale_proof.CreditTransition(
|
|
1408
|
+
axis="sevenDay", event_id=f"wcf:{account_key}:{week}:{row[0]}",
|
|
1409
|
+
event_revision=hashlib.sha256(json.dumps(
|
|
1410
|
+
[str(v) for v in row], separators=(",", ":")).encode(
|
|
1411
|
+
"utf-8")).hexdigest(),
|
|
1412
|
+
causal_kind="manual", requested_at_utc=None))
|
|
1413
|
+
return tuple(sorted(credits, key=lambda c: c.event_id))
|
|
1414
|
+
|
|
1415
|
+
|
|
1416
|
+
def _five_hour_accounted(conn, account_key: str, window: int):
|
|
1417
|
+
"""The five-hour accounted value of the window (#797 spec 4.4 (i), D11):
|
|
1418
|
+
the `statusline` source's detector baseline, only when it was
|
|
1419
|
+
established under the window's current five-hour credit generation. A
|
|
1420
|
+
baseline of an older generation (a pre-credit high), one the historical
|
|
1421
|
+
detector wrote, or a missing state earns nothing."""
|
|
1422
|
+
import _cctally_journal
|
|
1423
|
+
import _cctally_record
|
|
1424
|
+
import _lib_stale_proof
|
|
1425
|
+
|
|
1426
|
+
generation = _cctally_record.five_hour_credit_generation(
|
|
1427
|
+
conn, window, account_key=account_key)
|
|
1428
|
+
try:
|
|
1429
|
+
row = conn.execute(
|
|
1430
|
+
"SELECT baseline_pct, baseline_observation_id, baseline_generation "
|
|
1431
|
+
"FROM five_hour_credit_confirmation_state "
|
|
1432
|
+
"WHERE account_key = ? AND five_hour_window_key = ? AND source = ?",
|
|
1433
|
+
(account_key, int(window), "statusline")).fetchone()
|
|
1434
|
+
except sqlite3.Error:
|
|
1435
|
+
row = None
|
|
1436
|
+
current = row is not None and row[2] is not None and row[2] == generation
|
|
1437
|
+
return _lib_stale_proof.Accounted(
|
|
1438
|
+
account_key=account_key, axis="fiveHour", window=int(window),
|
|
1439
|
+
percent=float(row[0]) if current else None,
|
|
1440
|
+
support_id=row[1] if current else None,
|
|
1441
|
+
support_hash=hashlib.sha256(json.dumps(
|
|
1442
|
+
[float(row[0]), row[1], row[2]], separators=(",", ":")).encode(
|
|
1443
|
+
"utf-8")).hexdigest() if current else None,
|
|
1444
|
+
floor_epoch=None, credits=(), coverage=None,
|
|
1445
|
+
detector_policy=_cctally_journal.DETECTOR_POLICY)
|
|
1446
|
+
|
|
1447
|
+
|
|
1448
|
+
def _five_hour_sample_facts(conn, account_key: str, observation_id) -> tuple:
|
|
1449
|
+
"""``(q, raw five-hour percent)`` of an admitted provider sample, or
|
|
1450
|
+
``(None, None)`` when the observation is not one."""
|
|
1451
|
+
import _cctally_oauth_samples
|
|
1452
|
+
|
|
1453
|
+
sample = _cctally_oauth_samples.sample_for_observation(
|
|
1454
|
+
conn, account_key=account_key, observation_id=observation_id)
|
|
1455
|
+
if sample is None:
|
|
1456
|
+
return None, None
|
|
1457
|
+
return (sample.requested_at_utc,
|
|
1458
|
+
None if sample.five_hour_percent is None else float(sample.five_hour_percent))
|
|
1459
|
+
|
|
1460
|
+
|
|
1461
|
+
def _five_hour_window_credits(conn, account_key: str, window: int) -> tuple:
|
|
1462
|
+
"""Every five-hour credit of the window, resolved through its retained
|
|
1463
|
+
causal facts (#797 spec 4.4, 4.9): a credit the `statusline` source
|
|
1464
|
+
confirmed is a status-line credit; one the `api` source confirmed carries
|
|
1465
|
+
its baseline observation's request instant and RAW five-hour value and
|
|
1466
|
+
its ARMING observation's request instant, each read from the admitted
|
|
1467
|
+
provider sample of that exact observation (never a stored, MAX-clamped
|
|
1468
|
+
value, and never the later confirming observation). A credit without
|
|
1469
|
+
facts (a rebuild dropped them, or another source) is unknown."""
|
|
1470
|
+
import _cctally_record
|
|
1471
|
+
import _lib_journal
|
|
1472
|
+
import _lib_stale_proof
|
|
1473
|
+
|
|
1474
|
+
try:
|
|
1475
|
+
facts = {}
|
|
1476
|
+
for row in conn.execute(
|
|
1477
|
+
"SELECT source, pending_observation_id, baseline_observation_id, "
|
|
1478
|
+
" pending_generation "
|
|
1479
|
+
"FROM five_hour_credit_confirmation_state "
|
|
1480
|
+
"WHERE account_key = ? AND five_hour_window_key = ? "
|
|
1481
|
+
" AND source LIKE '#credit|%'", (account_key, int(window))):
|
|
1482
|
+
facts[row[3]] = (str(row[0]).split("|", 2)[-1], row[1], row[2])
|
|
1483
|
+
except sqlite3.Error:
|
|
1484
|
+
facts = {}
|
|
1485
|
+
credits = []
|
|
1486
|
+
for row in conn.execute(
|
|
1487
|
+
"SELECT detected_at_utc, prior_percent, post_percent, effective_reset_at_utc "
|
|
1488
|
+
"FROM five_hour_reset_events WHERE account_key = ? AND five_hour_window_key = ?",
|
|
1489
|
+
(account_key, int(window))):
|
|
1490
|
+
event_id = _lib_journal.evt_id("fhc", account_key, int(window), str(row[3]))
|
|
1491
|
+
revision = _cctally_record._five_hour_credit_revision(row)
|
|
1492
|
+
fact = facts.get(f"{event_id}@{revision}")
|
|
1493
|
+
kind, arming, baseline, baseline_percent = "unknown", None, None, None
|
|
1494
|
+
if fact is not None and fact[0] == "statusline":
|
|
1495
|
+
kind = "statusline"
|
|
1496
|
+
elif fact is not None and fact[0] == "api":
|
|
1497
|
+
kind = "provider"
|
|
1498
|
+
arming, _ = _five_hour_sample_facts(conn, account_key, fact[1])
|
|
1499
|
+
baseline, baseline_percent = _five_hour_sample_facts(
|
|
1500
|
+
conn, account_key, fact[2])
|
|
1501
|
+
credits.append(_lib_stale_proof.CreditTransition(
|
|
1502
|
+
axis="fiveHour", event_id=event_id, event_revision=revision,
|
|
1503
|
+
causal_kind=kind, requested_at_utc=arming,
|
|
1504
|
+
baseline_requested_at_utc=baseline, baseline_percent=baseline_percent,
|
|
1505
|
+
arming_requested_at_utc=arming))
|
|
1506
|
+
return tuple(sorted(credits, key=lambda c: c.event_id))
|
|
1507
|
+
|
|
1508
|
+
|
|
1509
|
+
def _read_stale_inputs(conn, account_key: str, *, groups, selected, projection,
|
|
1510
|
+
floor, reference, five_window=None) -> _StaleInputs:
|
|
1511
|
+
"""The account's `_StaleInputs` on the caller's snapshot (#797 spec 4.4).
|
|
1512
|
+
Provider samples are read only when the partition's members or receipts
|
|
1513
|
+
need them (`_STALE_SCOPE`); an uncovered sample projection with receipts
|
|
1514
|
+
held raises `ProjectionUnavailable`, so the classification is
|
|
1515
|
+
unavailable rather than silently "no proof"."""
|
|
1516
|
+
import _cctally_oauth_samples
|
|
1517
|
+
import _lib_stale_proof
|
|
1518
|
+
|
|
1519
|
+
identity = _stale_index_identity(conn)
|
|
1520
|
+
if not groups and five_window is None:
|
|
1521
|
+
return _StaleInputs(None, None, (), (), None, True, identity, frozenset())
|
|
1522
|
+
after, receipts_held = _STALE_SCOPE.get()
|
|
1523
|
+
covered = _cctally_oauth_samples.projection_covered(conn, account_key=account_key)
|
|
1524
|
+
if not covered and receipts_held:
|
|
1525
|
+
raise _cctally_oauth_samples.ProjectionUnavailable(
|
|
1526
|
+
"the provider sample projection does not cover this account")
|
|
1527
|
+
five_accounted, five_credits, five_samples = _five_hour_stale_inputs(
|
|
1528
|
+
conn, account_key, five_window, after=after if covered else None)
|
|
1529
|
+
if not groups:
|
|
1530
|
+
# Review F-I5-1: the five-hour refusal inputs never depend on a
|
|
1531
|
+
# weekly reading. The weekly fields keep the no-window shape.
|
|
1532
|
+
return _StaleInputs(None, None, (), (), None, True, identity, frozenset(),
|
|
1533
|
+
five_window, five_accounted, five_credits, five_samples)
|
|
1534
|
+
window = max(groups)
|
|
1535
|
+
supports = {f"row:{int(row['id'])}" for row in groups[window]}
|
|
1536
|
+
samples = ()
|
|
1537
|
+
if after is not None and covered:
|
|
1538
|
+
samples = _cctally_oauth_samples.read_samples(
|
|
1539
|
+
conn, account_key=account_key, axis="weekly", window=window,
|
|
1540
|
+
requested_after_utc=after)
|
|
1541
|
+
supports |= {sample.observation_id for sample in samples}
|
|
1542
|
+
support_id = support_hash = None
|
|
1543
|
+
if selected is not None:
|
|
1544
|
+
support_id = f"row:{int(selected['id'])}"
|
|
1545
|
+
support_hash = hashlib.sha256(json.dumps(
|
|
1546
|
+
[int(selected["id"]), float(selected["weekly_percent"]),
|
|
1547
|
+
str(selected["captured_at_utc"]), str(selected["source"])],
|
|
1548
|
+
separators=(",", ":")).encode("utf-8")).hexdigest()
|
|
1549
|
+
accounted = _lib_stale_proof.Accounted(
|
|
1550
|
+
account_key=account_key, axis="sevenDay", window=window,
|
|
1551
|
+
percent=None if projection is None else float(projection.percent),
|
|
1552
|
+
support_id=support_id, support_hash=support_hash,
|
|
1553
|
+
floor_epoch=None if floor is None else floor.floor_epoch,
|
|
1554
|
+
credits=_window_credits(conn, account_key, window, reference),
|
|
1555
|
+
coverage=None)
|
|
1556
|
+
return _StaleInputs(window, accounted, accounted.credits, tuple(samples),
|
|
1557
|
+
after if covered else None, covered, identity,
|
|
1558
|
+
frozenset(supports), five_window, five_accounted,
|
|
1559
|
+
five_credits, five_samples)
|
|
1560
|
+
|
|
1561
|
+
|
|
1562
|
+
def _five_hour_stale_inputs(conn, account_key: str, five_window, *, after) -> tuple:
|
|
1563
|
+
"""``(accounted, credits, samples)`` of the five-hour window
|
|
1564
|
+
``five_window`` (``(None, (), ())`` without one), its admitted provider
|
|
1565
|
+
samples read only when ``after`` is given (a covered projection)."""
|
|
1566
|
+
import _cctally_oauth_samples
|
|
1567
|
+
|
|
1568
|
+
if five_window is None:
|
|
1569
|
+
return None, (), ()
|
|
1570
|
+
credits = _five_hour_window_credits(conn, account_key, five_window)
|
|
1571
|
+
accounted = dataclasses.replace(
|
|
1572
|
+
_five_hour_accounted(conn, account_key, five_window), credits=credits)
|
|
1573
|
+
samples = ()
|
|
1574
|
+
if after is not None:
|
|
1575
|
+
samples = _cctally_oauth_samples.read_samples(
|
|
1576
|
+
conn, account_key=account_key, axis="five_hour",
|
|
1577
|
+
window=int(five_window), requested_after_utc=after)
|
|
1578
|
+
return accounted, credits, tuple(samples)
|
|
1579
|
+
|
|
1580
|
+
|
|
1236
1581
|
def _account_rows(conn, table: str, account_key: str, extra: str = "") -> int:
|
|
1237
1582
|
"""How many rows of ``table`` belong to ``account_key`` (``extra`` a
|
|
1238
1583
|
further literal predicate), in the caller's snapshot."""
|
|
@@ -2283,16 +2628,30 @@ def _reinstall_tail_slots(account_key: str, tail: "_Evaluation | None", *, at: i
|
|
|
2283
2628
|
_write_candidate(member)
|
|
2284
2629
|
|
|
2285
2630
|
|
|
2631
|
+
def _same_arrival(a: "_candidates.Candidate", b: "_candidates.Candidate") -> bool:
|
|
2632
|
+
"""One immutable arrival apart from its provider-activity witness: the
|
|
2633
|
+
same contributor, binding, receipt second and raw readings."""
|
|
2634
|
+
return dataclasses.replace(a, activity=None) == dataclasses.replace(b, activity=None)
|
|
2635
|
+
|
|
2636
|
+
|
|
2286
2637
|
def _intake_membership(account_key: str, *, at: int, incoming=None) -> tuple:
|
|
2287
2638
|
"""Under the intake lock, after `_reinstall_tail_slots`: each
|
|
2288
2639
|
contributor's latest committed arrival eligible at ``at``, with the
|
|
2289
2640
|
incoming arrival substituted for its own contributor exactly once and
|
|
2290
|
-
every other eligible contributor kept.
|
|
2641
|
+
every other eligible contributor kept.
|
|
2642
|
+
|
|
2643
|
+
Returns ``(members, incoming)``. An incoming arrival identical to its
|
|
2644
|
+
contributor's committed one except for a later capture's witness is the
|
|
2645
|
+
SAME arrival (#797 spec 4.2): it reuses the first committed witness, so
|
|
2646
|
+
its intake identity is the committed one's."""
|
|
2291
2647
|
members = {c.token: c for c in _read_account_candidates(
|
|
2292
2648
|
account_key, now_epoch=at, prune=False)}
|
|
2293
2649
|
if incoming is not None:
|
|
2650
|
+
committed = members.get(incoming.token)
|
|
2651
|
+
if committed is not None and _same_arrival(committed, incoming):
|
|
2652
|
+
incoming = committed
|
|
2294
2653
|
members[incoming.token] = incoming
|
|
2295
|
-
return tuple(members[token] for token in sorted(members))
|
|
2654
|
+
return tuple(members[token] for token in sorted(members)), incoming
|
|
2296
2655
|
|
|
2297
2656
|
|
|
2298
2657
|
def _intake_enroll(account_key: str, *, acknowledged: "int | None" = None,
|
|
@@ -2325,7 +2684,7 @@ def _intake_enroll(account_key: str, *, acknowledged: "int | None" = None,
|
|
|
2325
2684
|
at = incoming.received_at if incoming is not None else int(time.time())
|
|
2326
2685
|
tail = _intake_tail_evaluation(account_key)
|
|
2327
2686
|
_reinstall_tail_slots(account_key, tail, at=at)
|
|
2328
|
-
members = _intake_membership(account_key, at=at, incoming=incoming)
|
|
2687
|
+
members, incoming = _intake_membership(account_key, at=at, incoming=incoming)
|
|
2329
2688
|
if require_members and not members:
|
|
2330
2689
|
return None
|
|
2331
2690
|
if tail is None and acknowledged is None:
|
|
@@ -2433,6 +2792,38 @@ class _Transition:
|
|
|
2433
2792
|
#: against it (`_render_acknowledge`). Never carried over to a projection
|
|
2434
2793
|
#: it was not proven for; None when absent.
|
|
2435
2794
|
coverage: "dict | None" = None
|
|
2795
|
+
#: #797 spec 4.4 (D7b q3): the account's accounting receipts
|
|
2796
|
+
#: (`_lib_stale_proof.Receipt`), created, validated and pruned only by the
|
|
2797
|
+
#: detached reducer at a covered completion; every other writer carries
|
|
2798
|
+
#: them over unchanged.
|
|
2799
|
+
receipts: tuple = ()
|
|
2800
|
+
#: #797 spec 4.4 (D7c q3): the covered facts the render needs to tell
|
|
2801
|
+
#: whether a NOOP leaves receipt work (`_StaleCache`), written with the
|
|
2802
|
+
#: projection in ``control`` and, like ``coverage``, never carried over
|
|
2803
|
+
#: to a projection it was not read for. None when absent.
|
|
2804
|
+
stale_cache: "object | None" = None
|
|
2805
|
+
|
|
2806
|
+
|
|
2807
|
+
@dataclasses.dataclass(frozen=True)
|
|
2808
|
+
class _StaleCache:
|
|
2809
|
+
"""The render's cached receipt-earning inputs (#797 spec 4.4, D7c q3):
|
|
2810
|
+
the accounted weekly window and value, and the compact provider samples
|
|
2811
|
+
read after ``samples_after`` (`_lib_stale_proof.compact_samples`)."""
|
|
2812
|
+
|
|
2813
|
+
window: "int | None"
|
|
2814
|
+
accounted: object
|
|
2815
|
+
samples: tuple
|
|
2816
|
+
samples_after: "str | None"
|
|
2817
|
+
samples_covered: bool
|
|
2818
|
+
#: Task 5: the five-hour window and its accounted value (path (i) only,
|
|
2819
|
+
#: so no samples are needed to tell whether a member could earn).
|
|
2820
|
+
five_window: "int | None" = None
|
|
2821
|
+
five_accounted_percent: "float | None" = None
|
|
2822
|
+
#: F-I4-2: the sample identities whose receipts the completion that
|
|
2823
|
+
#: wrote this cache earned or held and then had to drop at the document
|
|
2824
|
+
#: limit. They are no longer receipt work for a render (eviction and
|
|
2825
|
+
#: omission still mean eligible) until a writer replaces this cache.
|
|
2826
|
+
omitted: frozenset = frozenset()
|
|
2436
2827
|
|
|
2437
2828
|
|
|
2438
2829
|
@dataclasses.dataclass(frozen=True)
|
|
@@ -2485,9 +2876,105 @@ def _transition_document(transition: _Transition) -> dict:
|
|
|
2485
2876
|
else {"attemptEvidence": transition.attempt_evidence}),
|
|
2486
2877
|
**({} if transition.coverage is None
|
|
2487
2878
|
else {"journalCoverage": transition.coverage}),
|
|
2879
|
+
**({} if not transition.receipts
|
|
2880
|
+
else {"accountingReceipts": [_receipt_document(r)
|
|
2881
|
+
for r in transition.receipts]}),
|
|
2882
|
+
**({} if transition.stale_cache is None
|
|
2883
|
+
else {"staleContributorCache": _stale_cache_document(transition.stale_cache)}),
|
|
2884
|
+
}
|
|
2885
|
+
|
|
2886
|
+
|
|
2887
|
+
def _receipt_document(receipt) -> dict:
|
|
2888
|
+
import _lib_stale_proof
|
|
2889
|
+
|
|
2890
|
+
return _lib_stale_proof.receipt_to_json(receipt)
|
|
2891
|
+
|
|
2892
|
+
|
|
2893
|
+
_SAMPLE_FIELDS = ("observationId", "requestedAt", "capturedAt", "weeklyPercent",
|
|
2894
|
+
"weeklyResetsAt")
|
|
2895
|
+
|
|
2896
|
+
|
|
2897
|
+
def _stale_cache_document(cache: _StaleCache) -> dict:
|
|
2898
|
+
accounted = cache.accounted
|
|
2899
|
+
return {
|
|
2900
|
+
"window": cache.window,
|
|
2901
|
+
"accountedPercent": None if accounted is None else accounted.percent,
|
|
2902
|
+
"floorEpoch": None if accounted is None else accounted.floor_epoch,
|
|
2903
|
+
"samplesAfter": cache.samples_after,
|
|
2904
|
+
"samplesCovered": cache.samples_covered,
|
|
2905
|
+
"samples": [[s.observation_id, s.requested_at_utc, s.captured_at_utc,
|
|
2906
|
+
s.weekly_percent, s.weekly_resets_at_utc] for s in cache.samples],
|
|
2907
|
+
"fiveHourWindow": cache.five_window,
|
|
2908
|
+
"fiveHourAccountedPercent": cache.five_accounted_percent,
|
|
2909
|
+
"omitted": sorted(cache.omitted),
|
|
2488
2910
|
}
|
|
2489
2911
|
|
|
2490
2912
|
|
|
2913
|
+
def _validate_stale_cache(value, *, account_key: str):
|
|
2914
|
+
"""The transition's render cache of receipt-earning inputs, or None."""
|
|
2915
|
+
import _lib_oauth_samples
|
|
2916
|
+
import _lib_stale_proof
|
|
2917
|
+
|
|
2918
|
+
if value is None:
|
|
2919
|
+
return None
|
|
2920
|
+
doc = _candidates._require_object(value, "staleContributorCache")
|
|
2921
|
+
base = {"window", "accountedPercent", "floorEpoch", "samplesAfter",
|
|
2922
|
+
"samplesCovered", "samples"}
|
|
2923
|
+
five_keys = {"fiveHourWindow", "fiveHourAccountedPercent", "omitted"}
|
|
2924
|
+
if set(doc) != base and set(doc) != base | five_keys:
|
|
2925
|
+
raise _candidates.StateValidationError("unexpected stale contributor cache keys")
|
|
2926
|
+
five_window = doc.get("fiveHourWindow")
|
|
2927
|
+
five_percent = doc.get("fiveHourAccountedPercent")
|
|
2928
|
+
omitted = doc.get("omitted", [])
|
|
2929
|
+
if (not (five_window is None or _candidates._is_int(five_window))
|
|
2930
|
+
or not isinstance(omitted, list)
|
|
2931
|
+
or not all(isinstance(k, str) and len(k) == 64 for k in omitted)):
|
|
2932
|
+
raise _candidates.StateValidationError("invalid stale contributor cache")
|
|
2933
|
+
if five_percent is not None:
|
|
2934
|
+
five_percent = _candidates._percent(five_percent)
|
|
2935
|
+
window, percent, floor = doc["window"], doc["accountedPercent"], doc["floorEpoch"]
|
|
2936
|
+
after, covered, rows = doc["samplesAfter"], doc["samplesCovered"], doc["samples"]
|
|
2937
|
+
if (not (window is None or _candidates._is_int(window))
|
|
2938
|
+
or not (floor is None or _candidates._is_int(floor))
|
|
2939
|
+
or not isinstance(covered, bool) or not isinstance(rows, list)
|
|
2940
|
+
or not (after is None or isinstance(after, str))):
|
|
2941
|
+
raise _candidates.StateValidationError("invalid stale contributor cache")
|
|
2942
|
+
if percent is not None:
|
|
2943
|
+
percent = _candidates._percent(percent)
|
|
2944
|
+
samples = []
|
|
2945
|
+
for row in rows:
|
|
2946
|
+
if (not isinstance(row, list) or len(row) != len(_SAMPLE_FIELDS)
|
|
2947
|
+
or not all(isinstance(row[i], str) for i in (0, 1, 2, 4))
|
|
2948
|
+
or isinstance(row[3], bool) or not isinstance(row[3], (int, float))):
|
|
2949
|
+
raise _candidates.StateValidationError("invalid cached provider sample")
|
|
2950
|
+
samples.append(_lib_oauth_samples.OAuthSample(
|
|
2951
|
+
account_key, row[0], row[1], row[2], float(row[3]), row[4], None, None, None))
|
|
2952
|
+
accounted = None
|
|
2953
|
+
if window is not None:
|
|
2954
|
+
accounted = _lib_stale_proof.Accounted(
|
|
2955
|
+
account_key=account_key, axis="sevenDay", window=window, percent=percent,
|
|
2956
|
+
support_id=None, support_hash=None, floor_epoch=floor, credits=(),
|
|
2957
|
+
coverage=None)
|
|
2958
|
+
return _StaleCache(window, accounted, tuple(samples), after, covered,
|
|
2959
|
+
five_window, five_percent, frozenset(omitted))
|
|
2960
|
+
|
|
2961
|
+
|
|
2962
|
+
def _validate_receipts(value, *, account_key: str) -> tuple:
|
|
2963
|
+
import _lib_stale_proof
|
|
2964
|
+
|
|
2965
|
+
if value is None:
|
|
2966
|
+
return ()
|
|
2967
|
+
if not isinstance(value, list) or not value:
|
|
2968
|
+
raise _candidates.StateValidationError("invalid accounting receipts")
|
|
2969
|
+
try:
|
|
2970
|
+
receipts = tuple(_lib_stale_proof.receipt_from_json(doc) for doc in value)
|
|
2971
|
+
except _lib_stale_proof.MalformedReceipt as exc:
|
|
2972
|
+
raise _candidates.StateValidationError(f"invalid accounting receipt: {exc}")
|
|
2973
|
+
if any(r.account_key != account_key for r in receipts):
|
|
2974
|
+
raise _candidates.StateValidationError("cross-account accounting receipt")
|
|
2975
|
+
return receipts
|
|
2976
|
+
|
|
2977
|
+
|
|
2491
2978
|
def _validate_publication(value, *, account_key: str):
|
|
2492
2979
|
"""A frozen publication plan document, or None. Strict: every axis it
|
|
2493
2980
|
publishes names this account and its own window in its provenance."""
|
|
@@ -2505,12 +2992,20 @@ def _validate_publication(value, *, account_key: str):
|
|
|
2505
2992
|
# its fold-policy revision; one frozen earlier has none and keeps the
|
|
2506
2993
|
# historical policy through recovery.
|
|
2507
2994
|
policy_key = _cctally_journal.PLAN_FOLD_POLICY_KEY
|
|
2508
|
-
|
|
2995
|
+
# #797/#823/#840 spec 4.9 (D12): a v2 plan also freezes its detector
|
|
2996
|
+
# revision, which must be exactly the integer 2; a plan frozen earlier has
|
|
2997
|
+
# none and keeps the historical detector through recovery.
|
|
2998
|
+
detector_key = _cctally_journal.PLAN_DETECTOR_POLICY_KEY
|
|
2999
|
+
if set(doc) - {policy_key, detector_key} != keys:
|
|
2509
3000
|
raise _candidates.StateValidationError("unexpected publication keys")
|
|
2510
3001
|
if policy_key in doc and (
|
|
2511
3002
|
type(doc[policy_key]) is not int
|
|
2512
3003
|
or doc[policy_key] != _cctally_journal.FOLD_POLICY_FIRST_POST_FLOOR):
|
|
2513
3004
|
raise _candidates.StateValidationError("invalid publication fold policy")
|
|
3005
|
+
if detector_key in doc and (
|
|
3006
|
+
type(doc[detector_key]) is not int
|
|
3007
|
+
or doc[detector_key] != _cctally_journal.DETECTOR_POLICY):
|
|
3008
|
+
raise _candidates.StateValidationError("invalid publication detector policy")
|
|
2514
3009
|
if (not isinstance(doc["publicationId"], str)
|
|
2515
3010
|
or _PUBLICATION_ID_RE.fullmatch(doc["publicationId"]) is None):
|
|
2516
3011
|
raise _candidates.StateValidationError("invalid publication identity")
|
|
@@ -2573,7 +3068,8 @@ def _read_transition(account_key: str, *, now_epoch: int) -> "_Transition | None
|
|
|
2573
3068
|
"intakeAcknowledged"}
|
|
2574
3069
|
_candidates._require_exact_keys(
|
|
2575
3070
|
doc, base_keys | ({"outcomeQuarantine", "attemptEvidence",
|
|
2576
|
-
"journalCoverage"
|
|
3071
|
+
"journalCoverage", "accountingReceipts",
|
|
3072
|
+
"staleContributorCache"} & set(doc)))
|
|
2577
3073
|
quarantine = _validate_transition_quarantine(doc.get("outcomeQuarantine"))
|
|
2578
3074
|
attempt_evidence = doc.get("attemptEvidence")
|
|
2579
3075
|
if attempt_evidence is not None and (
|
|
@@ -2581,6 +3077,9 @@ def _read_transition(account_key: str, *, now_epoch: int) -> "_Transition | None
|
|
|
2581
3077
|
or _EVIDENCE_RE.fullmatch(attempt_evidence) is None):
|
|
2582
3078
|
raise _candidates.StateValidationError("invalid attempt evidence")
|
|
2583
3079
|
coverage = _validate_coverage(doc.get("journalCoverage"))
|
|
3080
|
+
receipts = _validate_receipts(doc.get("accountingReceipts"), account_key=account_key)
|
|
3081
|
+
stale_cache = _validate_stale_cache(doc.get("staleContributorCache"),
|
|
3082
|
+
account_key=account_key)
|
|
2584
3083
|
acknowledged = doc["intakeAcknowledged"]
|
|
2585
3084
|
if not _candidates._is_int(acknowledged) or acknowledged < -1:
|
|
2586
3085
|
raise _candidates.StateValidationError("invalid intake checkpoint")
|
|
@@ -2619,6 +3118,8 @@ def _read_transition(account_key: str, *, now_epoch: int) -> "_Transition | None
|
|
|
2619
3118
|
intake_acknowledged=acknowledged,
|
|
2620
3119
|
attempt_evidence=attempt_evidence,
|
|
2621
3120
|
coverage=coverage,
|
|
3121
|
+
receipts=receipts,
|
|
3122
|
+
stale_cache=stale_cache,
|
|
2622
3123
|
)
|
|
2623
3124
|
|
|
2624
3125
|
|
|
@@ -2709,12 +3210,15 @@ def _statusline_control_db_agreement(*, now_epoch: int) -> bool | None:
|
|
|
2709
3210
|
def _replace_transition(transition: "_Transition | None", *, account_key: str,
|
|
2710
3211
|
binding, projection, control=None, anchor=..., publication=...,
|
|
2711
3212
|
quarantine=..., acknowledged=..., attempt_evidence=...,
|
|
2712
|
-
coverage=None):
|
|
3213
|
+
coverage=None, receipts=..., stale_cache=None):
|
|
2713
3214
|
"""The next generation of ``account_key``'s transition. Every field left
|
|
2714
3215
|
as ``...`` carries over — the intake checkpoint in particular survives
|
|
2715
|
-
every writer that does not acknowledge (an OAuth reconcile)
|
|
2716
|
-
|
|
2717
|
-
|
|
3216
|
+
every writer that does not acknowledge (an OAuth reconcile), and so do
|
|
3217
|
+
the accounting receipts (#797 spec 4.4: only a covered detached
|
|
3218
|
+
completion replaces them). The journal coverage proof and the render's
|
|
3219
|
+
stale-contributor cache are the exception: each is written only by a
|
|
3220
|
+
caller that proves it for THIS projection, and is otherwise cleared
|
|
3221
|
+
(P1-2)."""
|
|
2718
3222
|
if transition is None:
|
|
2719
3223
|
base_control = _empty_control(projection)
|
|
2720
3224
|
generation = 0
|
|
@@ -2723,6 +3227,7 @@ def _replace_transition(transition: "_Transition | None", *, account_key: str,
|
|
|
2723
3227
|
prior_quarantine = None
|
|
2724
3228
|
prior_acknowledged = -1
|
|
2725
3229
|
prior_evidence = None
|
|
3230
|
+
prior_receipts = ()
|
|
2726
3231
|
else:
|
|
2727
3232
|
base_control = transition.control
|
|
2728
3233
|
generation = transition.generation
|
|
@@ -2731,6 +3236,7 @@ def _replace_transition(transition: "_Transition | None", *, account_key: str,
|
|
|
2731
3236
|
prior_quarantine = transition.quarantine
|
|
2732
3237
|
prior_acknowledged = transition.intake_acknowledged
|
|
2733
3238
|
prior_evidence = transition.attempt_evidence
|
|
3239
|
+
prior_receipts = transition.receipts
|
|
2734
3240
|
return _Transition(
|
|
2735
3241
|
account_key=account_key,
|
|
2736
3242
|
binding=binding,
|
|
@@ -2745,6 +3251,8 @@ def _replace_transition(transition: "_Transition | None", *, account_key: str,
|
|
|
2745
3251
|
attempt_evidence=(prior_evidence if attempt_evidence is ...
|
|
2746
3252
|
else attempt_evidence),
|
|
2747
3253
|
coverage=coverage,
|
|
3254
|
+
receipts=prior_receipts if receipts is ... else tuple(receipts),
|
|
3255
|
+
stale_cache=stale_cache,
|
|
2748
3256
|
)
|
|
2749
3257
|
|
|
2750
3258
|
|
|
@@ -3255,7 +3763,8 @@ def _axis_value_document(value) -> "dict | None":
|
|
|
3255
3763
|
|
|
3256
3764
|
def _bound_publication_plan(decision, projection, candidates, binding, *,
|
|
3257
3765
|
generation: int, now_epoch: int,
|
|
3258
|
-
intake_sequence: int
|
|
3766
|
+
intake_sequence: int,
|
|
3767
|
+
refused=None) -> "dict | None":
|
|
3259
3768
|
"""Freeze one partition's combined two-axis publication plan.
|
|
3260
3769
|
|
|
3261
3770
|
Every published axis records its provenance — ``candidate`` (the winning
|
|
@@ -3265,9 +3774,18 @@ def _bound_publication_plan(decision, projection, candidates, binding, *,
|
|
|
3265
3774
|
the evidence) — and every one names THIS account. The plan carries its
|
|
3266
3775
|
binding, its captured instant and a stable publication identity; the
|
|
3267
3776
|
decision's control state rides along so a recovered plan commits the
|
|
3268
|
-
same control replacement.
|
|
3777
|
+
same control replacement.
|
|
3778
|
+
|
|
3779
|
+
``refused`` maps an axis to the contributor tokens the reducer refused
|
|
3780
|
+
on it (#797 spec 4.4): a candidate axis's provenance names the selected
|
|
3781
|
+
ELIGIBLE winner, never a refused member that happens to tie its value.
|
|
3782
|
+
Refusal filters only a rise: an accepted drop (consensus or deadline,
|
|
3783
|
+
below the same window's baseline) publishes exactly as today, its
|
|
3784
|
+
provenance included (review RV-3), because the reducer selects it from
|
|
3785
|
+
the unfiltered set."""
|
|
3269
3786
|
import _cctally_journal
|
|
3270
3787
|
|
|
3788
|
+
refused = refused or {}
|
|
3271
3789
|
account_key = binding.account_key
|
|
3272
3790
|
if projection.account_key not in (None, account_key):
|
|
3273
3791
|
raise ValueError("cross-account publication projection")
|
|
@@ -3287,18 +3805,26 @@ def _bound_publication_plan(decision, projection, candidates, binding, *,
|
|
|
3287
3805
|
"logicalResetGeneration": source.reset_generation,
|
|
3288
3806
|
}
|
|
3289
3807
|
continue
|
|
3808
|
+
baseline = _candidates._projection_axis(projection, axis)
|
|
3809
|
+
drop = (baseline is not None
|
|
3810
|
+
and baseline.canonical_key == value.canonical_key
|
|
3811
|
+
and value.percent < baseline.percent)
|
|
3812
|
+
excluded = () if drop else refused.get(axis, ())
|
|
3290
3813
|
winners = sorted(
|
|
3291
|
-
candidate
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3814
|
+
(candidate for candidate in candidates
|
|
3815
|
+
if (axis_value := _candidates._candidate_axis(candidate, axis)) is not None
|
|
3816
|
+
and axis_value.canonical_key == value.canonical_key
|
|
3817
|
+
and axis_value.percent == value.percent
|
|
3818
|
+
and candidate.token not in excluded),
|
|
3819
|
+
key=lambda candidate: candidate.token,
|
|
3295
3820
|
)
|
|
3296
3821
|
if winners:
|
|
3297
3822
|
provenance[axis] = {
|
|
3298
3823
|
"kind": "candidate", "account": account_key,
|
|
3299
|
-
"window": value.canonical_key, "token": winners[0],
|
|
3824
|
+
"window": value.canonical_key, "token": winners[0].token,
|
|
3300
3825
|
"observationIdentity": _candidates.observation_identity(
|
|
3301
|
-
winners[0], axis, value),
|
|
3826
|
+
winners[0].token, axis, value),
|
|
3827
|
+
**_activity_provenance(winners[0]),
|
|
3302
3828
|
}
|
|
3303
3829
|
else:
|
|
3304
3830
|
provenance[axis] = {
|
|
@@ -3316,6 +3842,9 @@ def _bound_publication_plan(decision, projection, candidates, binding, *,
|
|
|
3316
3842
|
# observation is decided under, frozen with (and inside the identity
|
|
3317
3843
|
# of) the plan so recovery never re-policies it.
|
|
3318
3844
|
_cctally_journal.PLAN_FOLD_POLICY_KEY: _cctally_journal.CURRENT_FOLD_POLICY,
|
|
3845
|
+
# #797/#823/#840 spec 4.9 (D12): the detector revision, frozen beside
|
|
3846
|
+
# the fold policy and inside the identity for the same reason.
|
|
3847
|
+
_cctally_journal.PLAN_DETECTOR_POLICY_KEY: _cctally_journal.DETECTOR_POLICY,
|
|
3319
3848
|
}
|
|
3320
3849
|
plan["publicationId"] = hashlib.sha256(json.dumps(
|
|
3321
3850
|
{"protocolGeneration": _candidates.PROTOCOL_GENERATION,
|
|
@@ -3330,6 +3859,307 @@ def _bound_publication_plan(decision, projection, candidates, binding, *,
|
|
|
3330
3859
|
return plan
|
|
3331
3860
|
|
|
3332
3861
|
|
|
3862
|
+
def _activity_provenance(candidate: "_candidates.Candidate") -> dict:
|
|
3863
|
+
"""The selected contributor's frozen provider-activity evidence as plan
|
|
3864
|
+
provenance (#797 spec 4.5): its instant (or null), its typed status and
|
|
3865
|
+
the evidence revision. Part of the publication identity; never a
|
|
3866
|
+
transcript path or session id."""
|
|
3867
|
+
import _lib_provider_activity
|
|
3868
|
+
|
|
3869
|
+
activity = candidate.activity
|
|
3870
|
+
if not isinstance(activity, _lib_provider_activity.ActivityWitness):
|
|
3871
|
+
raise ValueError("a current candidate requires its activity witness")
|
|
3872
|
+
return {
|
|
3873
|
+
"providerActivityAt": activity.instant_utc,
|
|
3874
|
+
"providerActivityStatus": activity.status,
|
|
3875
|
+
"providerActivityRevision": activity.evidence_revision,
|
|
3876
|
+
}
|
|
3877
|
+
|
|
3878
|
+
|
|
3879
|
+
# --- #797 spec 4.4: refusal facts and accounting receipts --------------------
|
|
3880
|
+
|
|
3881
|
+
|
|
3882
|
+
def _axis_members(candidates, axis: str) -> list:
|
|
3883
|
+
"""The ``axis`` members with known, supported provider-activity evidence
|
|
3884
|
+
as `_lib_stale_proof.Member`s, in their window: the normalized weekly
|
|
3885
|
+
reset, or the canonical five-hour window key (the candidate's own when
|
|
3886
|
+
it carries one). A member with unknown evidence is absent: it is always
|
|
3887
|
+
eligible and never earns (O5)."""
|
|
3888
|
+
import _lib_stale_proof
|
|
3889
|
+
|
|
3890
|
+
members = []
|
|
3891
|
+
for candidate in candidates:
|
|
3892
|
+
value = candidate.seven_day if axis == "sevenDay" else candidate.five_hour
|
|
3893
|
+
if value is None:
|
|
3894
|
+
continue
|
|
3895
|
+
if axis == "sevenDay":
|
|
3896
|
+
window = int(_cctally_core._normalize_week_boundary_dt(
|
|
3897
|
+
dt.datetime.fromtimestamp(value.raw_resets_at, tz=dt.timezone.utc)
|
|
3898
|
+
).timestamp())
|
|
3899
|
+
elif value.canonical_key is not None:
|
|
3900
|
+
window = int(value.canonical_key)
|
|
3901
|
+
else:
|
|
3902
|
+
window = int(_cctally()._canonical_5h_window_key(int(value.raw_resets_at)))
|
|
3903
|
+
sigma = _lib_stale_proof.sample_identity(
|
|
3904
|
+
candidate.token, candidate.activity, axis=axis,
|
|
3905
|
+
percent=value.percent, window=window)
|
|
3906
|
+
if sigma is not None:
|
|
3907
|
+
members.append(_lib_stale_proof.Member(sigma, int(value.raw_resets_at)))
|
|
3908
|
+
return members
|
|
3909
|
+
|
|
3910
|
+
|
|
3911
|
+
def _weekly_members(candidates) -> list:
|
|
3912
|
+
return _axis_members(candidates, "sevenDay")
|
|
3913
|
+
|
|
3914
|
+
|
|
3915
|
+
def _all_members(candidates) -> list:
|
|
3916
|
+
return [*_axis_members(candidates, "sevenDay"), *_axis_members(candidates, "fiveHour")]
|
|
3917
|
+
|
|
3918
|
+
|
|
3919
|
+
def _stale_axes(stale: "_StaleInputs") -> tuple:
|
|
3920
|
+
"""Per axis: ``(axis, window, samples, credits, supports)``. The
|
|
3921
|
+
five-hour axis checks no supports (its accounted value is detector
|
|
3922
|
+
state)."""
|
|
3923
|
+
return (("sevenDay", stale.window, stale.samples, stale.credits, stale.supports),
|
|
3924
|
+
("fiveHour", stale.five_window, stale.five_samples, stale.five_credits, None))
|
|
3925
|
+
|
|
3926
|
+
|
|
3927
|
+
def _stale_member_facts(stale: "_StaleInputs | None", receipts, candidates):
|
|
3928
|
+
"""The reducer's per-axis member facts (#797 spec 4.4), or None when no
|
|
3929
|
+
member is refused, so such a window reduces exactly as before. Each axis
|
|
3930
|
+
is decided on its own window's members, samples, credits and receipts;
|
|
3931
|
+
an axis without a refused member is absent (all eligible)."""
|
|
3932
|
+
import _lib_stale_proof
|
|
3933
|
+
|
|
3934
|
+
if stale is None or not receipts:
|
|
3935
|
+
return None
|
|
3936
|
+
out = {}
|
|
3937
|
+
for axis, window, samples, credits, supports in _stale_axes(stale):
|
|
3938
|
+
if window is None:
|
|
3939
|
+
continue
|
|
3940
|
+
members = [m for m in _axis_members(candidates, axis) if m.sigma.window == window]
|
|
3941
|
+
if not members:
|
|
3942
|
+
continue
|
|
3943
|
+
facts = _lib_stale_proof.refusal_facts(
|
|
3944
|
+
axis, members, samples, receipts, credits,
|
|
3945
|
+
index_identity=stale.index_identity, supports=supports)
|
|
3946
|
+
if any(f.refused for f in facts.values()):
|
|
3947
|
+
out[axis] = facts
|
|
3948
|
+
return out or None
|
|
3949
|
+
|
|
3950
|
+
|
|
3951
|
+
def _refused_tokens(member_facts) -> dict:
|
|
3952
|
+
return {axis: frozenset(token for token, f in facts.items() if f.refused)
|
|
3953
|
+
for axis, facts in (member_facts or {}).items()}
|
|
3954
|
+
|
|
3955
|
+
|
|
3956
|
+
def _completion_instant() -> str:
|
|
3957
|
+
"""``e_σ``: the wall clock at a covered completion, in the samples' ``q``
|
|
3958
|
+
form. It is an upper bound on the completion, taken after it; an earlier
|
|
3959
|
+
one is never reconstructed (from journal capture times or otherwise)."""
|
|
3960
|
+
import _lib_oauth_samples
|
|
3961
|
+
|
|
3962
|
+
return _lib_oauth_samples.format_request_started(
|
|
3963
|
+
dt.datetime.fromtimestamp(time.time(), tz=dt.timezone.utc))
|
|
3964
|
+
|
|
3965
|
+
|
|
3966
|
+
def _completion_receipts(transition, classified, members):
|
|
3967
|
+
"""The receipt map and render cache a covered, completed derivation
|
|
3968
|
+
installs (#797 spec 4.4, D7b q3): the transition's receipts revalidated
|
|
3969
|
+
and pruned against ``classified``, merged with what ``members`` earn
|
|
3970
|
+
now. Without stale inputs the map is carried over unchanged and no cache
|
|
3971
|
+
is written."""
|
|
3972
|
+
import _lib_stale_proof
|
|
3973
|
+
|
|
3974
|
+
existing = () if transition is None else transition.receipts
|
|
3975
|
+
stale = getattr(classified, "stale", None)
|
|
3976
|
+
if stale is None:
|
|
3977
|
+
return existing, None
|
|
3978
|
+
kept, earned = [], []
|
|
3979
|
+
completed = _completion_instant()
|
|
3980
|
+
for axis, window, samples, credits, supports in _stale_axes(stale):
|
|
3981
|
+
kept.extend(_lib_stale_proof.prune_receipts(
|
|
3982
|
+
[r for r in existing if r.sigma.axis == axis], credits,
|
|
3983
|
+
index_identity=stale.index_identity, window=window, supports=supports))
|
|
3984
|
+
accounted = stale.accounted if axis == "sevenDay" else stale.five_accounted
|
|
3985
|
+
if accounted is None or window is None:
|
|
3986
|
+
continue
|
|
3987
|
+
# The five-hour axis earns by exact equality alone (path (i)); its
|
|
3988
|
+
# samples feed refusal, never earning.
|
|
3989
|
+
earned.extend(_lib_stale_proof.earn_receipts(
|
|
3990
|
+
[m for m in _axis_members(members, axis) if m.sigma.window == window],
|
|
3991
|
+
accounted,
|
|
3992
|
+
stale.samples if axis == "sevenDay" and stale.samples_covered else (),
|
|
3993
|
+
completed_at_utc=completed, index_identity=stale.index_identity))
|
|
3994
|
+
receipts = _lib_stale_proof.latest_per_token(
|
|
3995
|
+
_lib_stale_proof.merge_receipts(kept, earned))
|
|
3996
|
+
floor = None if stale.accounted is None else stale.accounted.floor_epoch
|
|
3997
|
+
cache = _StaleCache(
|
|
3998
|
+
stale.window, stale.accounted,
|
|
3999
|
+
() if stale.window is None else _lib_stale_proof.compact_samples(
|
|
4000
|
+
stale.samples, window=stale.window, floor_epoch=floor),
|
|
4001
|
+
stale.samples_after, stale.samples_covered,
|
|
4002
|
+
stale.five_window,
|
|
4003
|
+
None if stale.five_accounted is None else stale.five_accounted.percent)
|
|
4004
|
+
return receipts, cache
|
|
4005
|
+
|
|
4006
|
+
|
|
4007
|
+
def _fit_transition(transition: _Transition, *, prior=(), referenced=()) -> _Transition:
|
|
4008
|
+
"""``transition`` within the 1 MiB control document limit (#797 spec
|
|
4009
|
+
4.4): the oldest receipts no outstanding work references are evicted
|
|
4010
|
+
first, then the receipts new since ``prior`` are omitted, and finally
|
|
4011
|
+
the render cache is dropped — accounting is never blocked, and eviction
|
|
4012
|
+
means eligible."""
|
|
4013
|
+
import _lib_stale_proof
|
|
4014
|
+
|
|
4015
|
+
def size(t) -> int:
|
|
4016
|
+
return len(json.dumps(_transition_document(t), sort_keys=True,
|
|
4017
|
+
separators=(",", ":"), allow_nan=False).encode("utf-8"))
|
|
4018
|
+
|
|
4019
|
+
limit = _candidates.CONTROL_DOCUMENT_MAX_BYTES
|
|
4020
|
+
if size(transition) <= limit:
|
|
4021
|
+
return transition
|
|
4022
|
+
before = {r.sigma.key() for r in prior}
|
|
4023
|
+
evaluated = frozenset(r.sigma.key() for r in transition.receipts)
|
|
4024
|
+
|
|
4025
|
+
def with_receipts(kept) -> _Transition:
|
|
4026
|
+
# F-I4-2: every receipt this completion evaluated and could not keep
|
|
4027
|
+
# (evicted or omitted) is recorded in the render cache, so a render
|
|
4028
|
+
# does not count it as receipt work again until a writer replaces
|
|
4029
|
+
# the cache. Its member stays eligible.
|
|
4030
|
+
fitted = dataclasses.replace(transition, receipts=tuple(kept))
|
|
4031
|
+
if fitted.stale_cache is not None:
|
|
4032
|
+
fitted = dataclasses.replace(fitted, stale_cache=dataclasses.replace(
|
|
4033
|
+
fitted.stale_cache,
|
|
4034
|
+
omitted=evaluated - {r.sigma.key() for r in kept}))
|
|
4035
|
+
return fitted
|
|
4036
|
+
|
|
4037
|
+
receipts = _lib_stale_proof.bounded_receipts(
|
|
4038
|
+
[r for r in transition.receipts if r.sigma.key() in before],
|
|
4039
|
+
[r for r in transition.receipts if r.sigma.key() not in before],
|
|
4040
|
+
referenced_tokens=frozenset(referenced),
|
|
4041
|
+
fits=lambda kept: size(with_receipts(kept)) <= limit)
|
|
4042
|
+
fitted = with_receipts(receipts)
|
|
4043
|
+
if size(fitted) > limit:
|
|
4044
|
+
fitted = dataclasses.replace(fitted, stale_cache=None)
|
|
4045
|
+
return fitted
|
|
4046
|
+
|
|
4047
|
+
|
|
4048
|
+
def _refusal_authority_lost(classified, facts, receipts, members) -> bool:
|
|
4049
|
+
"""Whether a member ``facts`` refused is no longer refused against
|
|
4050
|
+
``receipts`` — the same classification and members, only the receipt
|
|
4051
|
+
map changed (review RV-4: the document limit evicted the receipt a
|
|
4052
|
+
refusal rested on)."""
|
|
4053
|
+
refused = _refused_tokens(facts)
|
|
4054
|
+
if not any(refused.values()):
|
|
4055
|
+
return False
|
|
4056
|
+
after = _refused_tokens(_stale_member_facts(
|
|
4057
|
+
getattr(classified, "stale", None), receipts, members))
|
|
4058
|
+
return any(tokens - after.get(axis, frozenset())
|
|
4059
|
+
for axis, tokens in refused.items())
|
|
4060
|
+
|
|
4061
|
+
|
|
4062
|
+
def _referenced_tokens(transition, members) -> frozenset:
|
|
4063
|
+
tokens = {m.token for m in members}
|
|
4064
|
+
if transition is not None:
|
|
4065
|
+
for pending in transition.control.pending_drops.values():
|
|
4066
|
+
if pending is not None:
|
|
4067
|
+
tokens.update(pending.contributors)
|
|
4068
|
+
return frozenset(tokens)
|
|
4069
|
+
|
|
4070
|
+
|
|
4071
|
+
def _receipt_work(transition: _Transition, members) -> bool:
|
|
4072
|
+
"""Whether a covered detached completion over ``members`` could earn a
|
|
4073
|
+
receipt that ``transition`` does not hold (#797 spec 4.4, D7c q3), judged
|
|
4074
|
+
from the transition's cached facts alone (C4: the render reads no
|
|
4075
|
+
index). Without a cache any member with known evidence is work; a member
|
|
4076
|
+
with unknown evidence is never work."""
|
|
4077
|
+
import _lib_stale_proof
|
|
4078
|
+
|
|
4079
|
+
known = _all_members(members)
|
|
4080
|
+
if not known:
|
|
4081
|
+
return False
|
|
4082
|
+
cache = transition.stale_cache
|
|
4083
|
+
if cache is None:
|
|
4084
|
+
return True
|
|
4085
|
+
# F-I4-2: a receipt the writer evaluated and had to omit is not work.
|
|
4086
|
+
held = {r.sigma.key() for r in transition.receipts} | set(cache.omitted)
|
|
4087
|
+
for member in known:
|
|
4088
|
+
if member.sigma.axis != "fiveHour" or member.sigma.key() in held:
|
|
4089
|
+
continue
|
|
4090
|
+
# Path (i) only: exact equality with the cached five-hour baseline.
|
|
4091
|
+
if (cache.five_window is not None and member.sigma.window == cache.five_window
|
|
4092
|
+
and cache.five_accounted_percent is not None
|
|
4093
|
+
and member.sigma.percent == cache.five_accounted_percent):
|
|
4094
|
+
return True
|
|
4095
|
+
if cache.accounted is None:
|
|
4096
|
+
return False
|
|
4097
|
+
for member in known:
|
|
4098
|
+
if (member.sigma.axis != "sevenDay" or member.sigma.window != cache.window
|
|
4099
|
+
or member.sigma.key() in held):
|
|
4100
|
+
continue
|
|
4101
|
+
if member.sigma.percent == cache.accounted.percent:
|
|
4102
|
+
return True
|
|
4103
|
+
if not cache.samples_covered:
|
|
4104
|
+
continue
|
|
4105
|
+
bound = _lib_stale_proof.floor_microsecond(member.sigma.activity_at_utc)
|
|
4106
|
+
if cache.samples_after is None or bound is None or cache.samples_after > bound:
|
|
4107
|
+
return True
|
|
4108
|
+
if _lib_stale_proof.earn_receipts(
|
|
4109
|
+
[member], cache.accounted, cache.samples,
|
|
4110
|
+
completed_at_utc="1970-01-01T00:00:00.000000Z",
|
|
4111
|
+
index_identity="render"):
|
|
4112
|
+
return True
|
|
4113
|
+
return False
|
|
4114
|
+
|
|
4115
|
+
|
|
4116
|
+
def _stale_scope(account_key: str, candidates, transition) -> tuple:
|
|
4117
|
+
"""``(samples_after, receipts_held)`` for one partition's detached work
|
|
4118
|
+
(`_STALE_SCOPE`): the earliest instant any member of its logged and
|
|
4119
|
+
current evaluations, or any receipt it holds, needs a sample after."""
|
|
4120
|
+
import _lib_stale_proof
|
|
4121
|
+
|
|
4122
|
+
instants = [r.sigma.activity_at_utc
|
|
4123
|
+
for r in (() if transition is None else transition.receipts)]
|
|
4124
|
+
sets = [candidates]
|
|
4125
|
+
try:
|
|
4126
|
+
sets.extend(e.members for e in _intake_evaluations(
|
|
4127
|
+
account_key, after=_acknowledged_through(transition)))
|
|
4128
|
+
except (OSError, ValueError, TypeError, KeyError):
|
|
4129
|
+
pass
|
|
4130
|
+
for members in sets:
|
|
4131
|
+
instants.extend(m.sigma.activity_at_utc for m in _all_members(members))
|
|
4132
|
+
bounds = [b for b in (_lib_stale_proof.floor_microsecond(i) for i in instants)
|
|
4133
|
+
if b is not None]
|
|
4134
|
+
return (min(bounds) if bounds else None,
|
|
4135
|
+
bool(transition is not None and transition.receipts))
|
|
4136
|
+
|
|
4137
|
+
|
|
4138
|
+
def _plan_members(account_key: str, plan: dict) -> tuple:
|
|
4139
|
+
"""The members of a frozen plan's own intake evaluation, or () when it
|
|
4140
|
+
cannot be read back (nothing is then earned from it)."""
|
|
4141
|
+
sequence = int(plan["intakeSequence"])
|
|
4142
|
+
try:
|
|
4143
|
+
evaluations = list(_intake_evaluations(
|
|
4144
|
+
account_key, after=sequence - 1, through=sequence))
|
|
4145
|
+
except (OSError, _candidates.StateValidationError, ValueError, TypeError, KeyError):
|
|
4146
|
+
return ()
|
|
4147
|
+
if len(evaluations) != 1 or evaluations[0].sequence != sequence:
|
|
4148
|
+
return ()
|
|
4149
|
+
return tuple(evaluations[0].members)
|
|
4150
|
+
|
|
4151
|
+
|
|
4152
|
+
def _plan_winner_refused(plan: dict, member_facts) -> bool:
|
|
4153
|
+
"""Whether a frozen plan's candidate provenance names a member the
|
|
4154
|
+
current facts refuse on that axis (#797 spec 4.4, T4)."""
|
|
4155
|
+
refused = _refused_tokens(member_facts)
|
|
4156
|
+
for axis, source in (plan.get("provenance") or {}).items():
|
|
4157
|
+
if (isinstance(source, dict) and source.get("kind") == "candidate"
|
|
4158
|
+
and source.get("token") in refused.get(axis, ())):
|
|
4159
|
+
return True
|
|
4160
|
+
return False
|
|
4161
|
+
|
|
4162
|
+
|
|
3333
4163
|
def _plan_record_args(plan: dict) -> argparse.Namespace:
|
|
3334
4164
|
five = plan["fiveHour"]
|
|
3335
4165
|
return _record_args(
|
|
@@ -3709,7 +4539,8 @@ def _publication_superseded(plan: dict, *, account_key: str, now_epoch: int,
|
|
|
3709
4539
|
|
|
3710
4540
|
|
|
3711
4541
|
def _commit_publication(transition: _Transition, plan: dict, *, before, classified,
|
|
3712
|
-
now_epoch: int, refreshed_control=None
|
|
4542
|
+
now_epoch: int, refreshed_control=None,
|
|
4543
|
+
members=None) -> _Transition:
|
|
3713
4544
|
"""Acknowledge one publication (#825/#832 spec 4.3 revision 2, Codex
|
|
3714
4545
|
design-fix-round-3 Q4): replace the partition's transition with the
|
|
3715
4546
|
committed control, read from ``classified`` — a covered classification
|
|
@@ -3717,8 +4548,17 @@ def _commit_publication(transition: _Transition, plan: dict, *, before, classifi
|
|
|
3717
4548
|
advance the intake checkpoint past the plan's evaluation in the SAME
|
|
3718
4549
|
atomic write. Nothing else is evidence of completion: not the record
|
|
3719
4550
|
kernel's exit code, not an opportunistic ``ran=False``, not the raw
|
|
3720
|
-
observation alone.
|
|
4551
|
+
observation alone.
|
|
4552
|
+
|
|
4553
|
+
#797 spec 4.4 (D7b q3): this is the publication's completion, so the
|
|
4554
|
+
receipts its evaluation's ``members`` earn (read back from the plan's
|
|
4555
|
+
intake evaluation when not given) are installed in the same write, merged
|
|
4556
|
+
into the CURRENT receipt map revalidated against ``classified`` — never a
|
|
4557
|
+
map restored from when the plan was frozen."""
|
|
3721
4558
|
after = classified.projection
|
|
4559
|
+
if members is None:
|
|
4560
|
+
members = _plan_members(transition.account_key, plan)
|
|
4561
|
+
receipts, stale_cache = _completion_receipts(transition, classified, members)
|
|
3722
4562
|
if refreshed_control is None:
|
|
3723
4563
|
control = _candidates.validate_control_document(plan["control"], now_epoch=now_epoch)
|
|
3724
4564
|
refreshed_control = dataclasses.replace(control, db_projection=after)
|
|
@@ -3729,7 +4569,10 @@ def _commit_publication(transition: _Transition, plan: dict, *, before, classifi
|
|
|
3729
4569
|
binding=transition.binding, projection=after,
|
|
3730
4570
|
control=refreshed_control, publication=None,
|
|
3731
4571
|
acknowledged=max(transition.intake_acknowledged, plan["intakeSequence"]),
|
|
3732
|
-
coverage=_coverage_document(classified)
|
|
4572
|
+
coverage=_coverage_document(classified), receipts=receipts,
|
|
4573
|
+
stale_cache=stale_cache)
|
|
4574
|
+
committed = _fit_transition(committed, prior=transition.receipts,
|
|
4575
|
+
referenced=_referenced_tokens(committed, members))
|
|
3733
4576
|
_write_transition(committed)
|
|
3734
4577
|
_crash_point("after-checkpoint")
|
|
3735
4578
|
if before is None or _projection_changed(before, after):
|
|
@@ -3884,9 +4727,22 @@ def _resolve_frozen_publication(transition: _Transition, *, now_epoch: int, nudg
|
|
|
3884
4727
|
except _Unclassifiable as exc:
|
|
3885
4728
|
return _Resolution(transition, None, "deferred",
|
|
3886
4729
|
_blocked(account_key, "deferred", exc))
|
|
4730
|
+
# #797 spec 4.4 (T4): an unappended plan's dependencies are revalidated
|
|
4731
|
+
# before publication. A plan whose selected contributor the current facts
|
|
4732
|
+
# refuse is dropped like a superseded plan, earning nothing, and its
|
|
4733
|
+
# evaluation, still unacknowledged, is reduced again with those facts.
|
|
4734
|
+
facts = _stale_member_facts(classified.stale, transition.receipts,
|
|
4735
|
+
_plan_members(account_key, plan))
|
|
4736
|
+
if _plan_winner_refused(plan, facts):
|
|
4737
|
+
dropped = _replace_transition(
|
|
4738
|
+
transition, account_key=account_key, binding=transition.binding,
|
|
4739
|
+
projection=before, publication=None)
|
|
4740
|
+
_write_transition(dropped)
|
|
4741
|
+
return _Resolution(dropped, classified, "superseded")
|
|
4742
|
+
refused = _refused_tokens(facts)
|
|
3887
4743
|
authority = _plan_drop_authority(
|
|
3888
4744
|
plan, classified.projection,
|
|
3889
|
-
receipts=lambda: _plan_axis_receipts(account_key, plan))
|
|
4745
|
+
receipts=lambda: _plan_axis_receipts(account_key, plan, refused=refused))
|
|
3890
4746
|
if authority == _PLAN_STALE:
|
|
3891
4747
|
# Spec 4.3 revision 6 (*Floor retired*, *Stale expired window*):
|
|
3892
4748
|
# every reading the plan publishes on some axis is older than that
|
|
@@ -3950,11 +4806,15 @@ _PLAN_UNKNOWN = "unknown"
|
|
|
3950
4806
|
_PLAN_STALE = "stale"
|
|
3951
4807
|
|
|
3952
4808
|
|
|
3953
|
-
def _plan_axis_receipts(account_key: str, plan: dict
|
|
4809
|
+
def _plan_axis_receipts(account_key: str, plan: dict, *,
|
|
4810
|
+
refused=None) -> "dict | None":
|
|
3954
4811
|
"""Per axis, the (window, percent, receipt instant) of every member of
|
|
3955
4812
|
the plan's own intake evaluation (fix round K14c): what a floor-retired
|
|
3956
4813
|
window needs to decide whether the plan's reading was captured at or
|
|
3957
|
-
after the floor. None when the evaluation cannot be read back.
|
|
4814
|
+
after the floor. None when the evaluation cannot be read back. A member
|
|
4815
|
+
``refused`` (axis to tokens, #797 spec 4.4) on an axis is excluded from
|
|
4816
|
+
that axis's post-floor set, exactly as `CLASS_FLOOR` excludes it."""
|
|
4817
|
+
refused = refused or {}
|
|
3958
4818
|
sequence = int(plan["intakeSequence"])
|
|
3959
4819
|
try:
|
|
3960
4820
|
evaluations = list(_intake_evaluations(
|
|
@@ -3967,7 +4827,7 @@ def _plan_axis_receipts(account_key: str, plan: dict) -> "dict | None":
|
|
|
3967
4827
|
for member in evaluations[0].members:
|
|
3968
4828
|
for axis in _candidates.AXES:
|
|
3969
4829
|
value = _candidates._candidate_axis(member, axis)
|
|
3970
|
-
if value is not None:
|
|
4830
|
+
if value is not None and member.token not in refused.get(axis, ()):
|
|
3971
4831
|
receipts[axis].append(
|
|
3972
4832
|
(value.canonical_key, value.raw_resets_at, value.percent,
|
|
3973
4833
|
int(member.received_at)))
|
|
@@ -4120,7 +4980,7 @@ def _later_reading_fence(axis: str, tombstone, db_axis, *, at: int,
|
|
|
4120
4980
|
|
|
4121
4981
|
def _reduce_partition(account_key, candidates, transition, *, now_epoch, projection,
|
|
4122
4982
|
drop_consensus_allowed=True, fence_epoch=None,
|
|
4123
|
-
same_second_later=frozenset()):
|
|
4983
|
+
same_second_later=frozenset(), member_facts=None):
|
|
4124
4984
|
"""One partition's pure transition: (decision, anchor, canonical
|
|
4125
4985
|
candidates, tombstones). Only this account's candidates, control,
|
|
4126
4986
|
anchor and fences are inputs. ``now_epoch`` is the reduction instant (a
|
|
@@ -4129,7 +4989,9 @@ def _reduce_partition(account_key, candidates, transition, *, now_epoch, project
|
|
|
4129
4989
|
still a fence for it. ``same_second_later`` names the axes whose
|
|
4130
4990
|
projected reading shares the instant and was accepted after this
|
|
4131
4991
|
evaluation (`_later_reading_fence`); only the detached reducer, which
|
|
4132
|
-
can read the journal, ever supplies it.
|
|
4992
|
+
can read the journal, ever supplies it. ``member_facts`` are the
|
|
4993
|
+
per-axis refusal facts (#797 spec 4.4); only the detached reducer, which
|
|
4994
|
+
reads the provider samples, ever supplies them."""
|
|
4133
4995
|
control = (
|
|
4134
4996
|
_candidates.ControlState(projection, transition.control.pending_drops)
|
|
4135
4997
|
if transition is not None else _empty_control(projection)
|
|
@@ -4150,7 +5012,7 @@ def _reduce_partition(account_key, candidates, transition, *, now_epoch, project
|
|
|
4150
5012
|
decision = _candidates.reduce_candidates(
|
|
4151
5013
|
active, db=projection, control=control, tombstones=tombstones,
|
|
4152
5014
|
now_epoch=now_epoch, account_key=account_key,
|
|
4153
|
-
drop_consensus_allowed=drop_consensus_allowed)
|
|
5015
|
+
drop_consensus_allowed=drop_consensus_allowed, member_facts=member_facts)
|
|
4154
5016
|
return decision, anchor, active, tombstones
|
|
4155
5017
|
|
|
4156
5018
|
|
|
@@ -4432,6 +5294,9 @@ class _Classified:
|
|
|
4432
5294
|
#: projection (`_coverage_document`, P1-2 cycle 3).
|
|
4433
5295
|
extents: tuple = ()
|
|
4434
5296
|
render_valid: "tuple | None" = None
|
|
5297
|
+
#: #797 spec 4.4: the stale-contributor inputs read in the same snapshot
|
|
5298
|
+
#: (`_StaleInputs`), or None when they were not read.
|
|
5299
|
+
stale: "_StaleInputs | None" = None
|
|
4435
5300
|
|
|
4436
5301
|
def serves(self, at: int) -> bool:
|
|
4437
5302
|
"""Whether this classification is exactly the one an evaluation at
|
|
@@ -4525,8 +5390,9 @@ def _classify(account_key: str, frontier: "_PublicationFrontier | None", *,
|
|
|
4525
5390
|
# (K11R-8): only the file fingerprint; the dependency tokens
|
|
4526
5391
|
# are read only if a wait is installed (`_settle_wait`).
|
|
4527
5392
|
_CLASSIFIED_FINGERPRINTS[account_key] = before
|
|
5393
|
+
stale: list = []
|
|
4528
5394
|
projection, valid, render_valid = _read_db_projection_at(
|
|
4529
|
-
account_key=account_key, at=at)
|
|
5395
|
+
account_key=account_key, at=at, stale=stale)
|
|
4530
5396
|
checked = _covered_drop_status(account_key, frontier)
|
|
4531
5397
|
after = _db_file_fingerprint()
|
|
4532
5398
|
except _cctally().StatsRebuildDeferred:
|
|
@@ -4545,7 +5411,8 @@ def _classify(account_key: str, frontier: "_PublicationFrontier | None", *,
|
|
|
4545
5411
|
status, high_water, extents = checked
|
|
4546
5412
|
return _Classified(dataclasses.replace(projection, db_files=after),
|
|
4547
5413
|
status, high_water, int(at), valid,
|
|
4548
|
-
extents=tuple(extents), render_valid=render_valid
|
|
5414
|
+
extents=tuple(extents), render_valid=render_valid,
|
|
5415
|
+
stale=stale[0] if stale else None)
|
|
4549
5416
|
raise _Unclassifiable("coverage", blocks)
|
|
4550
5417
|
|
|
4551
5418
|
|
|
@@ -4593,7 +5460,7 @@ def _bound_append_kwargs(bound) -> dict:
|
|
|
4593
5460
|
|
|
4594
5461
|
def _gated_reduction(account_key, candidates, transition, *, eval_epoch,
|
|
4595
5462
|
fence_epoch, classified: _Classified, frontier=None,
|
|
4596
|
-
same_second_later=frozenset()):
|
|
5463
|
+
same_second_later=frozenset(), member_facts=None):
|
|
4597
5464
|
"""Reduce once with drop consensus open and once closed, against the
|
|
4598
5465
|
classified projection. Only when they differ — a drop below the baseline
|
|
4599
5466
|
would publish — is the quarantine gate consulted, so the ordinary tick
|
|
@@ -4603,12 +5470,14 @@ def _gated_reduction(account_key, candidates, transition, *, eval_epoch,
|
|
|
4603
5470
|
opened = _reduce_partition(account_key, candidates, transition,
|
|
4604
5471
|
now_epoch=eval_epoch, fence_epoch=fence_epoch,
|
|
4605
5472
|
projection=classified.projection,
|
|
4606
|
-
same_second_later=same_second_later
|
|
5473
|
+
same_second_later=same_second_later,
|
|
5474
|
+
member_facts=member_facts)
|
|
4607
5475
|
closed = _reduce_partition(account_key, candidates, transition,
|
|
4608
5476
|
now_epoch=eval_epoch, fence_epoch=fence_epoch,
|
|
4609
5477
|
projection=classified.projection,
|
|
4610
5478
|
drop_consensus_allowed=False,
|
|
4611
|
-
same_second_later=same_second_later
|
|
5479
|
+
same_second_later=same_second_later,
|
|
5480
|
+
member_facts=member_facts)
|
|
4612
5481
|
# Fix round K13 (review cycle 7 finding P1-2): unknown classification
|
|
4613
5482
|
# defers the whole evaluation whatever the gate says — an open gate is
|
|
4614
5483
|
# no classification evidence (spec 4.3 *Classification evidence*).
|
|
@@ -4714,11 +5583,15 @@ def _reduce_evaluation(account_key, evaluation: _Evaluation, transition, *,
|
|
|
4714
5583
|
axis for axis, order in orders.items() if order == _ORDER_LATER)
|
|
4715
5584
|
unknown = frozenset(
|
|
4716
5585
|
axis for axis, order in orders.items() if order == _ORDER_UNKNOWN)
|
|
5586
|
+
# #797 spec 4.4: the refusal facts, resolved before the pure kernel from
|
|
5587
|
+
# this classification's snapshot and the transition's receipts.
|
|
5588
|
+
facts = _stale_member_facts(
|
|
5589
|
+
classified.stale, () if transition is None else transition.receipts, members)
|
|
4717
5590
|
(decision, anchor, active, tombstones), withheld, unestablished = (
|
|
4718
5591
|
_gated_reduction(
|
|
4719
5592
|
account_key, members, transition, eval_epoch=evaluation.at,
|
|
4720
5593
|
fence_epoch=fence_epoch, classified=classified, frontier=frontier,
|
|
4721
|
-
same_second_later=same_second_later | unknown))
|
|
5594
|
+
same_second_later=same_second_later | unknown, member_facts=facts))
|
|
4722
5595
|
if unknown:
|
|
4723
5596
|
# Fix round K6 (825-832-CD6-V01): an order that cannot be established
|
|
4724
5597
|
# is neither later nor earlier. When the reduction is the same either
|
|
@@ -4730,7 +5603,7 @@ def _reduce_evaluation(account_key, evaluation: _Evaluation, transition, *,
|
|
|
4730
5603
|
unfenced, unfenced_withheld, unfenced_unestablished = _gated_reduction(
|
|
4731
5604
|
account_key, members, transition, eval_epoch=evaluation.at,
|
|
4732
5605
|
fence_epoch=fence_epoch, classified=classified, frontier=frontier,
|
|
4733
|
-
same_second_later=same_second_later)
|
|
5606
|
+
same_second_later=same_second_later, member_facts=facts)
|
|
4734
5607
|
if (unfenced[0] != decision or unfenced_withheld != withheld
|
|
4735
5608
|
or unfenced_unestablished != unestablished):
|
|
4736
5609
|
return _Resolution(transition, classified, "deferred", AccountResult(
|
|
@@ -4783,15 +5656,44 @@ def _reduce_evaluation(account_key, evaluation: _Evaluation, transition, *,
|
|
|
4783
5656
|
if decision.action == "PUBLISH_DB":
|
|
4784
5657
|
plan = _bound_publication_plan(
|
|
4785
5658
|
decision, projection, active, binding, generation=generation + 1,
|
|
4786
|
-
now_epoch=evaluation.at, intake_sequence=evaluation.sequence
|
|
5659
|
+
now_epoch=evaluation.at, intake_sequence=evaluation.sequence,
|
|
5660
|
+
refused=_refused_tokens(facts))
|
|
4787
5661
|
if plan is None:
|
|
4788
5662
|
if decision.action == "NOOP" and control_repair_required:
|
|
4789
5663
|
decision = dataclasses.replace(decision, action="WRITE_CONTROL")
|
|
4790
|
-
|
|
5664
|
+
# #797 spec 4.4 (D7a-D7c): a covered detached evaluation that
|
|
5665
|
+
# publishes nothing is a completion — its receipts are earned and
|
|
5666
|
+
# installed in the same atomic write as its control and checkpoint.
|
|
5667
|
+
receipts, stale_cache = _completion_receipts(transition, classified, members)
|
|
5668
|
+
updated = _fit_transition(_replace_transition(
|
|
4791
5669
|
transition, account_key=account_key, binding=binding,
|
|
4792
5670
|
projection=projection, control=decision.control, anchor=anchor,
|
|
4793
5671
|
quarantine=quarantine, acknowledged=acknowledged,
|
|
4794
|
-
coverage=_coverage_document(classified)
|
|
5672
|
+
coverage=_coverage_document(classified), receipts=receipts,
|
|
5673
|
+
stale_cache=stale_cache),
|
|
5674
|
+
prior=() if transition is None else transition.receipts,
|
|
5675
|
+
referenced=_referenced_tokens(transition, members))
|
|
5676
|
+
if (transition is not None
|
|
5677
|
+
and _refusal_authority_lost(classified, facts, updated.receipts,
|
|
5678
|
+
members)):
|
|
5679
|
+
# Review RV-4 (spec 4.4: dependencies of outstanding work are
|
|
5680
|
+
# retained or their refusal authority is invalidated and the
|
|
5681
|
+
# work re-reduced). The document limit's last resort evicted a
|
|
5682
|
+
# receipt this very reduction refused a member on. That refusal
|
|
5683
|
+
# has nothing left to rest on, so the evaluation is not
|
|
5684
|
+
# acknowledged with it: the fitted receipts are installed with
|
|
5685
|
+
# the checkpoint left below this evaluation (a crash leaves it
|
|
5686
|
+
# unacknowledged, for the next pass) and it is reduced again
|
|
5687
|
+
# now, the evicted member eligible. Each round evicts at least
|
|
5688
|
+
# one of the receipts it started from, so this terminates.
|
|
5689
|
+
invalidated = _replace_transition(
|
|
5690
|
+
transition, account_key=account_key, binding=transition.binding,
|
|
5691
|
+
projection=transition.control.db_projection,
|
|
5692
|
+
receipts=updated.receipts)
|
|
5693
|
+
_write_transition(invalidated)
|
|
5694
|
+
return _reduce_evaluation(
|
|
5695
|
+
account_key, evaluation, invalidated, classified=classified,
|
|
5696
|
+
fence_epoch=fence_epoch, nudge_sink=nudge_sink, frontier=frontier)
|
|
4795
5697
|
_write_transition(updated)
|
|
4796
5698
|
_crash_point("after-checkpoint")
|
|
4797
5699
|
return _Resolution(updated, classified, "noop", decision=decision)
|
|
@@ -4820,7 +5722,7 @@ def _reduce_evaluation(account_key, evaluation: _Evaluation, transition, *,
|
|
|
4820
5722
|
active, db=after, control=decision.control,
|
|
4821
5723
|
tombstones=tombstones, now_epoch=evaluation.at,
|
|
4822
5724
|
account_key=account_key,
|
|
4823
|
-
drop_consensus_allowed=not withheld).control
|
|
5725
|
+
drop_consensus_allowed=not withheld, member_facts=facts).control
|
|
4824
5726
|
# The record kernel can deliberately leave the DB unchanged: the
|
|
4825
5727
|
# first zero arms its own debounce and unsupported small drops are HWM
|
|
4826
5728
|
# rejected. Preserve the precomputed bounded attempt state so the next
|
|
@@ -4856,6 +5758,27 @@ def _reduce_and_publish_partition(account_key, candidates, transition, *,
|
|
|
4856
5758
|
frontier: "_PublicationFrontier | None" = None,
|
|
4857
5759
|
drained: "dict | None" = None,
|
|
4858
5760
|
) -> AccountResult:
|
|
5761
|
+
"""`_reduce_and_publish_partition_work` under this partition's #797
|
|
5762
|
+
stale scope (`_STALE_SCOPE`): which provider samples its
|
|
5763
|
+
classifications read for the refusal facts and receipts."""
|
|
5764
|
+
try:
|
|
5765
|
+
scope = _stale_scope(account_key, candidates, transition)
|
|
5766
|
+
except (OSError, ValueError, TypeError, KeyError):
|
|
5767
|
+
scope = (None, bool(transition is not None and transition.receipts))
|
|
5768
|
+
token = _STALE_SCOPE.set(scope)
|
|
5769
|
+
try:
|
|
5770
|
+
return _reduce_and_publish_partition_work(
|
|
5771
|
+
account_key, candidates, transition, now_epoch=now_epoch,
|
|
5772
|
+
nudge_sink=nudge_sink, frontier=frontier, drained=drained)
|
|
5773
|
+
finally:
|
|
5774
|
+
_STALE_SCOPE.reset(token)
|
|
5775
|
+
|
|
5776
|
+
|
|
5777
|
+
def _reduce_and_publish_partition_work(account_key, candidates, transition, *,
|
|
5778
|
+
now_epoch: int, nudge_sink,
|
|
5779
|
+
frontier: "_PublicationFrontier | None" = None,
|
|
5780
|
+
drained: "dict | None" = None,
|
|
5781
|
+
) -> AccountResult:
|
|
4859
5782
|
"""One partition's ordered work (#825/#832 spec 4.3 revision 2, Codex
|
|
4860
5783
|
design-fix-round-3 Q3):
|
|
4861
5784
|
|
|
@@ -5231,10 +6154,15 @@ def _rescan_legacy_spool() -> bool:
|
|
|
5231
6154
|
import _lib_account_protocol
|
|
5232
6155
|
import _lib_legacy_spool
|
|
5233
6156
|
|
|
5234
|
-
|
|
6157
|
+
cutover = _cctally_journal.find_account_bound_cutover()
|
|
6158
|
+
if cutover is None:
|
|
5235
6159
|
return False
|
|
5236
6160
|
try:
|
|
5237
|
-
state
|
|
6161
|
+
# #797/#823/#840 spec 4.6 B4: the v1 coordination state joins the
|
|
6162
|
+
# inventory at the revision-2 boundary, never before it, so a store
|
|
6163
|
+
# whose revision is not yet established compares like for like.
|
|
6164
|
+
state = _cctally_journal.capture_legacy_spool_state(
|
|
6165
|
+
include_v1=cutover.revision is not None)
|
|
5238
6166
|
except _cctally_journal.LegacySpoolInventoryIncomplete as failure:
|
|
5239
6167
|
return _cctally_journal.record_legacy_spool_inventory_incomplete(
|
|
5240
6168
|
failure) is not None
|
|
@@ -6804,6 +7732,10 @@ def _render_acknowledge(account_key: str, transition: _Transition, *,
|
|
|
6804
7732
|
# publishes a proven-empty or new-window reading, retains an
|
|
6805
7733
|
# unknown one.
|
|
6806
7734
|
return False
|
|
7735
|
+
if _receipt_work(transition, evaluation.members):
|
|
7736
|
+
# #797 spec 4.4 (D7c q3): a NOOP that leaves receipt work for
|
|
7737
|
+
# either earning path is the detached reducer's to complete.
|
|
7738
|
+
return False
|
|
6807
7739
|
last = evaluation.sequence
|
|
6808
7740
|
if last is None:
|
|
6809
7741
|
return True
|
|
@@ -6812,7 +7744,7 @@ def _render_acknowledge(account_key: str, transition: _Transition, *,
|
|
|
6812
7744
|
_write_transition(_replace_transition(
|
|
6813
7745
|
transition, account_key=account_key, binding=transition.binding,
|
|
6814
7746
|
projection=transition.control.db_projection, control=transition.control,
|
|
6815
|
-
acknowledged=last, coverage=proof))
|
|
7747
|
+
acknowledged=last, coverage=proof, stale_cache=transition.stale_cache))
|
|
6816
7748
|
_crash_point("after-render-acknowledgement")
|
|
6817
7749
|
_intake_compact(account_key, last)
|
|
6818
7750
|
return True
|
|
@@ -6907,6 +7839,11 @@ def _preliminary_decision() -> "_candidates.ReductionDecision | None":
|
|
|
6907
7839
|
# projection cannot classify is the detached reducer's to
|
|
6908
7840
|
# classify, under either gate (C4: the render reads no index).
|
|
6909
7841
|
return fork
|
|
7842
|
+
if _receipt_work(transition, active):
|
|
7843
|
+
# #797 spec 4.4 (D7c q3): the NOOP leaves receipt work, so the
|
|
7844
|
+
# preliminary NOOP exit must not swallow it: the detached reducer
|
|
7845
|
+
# completes the evaluation and earns.
|
|
7846
|
+
return fork
|
|
6910
7847
|
return None
|
|
6911
7848
|
|
|
6912
7849
|
|