cctally 1.116.1 → 1.117.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +57 -3
- package/bin/_cctally_cache.py +5720 -885
- package/bin/_cctally_claude_attribution.py +997 -0
- package/bin/_cctally_completion.py +101 -28
- package/bin/_cctally_core.py +235 -12
- package/bin/_cctally_dashboard.py +263 -17
- package/bin/_cctally_dashboard_conversation.py +107 -21
- package/bin/_cctally_dashboard_envelope.py +44 -0
- package/bin/_cctally_db.py +928 -80
- package/bin/_cctally_diagnosis_sources.py +78 -13
- package/bin/_cctally_doctor.py +211 -86
- package/bin/_cctally_five_hour.py +446 -22
- package/bin/_cctally_forecast.py +302 -6
- package/bin/_cctally_journal.py +2418 -158
- package/bin/_cctally_milestone_history.py +379 -91
- package/bin/_cctally_oauth_samples.py +185 -7
- package/bin/_cctally_parser.py +38 -0
- package/bin/_cctally_project.py +34 -8
- package/bin/_cctally_quarantine.py +821 -343
- package/bin/_cctally_quota.py +69 -4
- package/bin/_cctally_quota_calibration.py +677 -3
- package/bin/_cctally_quota_model.py +927 -127
- package/bin/_cctally_record.py +922 -445
- package/bin/_cctally_rederive.py +119 -15
- package/bin/_cctally_statusline.py +1048 -176
- package/bin/_cctally_store.py +544 -110
- package/bin/_cctally_tui.py +455 -42
- package/bin/_cctally_update.py +77 -14
- package/bin/_lib_account_protocol.py +227 -19
- package/bin/_lib_claude_attribution.py +421 -0
- package/bin/_lib_clone_copy.py +30 -0
- package/bin/_lib_codex_conversation_query.py +45 -9
- package/bin/_lib_codex_conversation_watch.py +72 -13
- package/bin/_lib_codex_rollout_container.py +234 -19
- package/bin/_lib_codex_worker_lifecycle.py +374 -0
- package/bin/_lib_credit.py +22 -13
- package/bin/_lib_dashboard_sources.py +257 -18
- package/bin/_lib_detector_access_registry.py +1335 -166
- package/bin/_lib_doctor.py +276 -92
- package/bin/_lib_ingest_frontier.py +269 -36
- package/bin/_lib_journal.py +273 -22
- package/bin/_lib_jsonl.py +12 -0
- package/bin/_lib_meter_rate_change.py +72 -0
- package/bin/_lib_outcome_quarantine.py +917 -554
- package/bin/_lib_provider_activity.py +24 -6
- package/bin/_lib_quota_calibration.py +57 -3
- package/bin/_lib_quota_copy.py +7 -2
- package/bin/_lib_quota_lineage.py +613 -0
- package/bin/_lib_quota_model.py +464 -33
- package/bin/_lib_rate_change_delivery.py +16 -0
- package/bin/_lib_reclaim_planner.py +6 -4
- package/bin/_lib_rederive.py +2046 -21
- package/bin/_lib_selector_state.py +131 -3
- package/bin/_lib_snapshot_cache.py +8 -0
- package/bin/_lib_stale_proof.py +113 -4
- package/bin/_lib_statusline.py +72 -3
- package/bin/_lib_statusline_candidates.py +246 -42
- package/bin/_lib_tick_stats.py +10 -5
- package/bin/_lib_write_io.py +419 -6
- package/bin/cctally +28 -11
- package/dashboard/static/assets/{ConversationsView-Dflb_myA.js → ConversationsView-oqNsWzWV.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-DFX5CfIs.js → DoctorModal-B0DnU9d9.js} +1 -1
- package/dashboard/static/assets/ModalRoot-CYua2wP_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-C3PJy1xx.js → ProjectsDrillPanel-DSxXQSa-.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-BczsWKhD.js → SourceDetailModal-BleDpQqG.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-Os-Lo5cs.js → UpdateModal-CMqaAkSK.js} +1 -1
- package/dashboard/static/assets/{index-CYZ3NLqF.js → index-BjQV-0-D.js} +3 -3
- package/dashboard/static/assets/index-D34eYhAc.css +1 -0
- package/dashboard/static/assets/{outlineNavigation-k-zIif6W.js → outlineNavigation-CO-VWBm7.js} +1 -1
- package/dashboard/static/assets/{useKeymap-CW7Up0Ay.js → useKeymap-CUgEnnua.js} +1 -1
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +5 -1
- package/dashboard/static/assets/ModalRoot-HlxoagIa.js +0 -1
- package/dashboard/static/assets/index-BKITk1CT.css +0 -1
|
@@ -22,10 +22,12 @@ SCHEMA_VERSION = 1
|
|
|
22
22
|
#: The control document, versioned independently (#755). It is written by
|
|
23
23
|
#: whichever session holds the persist lock and read by the next tick, so a
|
|
24
24
|
#: shape change here must not invalidate the spool documents peers are writing.
|
|
25
|
-
CONTROL_SCHEMA_VERSION =
|
|
25
|
+
CONTROL_SCHEMA_VERSION = 3
|
|
26
26
|
#: Control documents this binary can read. Version 1 predates the pending
|
|
27
|
-
#: drop's deadline and retained evidence; see `_pending_drop`.
|
|
28
|
-
|
|
27
|
+
#: drop's deadline and retained evidence; see `_pending_drop`. Version 2
|
|
28
|
+
#: predates the retained observation's witness (#1019/#1014 spec 4.2): its
|
|
29
|
+
#: observations read with ``witness`` None.
|
|
30
|
+
CONTROL_SCHEMA_VERSIONS_READ = (1, 2, 3)
|
|
29
31
|
#: #825/#832: the account-bound protocol generation. Every current document,
|
|
30
32
|
#: retry namespace and plan names it; the legacy spool never carries it.
|
|
31
33
|
PROTOCOL_GENERATION = "account-bound-v1"
|
|
@@ -202,6 +204,11 @@ class SupportingObservation:
|
|
|
202
204
|
percent: float
|
|
203
205
|
raw_resets_at: int
|
|
204
206
|
received_at: int
|
|
207
|
+
#: #1019/#1014 (spec 4.2): the provider-activity witness the
|
|
208
|
+
#: contributor's candidate carried when this observation was retained
|
|
209
|
+
#: (`_lib_provider_activity.ActivityWitness`), or None when it carried
|
|
210
|
+
#: none (or the record predates control schema 3).
|
|
211
|
+
witness: object | None = None
|
|
205
212
|
|
|
206
213
|
|
|
207
214
|
@dataclasses.dataclass(frozen=True)
|
|
@@ -507,11 +514,14 @@ def _retry_signature(value: object) -> RetrySignature | None:
|
|
|
507
514
|
)
|
|
508
515
|
|
|
509
516
|
|
|
510
|
-
def _supporting_observation(
|
|
517
|
+
def _supporting_observation(
|
|
518
|
+
value: object, *, now_epoch: int, control_version: int = CONTROL_SCHEMA_VERSION
|
|
519
|
+
) -> SupportingObservation:
|
|
511
520
|
doc = _require_object(value, "supporting observation")
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
521
|
+
required = {"observationId", "token", "percent", "rawResetsAt", "receivedAt"}
|
|
522
|
+
if control_version >= 3:
|
|
523
|
+
required |= {"witness"}
|
|
524
|
+
_require_exact_keys(doc, required)
|
|
515
525
|
if not isinstance(doc["observationId"], str) or not OBSERVATION_ID_RE.fullmatch(
|
|
516
526
|
doc["observationId"]
|
|
517
527
|
):
|
|
@@ -523,16 +533,20 @@ def _supporting_observation(value: object, *, now_epoch: int) -> SupportingObser
|
|
|
523
533
|
received = doc["receivedAt"]
|
|
524
534
|
if not _is_int(received) or not 0 <= received <= now_epoch + 5:
|
|
525
535
|
raise StateValidationError("invalid supporting receivedAt")
|
|
536
|
+
witness = doc.get("witness")
|
|
526
537
|
return SupportingObservation(
|
|
527
538
|
observation_id=doc["observationId"],
|
|
528
539
|
token=doc["token"],
|
|
529
540
|
percent=_percent(doc["percent"]),
|
|
530
541
|
raw_resets_at=doc["rawResetsAt"],
|
|
531
542
|
received_at=received,
|
|
543
|
+
witness=None if witness is None else validate_activity(witness),
|
|
532
544
|
)
|
|
533
545
|
|
|
534
546
|
|
|
535
|
-
def _retained_low(
|
|
547
|
+
def _retained_low(
|
|
548
|
+
value: object, *, now_epoch: int, control_version: int = CONTROL_SCHEMA_VERSION
|
|
549
|
+
) -> RetainedLow | None:
|
|
536
550
|
if value is None:
|
|
537
551
|
return None
|
|
538
552
|
doc = _require_object(value, "retained low")
|
|
@@ -548,7 +562,9 @@ def _retained_low(value: object, *, now_epoch: int) -> RetainedLow | None:
|
|
|
548
562
|
raw_resets_at=doc["rawResetsAt"],
|
|
549
563
|
canonical_key=doc["canonicalKey"],
|
|
550
564
|
observations=tuple(
|
|
551
|
-
_supporting_observation(
|
|
565
|
+
_supporting_observation(
|
|
566
|
+
item, now_epoch=now_epoch, control_version=control_version)
|
|
567
|
+
for item in observations
|
|
552
568
|
),
|
|
553
569
|
)
|
|
554
570
|
|
|
@@ -611,7 +627,8 @@ def _pending_drop(
|
|
|
611
627
|
deadline_at = doc["deadlineAt"]
|
|
612
628
|
if not _is_int(deadline_at) or deadline_at < doc["firstSeenAt"]:
|
|
613
629
|
raise StateValidationError("invalid deadlineAt")
|
|
614
|
-
retained = _retained_low(doc["retained"], now_epoch=now_epoch
|
|
630
|
+
retained = _retained_low(doc["retained"], now_epoch=now_epoch,
|
|
631
|
+
control_version=control_version)
|
|
615
632
|
return PendingDrop(
|
|
616
633
|
canonical_key=doc["canonicalKey"],
|
|
617
634
|
reduced_percent=reduced_percent,
|
|
@@ -890,10 +907,50 @@ def _supporting_observations(
|
|
|
890
907
|
percent=value.percent,
|
|
891
908
|
raw_resets_at=value.raw_resets_at,
|
|
892
909
|
received_at=candidate.received_at,
|
|
910
|
+
witness=candidate.activity,
|
|
893
911
|
)
|
|
894
912
|
return tuple(sorted(seen.values(), key=lambda item: item.observation_id))
|
|
895
913
|
|
|
896
914
|
|
|
915
|
+
def _witness_json(witness) -> str:
|
|
916
|
+
"""The canonical JSON of a retained observation's witness (``null`` for
|
|
917
|
+
none): the tie-break of every ordering of supports (#1019/#1014 spec
|
|
918
|
+
4.3) and the digest input of `support_identity`."""
|
|
919
|
+
document = (None if witness is None
|
|
920
|
+
else _lib_provider_activity.witness_to_json(witness))
|
|
921
|
+
return json.dumps(document, sort_keys=True, separators=(",", ":"), allow_nan=False)
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
def support_identity(obs: SupportingObservation) -> tuple[str, str]:
|
|
925
|
+
"""The proof-support identity of one retained observation (#1019/#1014
|
|
926
|
+
spec 4.2, risk K3): its confirmation identity ``observation_id`` plus
|
|
927
|
+
the SHA-256 of its frozen witness's canonical JSON, or ``""`` for no
|
|
928
|
+
witness. Two captures sharing token, value and window but carrying
|
|
929
|
+
different witnesses are separate supports, each refuted on its own
|
|
930
|
+
bound; the confirmation identity alone still decides what is the same
|
|
931
|
+
upstream reading."""
|
|
932
|
+
if obs.witness is None:
|
|
933
|
+
return obs.observation_id, ""
|
|
934
|
+
return obs.observation_id, hashlib.sha256(
|
|
935
|
+
_witness_json(obs.witness).encode("utf-8")).hexdigest()
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
def _support_order(item: SupportingObservation) -> tuple[str, str]:
|
|
939
|
+
return item.observation_id, _witness_json(item.witness)
|
|
940
|
+
|
|
941
|
+
|
|
942
|
+
def _support_bound_end(item: SupportingObservation) -> float:
|
|
943
|
+
"""The end of a retained support's drop-bound interval in nanoseconds
|
|
944
|
+
(#1019 spec 4.1: its witness's supported instant, else its own
|
|
945
|
+
``received_at``); infinite when it has no usable bound, which no sample
|
|
946
|
+
can refute."""
|
|
947
|
+
import _lib_stale_proof
|
|
948
|
+
|
|
949
|
+
end = _lib_stale_proof.bound_end(
|
|
950
|
+
_lib_stale_proof.witness_bound(item.witness, item.received_at))
|
|
951
|
+
return math.inf if end is None else float(end)
|
|
952
|
+
|
|
953
|
+
|
|
897
954
|
def _merged_retained(
|
|
898
955
|
previous: RetainedLow | None,
|
|
899
956
|
*,
|
|
@@ -908,16 +965,37 @@ def _merged_retained(
|
|
|
908
965
|
second confirmation. A different low replaces the record outright, which
|
|
909
966
|
is how an upward correction moves the target without restarting anything
|
|
910
967
|
else.
|
|
968
|
+
|
|
969
|
+
#1019/#1014 (spec 4.2, K3): "already retained" is decided on the
|
|
970
|
+
proof-support identity (`support_identity`). A repeated observation
|
|
971
|
+
identity carrying a different witness is a separate support, judged on
|
|
972
|
+
its own bound, so a fresh same-value capture is never discarded in
|
|
973
|
+
favour of an older one a sample can refute. Of the supports sharing one
|
|
974
|
+
observation identity only the one with the LATEST drop bound
|
|
975
|
+
(`_lib_stale_proof.witness_bound`) is kept: identity fixes token, value
|
|
976
|
+
and raw reset, so every sample refuting it refutes the others too, and
|
|
977
|
+
keeping them would change no decision while growing the record by one
|
|
978
|
+
support per response at an unchanged reading (risk K3; the control
|
|
979
|
+
caps would refuse, not bound, the document). Equal bounds keep the
|
|
980
|
+
support already retained.
|
|
911
981
|
"""
|
|
912
982
|
merged: dict[str, SupportingObservation] = {}
|
|
983
|
+
|
|
984
|
+
def offer(item: SupportingObservation) -> None:
|
|
985
|
+
current = merged.get(item.observation_id)
|
|
986
|
+
if current is None or (support_identity(current) != support_identity(item)
|
|
987
|
+
and _support_bound_end(item) > _support_bound_end(current)):
|
|
988
|
+
merged[item.observation_id] = item
|
|
989
|
+
|
|
913
990
|
if previous is not None and previous.canonical_key == canonical_key and previous.percent == low:
|
|
914
|
-
|
|
991
|
+
for item in previous.observations:
|
|
992
|
+
offer(item)
|
|
915
993
|
for item in observations:
|
|
916
|
-
#
|
|
917
|
-
# upstream reading seen again
|
|
994
|
+
# An identity already retained with the same witness is the same
|
|
995
|
+
# upstream reading seen again: rewriting its receipt time would make
|
|
918
996
|
# the control document differ on every tick while nothing was confirmed.
|
|
919
|
-
|
|
920
|
-
ordered = tuple(sorted(merged.values(), key=
|
|
997
|
+
offer(item)
|
|
998
|
+
ordered = tuple(sorted(merged.values(), key=_support_order))
|
|
921
999
|
return RetainedLow(
|
|
922
1000
|
percent=low,
|
|
923
1001
|
raw_resets_at=min(item.raw_resets_at for item in ordered),
|
|
@@ -926,6 +1004,47 @@ def _merged_retained(
|
|
|
926
1004
|
)
|
|
927
1005
|
|
|
928
1006
|
|
|
1007
|
+
def _prune_retained(pending: PendingDrop | None,
|
|
1008
|
+
refuted: "frozenset | None") -> PendingDrop | None:
|
|
1009
|
+
"""``pending`` without the retained supports the current samples refute
|
|
1010
|
+
(#1019/#1014 spec 4.2), whether their contributors are present or
|
|
1011
|
+
absent. ``refuted`` holds `support_identity` values. A low that keeps
|
|
1012
|
+
some support keeps its value, its deadline and every other field; one
|
|
1013
|
+
left with none supports nothing (`_has_evidence`), which every reader of
|
|
1014
|
+
a pending record already treats as cancelled."""
|
|
1015
|
+
if pending is None or pending.retained is None or not refuted:
|
|
1016
|
+
return pending
|
|
1017
|
+
kept = tuple(item for item in pending.retained.observations
|
|
1018
|
+
if support_identity(item) not in refuted)
|
|
1019
|
+
if len(kept) == len(pending.retained.observations):
|
|
1020
|
+
return pending
|
|
1021
|
+
raw = (min(item.raw_resets_at for item in kept) if kept
|
|
1022
|
+
else pending.retained.raw_resets_at)
|
|
1023
|
+
return dataclasses.replace(pending, retained=dataclasses.replace(
|
|
1024
|
+
pending.retained, raw_resets_at=raw, observations=kept))
|
|
1025
|
+
|
|
1026
|
+
|
|
1027
|
+
def _drop_side(
|
|
1028
|
+
reduced: AxisValue,
|
|
1029
|
+
in_window: tuple[tuple[Candidate, AxisValue], ...],
|
|
1030
|
+
drop_window: tuple[tuple[Candidate, AxisValue], ...],
|
|
1031
|
+
db_axis: AxisProjection | None,
|
|
1032
|
+
) -> AxisValue | None:
|
|
1033
|
+
"""The drop-side maximum (#1019 spec 4.2): the same-window maximum over
|
|
1034
|
+
the drop-eligible members, or None when there are none. Without a
|
|
1035
|
+
drop-refused member it is ``reduced`` itself."""
|
|
1036
|
+
if len(drop_window) == len(in_window):
|
|
1037
|
+
return reduced
|
|
1038
|
+
if not drop_window:
|
|
1039
|
+
return None
|
|
1040
|
+
maximum = max(value.percent for _, value in drop_window)
|
|
1041
|
+
if db_axis is not None and db_axis.canonical_key == reduced.canonical_key:
|
|
1042
|
+
raw = db_axis.raw_resets_at
|
|
1043
|
+
else:
|
|
1044
|
+
raw = min(value.raw_resets_at for _, value in drop_window if value.percent == maximum)
|
|
1045
|
+
return AxisValue(maximum, raw, canonical_key=reduced.canonical_key)
|
|
1046
|
+
|
|
1047
|
+
|
|
929
1048
|
def _retained_axis_value(retained: RetainedLow, db_axis: AxisProjection | None) -> AxisValue:
|
|
930
1049
|
"""The value a pending drop publishes, using the DB's own raw reset when it
|
|
931
1050
|
describes the same physical window — the rule `_reduced_candidate` applies."""
|
|
@@ -1136,13 +1255,44 @@ def _reduce_axis(
|
|
|
1136
1255
|
drop_allowed: bool = True,
|
|
1137
1256
|
evidence: AxisEvidence | None = None,
|
|
1138
1257
|
member_facts: Mapping[str, object] | None = None,
|
|
1258
|
+
refuted_supports: "frozenset | None" = None,
|
|
1259
|
+
) -> _AxisDecision:
|
|
1260
|
+
# #1019/#1014 (spec 4.2): retained support the current samples refute is
|
|
1261
|
+
# pruned observation by observation BEFORE the reduction and before
|
|
1262
|
+
# `_hold_pending`, so no branch can publish it. A pruned record that the
|
|
1263
|
+
# reduction would otherwise leave untouched is still a control change.
|
|
1264
|
+
pruned = _prune_retained(pending, refuted_supports)
|
|
1265
|
+
decided = _reduce_pruned_axis(
|
|
1266
|
+
axis, candidates, db_axis, pruned, tombstone, now_epoch, scope=scope,
|
|
1267
|
+
drop_allowed=drop_allowed, evidence=evidence, member_facts=member_facts)
|
|
1268
|
+
if (pruned is not pending and decided.action == "NOOP"
|
|
1269
|
+
and decided.pending != pending):
|
|
1270
|
+
return dataclasses.replace(decided, action="WRITE_CONTROL")
|
|
1271
|
+
return decided
|
|
1272
|
+
|
|
1273
|
+
|
|
1274
|
+
def _reduce_pruned_axis(
|
|
1275
|
+
axis: str,
|
|
1276
|
+
candidates: tuple[Candidate, ...],
|
|
1277
|
+
db_axis: AxisProjection | None,
|
|
1278
|
+
pending: PendingDrop | None,
|
|
1279
|
+
tombstone: Tombstone | None,
|
|
1280
|
+
now_epoch: int,
|
|
1281
|
+
*,
|
|
1282
|
+
scope=None,
|
|
1283
|
+
drop_allowed: bool = True,
|
|
1284
|
+
evidence: AxisEvidence | None = None,
|
|
1285
|
+
member_facts: Mapping[str, object] | None = None,
|
|
1139
1286
|
) -> _AxisDecision:
|
|
1140
1287
|
# #797 spec 4.4 (Q3): the per-member refusal facts
|
|
1141
1288
|
# (`_lib_stale_proof.MemberFacts`, keyed by contributor token) are
|
|
1142
1289
|
# resolved before the kernel. None, or no refused member, reduces
|
|
1143
1290
|
# exactly as before.
|
|
1144
|
-
refused
|
|
1145
|
-
|
|
1291
|
+
# #1019 (spec 4.2, P-R2): the drop-refused members, decided
|
|
1292
|
+
# independently, are ineligible in BOTH directions; ``refused`` is the
|
|
1293
|
+
# rise-ineligible set (receipt-refused or drop-refused).
|
|
1294
|
+
drop_refused = drop_refused_tokens(member_facts)
|
|
1295
|
+
refused = rise_ineligible_tokens(member_facts)
|
|
1146
1296
|
reduced_data = _reduced_candidate(
|
|
1147
1297
|
candidates, axis=axis, tombstone=tombstone, db_axis=db_axis
|
|
1148
1298
|
)
|
|
@@ -1162,7 +1312,9 @@ def _reduce_axis(
|
|
|
1162
1312
|
# stale and is never relabelled as post-floor. The reading opens
|
|
1163
1313
|
# no consensus and spends no attempt, under either gate.
|
|
1164
1314
|
# #797 spec 4.4: a refused member is excluded from the post-floor
|
|
1165
|
-
# set; every other member keeps the receipt-floor test.
|
|
1315
|
+
# set; every other member keeps the receipt-floor test. #1019
|
|
1316
|
+
# (P-R4): so is a drop-refused one — ``refused`` is the
|
|
1317
|
+
# rise-ineligible set.
|
|
1166
1318
|
post = tuple((candidate, value) for candidate, value in in_window
|
|
1167
1319
|
if candidate.token not in refused
|
|
1168
1320
|
and candidate.received_at >= int(evidence.floor_epoch))
|
|
@@ -1212,13 +1364,20 @@ def _reduce_axis(
|
|
|
1212
1364
|
# evidence that a drop is pending, which is why the low is read separately
|
|
1213
1365
|
# here and why a rise is a publication rather than a branch of its own.
|
|
1214
1366
|
# #797 spec 4.4 (Q3): `rising` and the published value come from the
|
|
1215
|
-
# members that are not refused;
|
|
1216
|
-
#
|
|
1217
|
-
#
|
|
1218
|
-
#
|
|
1367
|
+
# members that are not refused; supporter retraction, deadlines and
|
|
1368
|
+
# attempt accounting keep the unfiltered set. The eligible value applies
|
|
1369
|
+
# at every rising return (plain, armed, cancelled and reconciled).
|
|
1370
|
+
# #1019 (spec 4.2): the drop side — the low that arms, its supporters,
|
|
1371
|
+
# pending membership, `reduced_percent`, unanimity and the unanimous
|
|
1372
|
+
# path's published maximum — uses the drop-eligible members only. An
|
|
1373
|
+
# empty drop-eligible set arms nothing and is never unanimous.
|
|
1219
1374
|
rise = _eligible_rise(reduced, in_window, refused, db_axis)
|
|
1220
1375
|
rising = rise is not None and rise.percent > db_axis.percent
|
|
1221
|
-
|
|
1376
|
+
drop_window = (in_window if not drop_refused else tuple(
|
|
1377
|
+
(candidate, value) for candidate, value in in_window
|
|
1378
|
+
if candidate.token not in drop_refused))
|
|
1379
|
+
drop_reduced = _drop_side(reduced, in_window, drop_window, db_axis)
|
|
1380
|
+
low = min((value.percent for _, value in drop_window), default=None)
|
|
1222
1381
|
armed = (
|
|
1223
1382
|
pending is not None
|
|
1224
1383
|
and pending.canonical_key == key
|
|
@@ -1230,16 +1389,17 @@ def _reduce_axis(
|
|
|
1230
1389
|
# The rise moves the baseline and no drop is pending against the old
|
|
1231
1390
|
# one, so there is nothing to preserve.
|
|
1232
1391
|
return _AxisDecision("PUBLISH_DB", rise, None)
|
|
1233
|
-
if low >= db_axis.percent:
|
|
1392
|
+
if low is None or low >= db_axis.percent:
|
|
1234
1393
|
return _AxisDecision(
|
|
1235
1394
|
"WRITE_CONTROL" if pending is not None else "NOOP", None, None
|
|
1236
1395
|
)
|
|
1237
|
-
supporters = _supporting_observations(
|
|
1396
|
+
supporters = _supporting_observations(drop_window, axis, low)
|
|
1238
1397
|
retained = _merged_retained(
|
|
1239
1398
|
None, canonical_key=key, low=low, observations=supporters
|
|
1240
1399
|
)
|
|
1241
1400
|
return _AxisDecision(
|
|
1242
|
-
"WRITE_CONTROL", None,
|
|
1401
|
+
"WRITE_CONTROL", None,
|
|
1402
|
+
_new_pending(drop_reduced, drop_window, now_epoch, retained=retained)
|
|
1243
1403
|
)
|
|
1244
1404
|
|
|
1245
1405
|
# A cancelled or already-recorded drop stops being pending, and a rise still
|
|
@@ -1252,9 +1412,9 @@ def _reduce_axis(
|
|
|
1252
1412
|
if db_axis.percent <= pending.retained.percent:
|
|
1253
1413
|
return cancelled
|
|
1254
1414
|
|
|
1255
|
-
reconciled = _reconcile_pending(pending,
|
|
1256
|
-
if low < db_axis.percent:
|
|
1257
|
-
supporters = _supporting_observations(
|
|
1415
|
+
reconciled = _reconcile_pending(pending, drop_window)
|
|
1416
|
+
if low is not None and low < db_axis.percent:
|
|
1417
|
+
supporters = _supporting_observations(drop_window, axis, low)
|
|
1258
1418
|
retained = _merged_retained(
|
|
1259
1419
|
pending.retained, canonical_key=key, low=low, observations=supporters
|
|
1260
1420
|
)
|
|
@@ -1267,7 +1427,10 @@ def _reduce_axis(
|
|
|
1267
1427
|
# never restarts the generation: `first_seen_at`, `deadline_at` and every
|
|
1268
1428
|
# other contributor's progress are carried through unchanged.
|
|
1269
1429
|
updated = dataclasses.replace(
|
|
1270
|
-
reconciled,
|
|
1430
|
+
reconciled,
|
|
1431
|
+
reduced_percent=(reconciled.reduced_percent if drop_reduced is None
|
|
1432
|
+
else drop_reduced.percent),
|
|
1433
|
+
retained=retained,
|
|
1271
1434
|
)
|
|
1272
1435
|
|
|
1273
1436
|
if rising:
|
|
@@ -1280,8 +1443,16 @@ def _reduce_axis(
|
|
|
1280
1443
|
# a low armed below the old baseline is still a drop below the new one.
|
|
1281
1444
|
return _AxisDecision("PUBLISH_DB", rise, updated)
|
|
1282
1445
|
|
|
1446
|
+
# #1019 (spec 4.2 *Dissent is kept*, P-R1, P-R2): an active,
|
|
1447
|
+
# same-window rise-eligible member at or above the baseline dissents
|
|
1448
|
+
# from the drop; an empty rise-eligible set dissents from nothing. With
|
|
1449
|
+
# no refused member this is today's ``reduced < baseline`` test.
|
|
1450
|
+
dissent = any(value.percent >= db_axis.percent
|
|
1451
|
+
for candidate, value in in_window if candidate.token not in refused)
|
|
1283
1452
|
unanimous = (
|
|
1284
|
-
|
|
1453
|
+
drop_reduced is not None
|
|
1454
|
+
and drop_reduced.percent < db_axis.percent
|
|
1455
|
+
and not dissent
|
|
1285
1456
|
and bool(updated.contributors)
|
|
1286
1457
|
and all(item.satisfied for item in updated.contributors.values())
|
|
1287
1458
|
)
|
|
@@ -1291,9 +1462,9 @@ def _reduce_axis(
|
|
|
1291
1462
|
# is spent; the record stays pending with its evidence.
|
|
1292
1463
|
pass
|
|
1293
1464
|
elif unanimous:
|
|
1294
|
-
attempted = _pending_kernel_attempt(updated,
|
|
1465
|
+
attempted = _pending_kernel_attempt(updated, drop_reduced, db_axis, scope=scope)
|
|
1295
1466
|
if attempted is not None:
|
|
1296
|
-
return _AxisDecision("PUBLISH_DB",
|
|
1467
|
+
return _AxisDecision("PUBLISH_DB", drop_reduced, attempted)
|
|
1297
1468
|
elif now_epoch >= updated.deadline_at:
|
|
1298
1469
|
# D5: the first eligible evaluation after expiry publishes the retained
|
|
1299
1470
|
# supported low despite unsatisfied contributors. Never the
|
|
@@ -1322,6 +1493,23 @@ def _reduce_axis(
|
|
|
1322
1493
|
)
|
|
1323
1494
|
|
|
1324
1495
|
|
|
1496
|
+
def drop_refused_tokens(member_facts: Mapping[str, object] | None) -> frozenset:
|
|
1497
|
+
"""The contributor tokens whose LOW the facts refuse (#1019 spec 4.2)."""
|
|
1498
|
+
return frozenset(token for token, facts in (member_facts or {}).items()
|
|
1499
|
+
if getattr(facts, "drop_refused", False))
|
|
1500
|
+
|
|
1501
|
+
|
|
1502
|
+
def rise_ineligible_tokens(member_facts: Mapping[str, object] | None) -> frozenset:
|
|
1503
|
+
"""The contributor tokens that may not RISE (#797 spec 4.4, #1019 spec
|
|
1504
|
+
4.2 P-R2): receipt-refused (``refused``) or drop-refused. A drop-refused
|
|
1505
|
+
member's reading predates an accounted provider sample reading more than
|
|
1506
|
+
1 pp higher, so its rise would re-raise a credited account to a
|
|
1507
|
+
pre-credit reading. The reducer, plan construction and frozen-plan
|
|
1508
|
+
recovery all read rise eligibility here."""
|
|
1509
|
+
return frozenset(token for token, facts in (member_facts or {}).items()
|
|
1510
|
+
if facts.refused or getattr(facts, "drop_refused", False))
|
|
1511
|
+
|
|
1512
|
+
|
|
1325
1513
|
def _eligible_rise(
|
|
1326
1514
|
reduced: AxisValue,
|
|
1327
1515
|
in_window: tuple[tuple[Candidate, AxisValue], ...],
|
|
@@ -1349,8 +1537,9 @@ def reduce_candidates(
|
|
|
1349
1537
|
tombstones: Mapping[str, Tombstone | None],
|
|
1350
1538
|
now_epoch: int,
|
|
1351
1539
|
account_key: str | None = None,
|
|
1352
|
-
drop_consensus_allowed: bool = True,
|
|
1540
|
+
drop_consensus_allowed: "bool | Mapping[str, bool]" = True,
|
|
1353
1541
|
member_facts: Mapping[str, Mapping[str, object]] | None = None,
|
|
1542
|
+
refuted_supports: "Mapping[str, frozenset[tuple[str, str]]] | None" = None,
|
|
1354
1543
|
) -> ReductionDecision:
|
|
1355
1544
|
"""Reduce active candidates independently for 5h and 7d axes.
|
|
1356
1545
|
|
|
@@ -1362,11 +1551,12 @@ def reduce_candidates(
|
|
|
1362
1551
|
``(PROTOCOL_GENERATION, account_key, axis)``. The caller passes only that
|
|
1363
1552
|
partition's candidates, control and tombstones.
|
|
1364
1553
|
|
|
1365
|
-
``drop_consensus_allowed`` is the outcome-quarantine
|
|
1366
|
-
(#825/#832 spec 5.6 item 7)
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1554
|
+
``drop_consensus_allowed`` is the outcome-quarantine drop gate
|
|
1555
|
+
(#825/#832 spec 5.6 item 7), one boolean for both axes or, since #948
|
|
1556
|
+
spec 6.8, a mapping ``{axis: bool}`` (a missing axis is withheld). False
|
|
1557
|
+
while the axis is paused: a pending drop below the database baseline is
|
|
1558
|
+
then published neither by consensus nor at its deadline, and stays
|
|
1559
|
+
pending; rises and new windows publish as always. An axis with no projected baseline publishes
|
|
1370
1560
|
under either gate only when ``db`` carries classification evidence for
|
|
1371
1561
|
it (`classify_absent_axis`: a first reading, a new window or the first
|
|
1372
1562
|
reading after a retiring floor); a stale reading takes the older-window
|
|
@@ -1375,9 +1565,21 @@ def reduce_candidates(
|
|
|
1375
1565
|
|
|
1376
1566
|
``member_facts`` maps an axis to its per-member refusal facts (#797 spec
|
|
1377
1567
|
4.4, `_lib_stale_proof.MemberFacts` by contributor token). None, or an
|
|
1378
|
-
axis without facts, means every member is eligible.
|
|
1568
|
+
axis without facts, means every member is eligible, in both directions.
|
|
1569
|
+
|
|
1570
|
+
``refuted_supports`` maps an axis (``"sevenDay"``, ``"fiveHour"``) to the
|
|
1571
|
+
`support_identity` values of the retained supports the current samples
|
|
1572
|
+
refute (#1019/#1014 spec 4.2); they are pruned before the axis is
|
|
1573
|
+
reduced. None, or an axis without an entry, refutes nothing.
|
|
1379
1574
|
"""
|
|
1380
1575
|
facts = member_facts or {}
|
|
1576
|
+
refuted = refuted_supports or {}
|
|
1577
|
+
if isinstance(drop_consensus_allowed, bool):
|
|
1578
|
+
drop_gate = {"fiveHour": drop_consensus_allowed,
|
|
1579
|
+
"sevenDay": drop_consensus_allowed}
|
|
1580
|
+
else:
|
|
1581
|
+
drop_gate = {axis: drop_consensus_allowed.get(axis) is True
|
|
1582
|
+
for axis in ("fiveHour", "sevenDay")}
|
|
1381
1583
|
active = tuple(
|
|
1382
1584
|
candidate
|
|
1383
1585
|
for candidate in candidates
|
|
@@ -1391,9 +1593,10 @@ def reduce_candidates(
|
|
|
1391
1593
|
tombstones.get("fiveHour"),
|
|
1392
1594
|
now_epoch,
|
|
1393
1595
|
scope=None if account_key is None else (PROTOCOL_GENERATION, account_key, "fiveHour"),
|
|
1394
|
-
drop_allowed=
|
|
1596
|
+
drop_allowed=drop_gate["fiveHour"],
|
|
1395
1597
|
evidence=axis_evidence(db, "fiveHour"),
|
|
1396
1598
|
member_facts=facts.get("fiveHour"),
|
|
1599
|
+
refuted_supports=refuted.get("fiveHour"),
|
|
1397
1600
|
)
|
|
1398
1601
|
seven = _reduce_axis(
|
|
1399
1602
|
"sevenDay",
|
|
@@ -1403,9 +1606,10 @@ def reduce_candidates(
|
|
|
1403
1606
|
tombstones.get("sevenDay"),
|
|
1404
1607
|
now_epoch,
|
|
1405
1608
|
scope=None if account_key is None else (PROTOCOL_GENERATION, account_key, "sevenDay"),
|
|
1406
|
-
drop_allowed=
|
|
1609
|
+
drop_allowed=drop_gate["sevenDay"],
|
|
1407
1610
|
evidence=axis_evidence(db, "sevenDay"),
|
|
1408
1611
|
member_facts=facts.get("sevenDay"),
|
|
1612
|
+
refuted_supports=refuted.get("sevenDay"),
|
|
1409
1613
|
)
|
|
1410
1614
|
next_control = ControlState(
|
|
1411
1615
|
db_projection=db,
|
package/bin/_lib_tick_stats.py
CHANGED
|
@@ -35,12 +35,17 @@ import types
|
|
|
35
35
|
import _lib_write_io
|
|
36
36
|
|
|
37
37
|
RING_CAPACITY = 64
|
|
38
|
-
|
|
38
|
+
#: Raised from 65,536 for the v1.117.1 cut: the #939 D5 admission-hold
|
|
39
|
+
#: state put the full rings plus the write state at 65,032 bytes on
|
|
40
|
+
#: Python 3.12 and 3.13 but 66,000 on 3.11, whose objects are larger.
|
|
41
|
+
#: The bound stays structural (ring capacities, PUBLISHED_AT_MAX_CHARS);
|
|
42
|
+
#: this figure only checks it on every supported interpreter.
|
|
43
|
+
MEMORY_BUDGET_BYTES = 73728
|
|
39
44
|
|
|
40
45
|
#: `published_at` is the only variable-length field in a record, so it is the
|
|
41
46
|
#: only one that can put the whole state over `MEMORY_BUDGET_BYTES`. Measured:
|
|
42
47
|
#: the stored field set is 27,189 bytes over a full ring, and 64 distinct 4 KiB
|
|
43
|
-
#: strings would be 283,039 —
|
|
48
|
+
#: strings would be 283,039 — 3.8x over budget. All three production callers
|
|
44
49
|
#: pass a UTC ISO-8601 instant (32 characters), so this is not reachable today;
|
|
45
50
|
#: the cap makes the budget hold by CONSTRUCTION rather than by convention.
|
|
46
51
|
PUBLISHED_AT_MAX_CHARS = 64
|
|
@@ -154,7 +159,7 @@ class ConversationSyncRecord:
|
|
|
154
159
|
MAINTENANCE_PHASES = ("delete", "reclaim", "checkpoint", "other")
|
|
155
160
|
|
|
156
161
|
#: The maintenance ring is deliberately SHORTER than the two tick rings. The
|
|
157
|
-
#: whole published state is capped at a frozen
|
|
162
|
+
#: whole published state is capped at a frozen 73,728 bytes, and a third
|
|
158
163
|
#: full-length ring would spend that budget on a phase that runs about once a
|
|
159
164
|
#: day rather than about once a second. Sixteen records is several weeks of
|
|
160
165
|
#: maintenance history at the throttled cadence, and many continuation passes
|
|
@@ -181,7 +186,7 @@ MAINTENANCE_SKIP_REASONS = (
|
|
|
181
186
|
SQLITE_SOURCE_ID_MAX_CHARS = 96
|
|
182
187
|
PLAN_DIGEST_MAX_CHARS = 16
|
|
183
188
|
#: One shared object per distinct SQLite source id, so sixteen records of
|
|
184
|
-
#: one runtime hold one string (the
|
|
189
|
+
#: one runtime hold one string (the 73,728-byte budget).
|
|
185
190
|
_SOURCE_IDS: "dict[str, str]" = {}
|
|
186
191
|
|
|
187
192
|
|
|
@@ -224,7 +229,7 @@ class MaintenancePhaseRecord:
|
|
|
224
229
|
# #901 Q8 (a deletion's reservation inputs) and Q9 (a reclaim chunk's
|
|
225
230
|
# plan and outcome, §5.5): `OPERATION_INT_FIELDS` packed into ONE
|
|
226
231
|
# array, -1 for "not recorded", because sixteen records of twenty
|
|
227
|
-
# separate int objects do not fit the frozen
|
|
232
|
+
# separate int objects do not fit the frozen 73,728-byte budget. Read
|
|
228
233
|
# them by name (`record.page_count`); `as_wire` publishes them by name.
|
|
229
234
|
numbers: "array.array | None" = None # typecode "i"
|
|
230
235
|
|