cctally 1.116.1 → 1.117.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +57 -3
- package/bin/_cctally_cache.py +5720 -885
- package/bin/_cctally_claude_attribution.py +997 -0
- package/bin/_cctally_completion.py +101 -28
- package/bin/_cctally_core.py +235 -12
- package/bin/_cctally_dashboard.py +263 -17
- package/bin/_cctally_dashboard_conversation.py +107 -21
- package/bin/_cctally_dashboard_envelope.py +44 -0
- package/bin/_cctally_db.py +928 -80
- package/bin/_cctally_diagnosis_sources.py +78 -13
- package/bin/_cctally_doctor.py +211 -86
- package/bin/_cctally_five_hour.py +446 -22
- package/bin/_cctally_forecast.py +302 -6
- package/bin/_cctally_journal.py +2418 -158
- package/bin/_cctally_milestone_history.py +379 -91
- package/bin/_cctally_oauth_samples.py +185 -7
- package/bin/_cctally_parser.py +38 -0
- package/bin/_cctally_project.py +34 -8
- package/bin/_cctally_quarantine.py +821 -343
- package/bin/_cctally_quota.py +69 -4
- package/bin/_cctally_quota_calibration.py +677 -3
- package/bin/_cctally_quota_model.py +927 -127
- package/bin/_cctally_record.py +922 -445
- package/bin/_cctally_rederive.py +119 -15
- package/bin/_cctally_statusline.py +1048 -176
- package/bin/_cctally_store.py +544 -110
- package/bin/_cctally_tui.py +455 -42
- package/bin/_cctally_update.py +77 -14
- package/bin/_lib_account_protocol.py +227 -19
- package/bin/_lib_claude_attribution.py +421 -0
- package/bin/_lib_clone_copy.py +30 -0
- package/bin/_lib_codex_conversation_query.py +45 -9
- package/bin/_lib_codex_conversation_watch.py +72 -13
- package/bin/_lib_codex_rollout_container.py +234 -19
- package/bin/_lib_codex_worker_lifecycle.py +374 -0
- package/bin/_lib_credit.py +22 -13
- package/bin/_lib_dashboard_sources.py +257 -18
- package/bin/_lib_detector_access_registry.py +1335 -166
- package/bin/_lib_doctor.py +276 -92
- package/bin/_lib_ingest_frontier.py +269 -36
- package/bin/_lib_journal.py +273 -22
- package/bin/_lib_jsonl.py +12 -0
- package/bin/_lib_meter_rate_change.py +72 -0
- package/bin/_lib_outcome_quarantine.py +917 -554
- package/bin/_lib_provider_activity.py +24 -6
- package/bin/_lib_quota_calibration.py +57 -3
- package/bin/_lib_quota_copy.py +7 -2
- package/bin/_lib_quota_lineage.py +613 -0
- package/bin/_lib_quota_model.py +464 -33
- package/bin/_lib_rate_change_delivery.py +16 -0
- package/bin/_lib_reclaim_planner.py +6 -4
- package/bin/_lib_rederive.py +2046 -21
- package/bin/_lib_selector_state.py +131 -3
- package/bin/_lib_snapshot_cache.py +8 -0
- package/bin/_lib_stale_proof.py +113 -4
- package/bin/_lib_statusline.py +72 -3
- package/bin/_lib_statusline_candidates.py +246 -42
- package/bin/_lib_tick_stats.py +10 -5
- package/bin/_lib_write_io.py +419 -6
- package/bin/cctally +28 -11
- package/dashboard/static/assets/{ConversationsView-Dflb_myA.js → ConversationsView-oqNsWzWV.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-DFX5CfIs.js → DoctorModal-B0DnU9d9.js} +1 -1
- package/dashboard/static/assets/ModalRoot-CYua2wP_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-C3PJy1xx.js → ProjectsDrillPanel-DSxXQSa-.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-BczsWKhD.js → SourceDetailModal-BleDpQqG.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-Os-Lo5cs.js → UpdateModal-CMqaAkSK.js} +1 -1
- package/dashboard/static/assets/{index-CYZ3NLqF.js → index-BjQV-0-D.js} +3 -3
- package/dashboard/static/assets/index-D34eYhAc.css +1 -0
- package/dashboard/static/assets/{outlineNavigation-k-zIif6W.js → outlineNavigation-CO-VWBm7.js} +1 -1
- package/dashboard/static/assets/{useKeymap-CW7Up0Ay.js → useKeymap-CUgEnnua.js} +1 -1
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +5 -1
- package/dashboard/static/assets/ModalRoot-HlxoagIa.js +0 -1
- package/dashboard/static/assets/index-BKITk1CT.css +0 -1
package/bin/_lib_rederive.py
CHANGED
|
@@ -12,14 +12,16 @@ No filesystem, SQLite, provider, alert, or journal operation belongs here.
|
|
|
12
12
|
|
|
13
13
|
from __future__ import annotations
|
|
14
14
|
|
|
15
|
+
import bisect
|
|
15
16
|
import dataclasses
|
|
16
17
|
import datetime as dt
|
|
17
18
|
import hashlib
|
|
18
19
|
import json
|
|
19
20
|
import math
|
|
20
|
-
from collections.abc import Iterable, Mapping
|
|
21
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
21
22
|
|
|
22
23
|
import _lib_accounts
|
|
24
|
+
import _lib_journal
|
|
23
25
|
from _lib_cost_provenance import (
|
|
24
26
|
HISTORICAL_CLOSE_BASIS, RECORDED_CLOSE_CORRECTION_BASIS,
|
|
25
27
|
)
|
|
@@ -82,6 +84,12 @@ _EVT_CLASSIFICATIONS = {
|
|
|
82
84
|
"rederived", "Claude weekly reset or credit decision"),
|
|
83
85
|
"five_hour_credit": KindClassification(
|
|
84
86
|
"rederived", "Claude five-hour credit decision"),
|
|
87
|
+
# #943 spec 4.2.2: the exact boundary companion of a five-hour credit.
|
|
88
|
+
"five_hour_credit_boundary": KindClassification(
|
|
89
|
+
"rederived", "Claude five-hour credit boundary"),
|
|
90
|
+
# #956 spec 4.6.1: every five-hour block creation's opening fact.
|
|
91
|
+
"five_hour_block_open": KindClassification(
|
|
92
|
+
"rederived", "Claude five-hour block opening"),
|
|
85
93
|
"five_hour_block_close": KindClassification(
|
|
86
94
|
"rederived", "closed Claude five-hour projection"),
|
|
87
95
|
"percent_milestone": KindClassification(
|
|
@@ -115,6 +123,12 @@ _EVT_CLASSIFICATIONS = {
|
|
|
115
123
|
"meter_rate_change": KindClassification(
|
|
116
124
|
"retained",
|
|
117
125
|
"the metering-rate latch is replayed, never re-derived"),
|
|
126
|
+
# #1015 spec 5.7: the terminal supersession of one metering-rate
|
|
127
|
+
# transition identity. `retained` like its latch: a self-sufficient
|
|
128
|
+
# payload replayed by its own natural-key fold, never re-derived.
|
|
129
|
+
"meter_rate_change_supersession": KindClassification(
|
|
130
|
+
"retained",
|
|
131
|
+
"a transition's supersession is replayed, never re-derived"),
|
|
118
132
|
}
|
|
119
133
|
|
|
120
134
|
_OP_CLASSIFICATIONS = {
|
|
@@ -145,6 +159,13 @@ _OP_CLASSIFICATIONS = {
|
|
|
145
159
|
"retained", "operator Codex window attribution is outside claude-usage"),
|
|
146
160
|
"codex_window_attribution_retract": KindClassification(
|
|
147
161
|
"retained", "operator Codex window retraction is outside claude-usage"),
|
|
162
|
+
# #944 spec 5.10: the durable Claude cost attribution decision. `retained`
|
|
163
|
+
# like `codex_file_account`: it is replayed by the Claude cache leg into
|
|
164
|
+
# `claude_cost_decisions`, and re-derivation reads the decided
|
|
165
|
+
# `session_entries.account_key` it materialized rather than deriving it.
|
|
166
|
+
"claude_cost_account": KindClassification(
|
|
167
|
+
"retained", "Claude cost attribution decisions are replayed by the "
|
|
168
|
+
"cache leg, never re-derived"),
|
|
148
169
|
# #825/#832 spec 5.1. The versioned reviewed decision is replayed into
|
|
149
170
|
# scratch exactly like its previous-release twin.
|
|
150
171
|
"claude_weekly_observation_decision_v2": KindClassification(
|
|
@@ -171,6 +192,25 @@ _OP_CLASSIFICATIONS = {
|
|
|
171
192
|
"re-derived"),
|
|
172
193
|
"journal_protocol_resolution_v3": KindClassification(
|
|
173
194
|
"retained", "journal-repair acknowledgements are audit records"),
|
|
195
|
+
# #1019/#1014 spec 4.7 C2, C5: the revision-3 boundary and the v4
|
|
196
|
+
# acknowledgement, likewise.
|
|
197
|
+
"account_bound_revision_v3": KindClassification(
|
|
198
|
+
"retained", "the revision-3 journal fence is replayed, never "
|
|
199
|
+
"re-derived"),
|
|
200
|
+
"journal_protocol_resolution_v4": KindClassification(
|
|
201
|
+
"retained", "journal-repair acknowledgements are audit records"),
|
|
202
|
+
# #948 spec 6.2: the outcome quarantine's recovery ops are preserved
|
|
203
|
+
# machinery. The fold replays them in journal order; nothing in the
|
|
204
|
+
# claude-usage family derives them.
|
|
205
|
+
"legacy_spool_inventory_completion": KindClassification(
|
|
206
|
+
"retained", "inventory-episode completions are replayed by the "
|
|
207
|
+
"quarantine fold, never re-derived"),
|
|
208
|
+
"outcome_authority_completion": KindClassification(
|
|
209
|
+
"retained", "authority completions are replayed by the quarantine "
|
|
210
|
+
"fold, never re-derived"),
|
|
211
|
+
"outcome_axis_release": KindClassification(
|
|
212
|
+
"retained", "axis releases are replayed by the quarantine fold, "
|
|
213
|
+
"never re-derived"),
|
|
174
214
|
}
|
|
175
215
|
|
|
176
216
|
|
|
@@ -472,6 +512,73 @@ def _with_close_marker(record: Mapping, marker: Mapping) -> dict:
|
|
|
472
512
|
return corrected
|
|
473
513
|
|
|
474
514
|
|
|
515
|
+
#: The non-``total_`` members of a close's money group (#1040 spec §5.2).
|
|
516
|
+
_CLOSE_MONEY_EXTRA_KEYS = frozenset({
|
|
517
|
+
"_models", "_projects", "_pricing", "pricing_provenance_json",
|
|
518
|
+
})
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def _close_money_keys(payload: Mapping) -> frozenset:
|
|
522
|
+
"""#1040 (spec §5.2): the keys of a close payload's money group.
|
|
523
|
+
|
|
524
|
+
Every key starting with ``total_`` plus the child sets and both pricing
|
|
525
|
+
provenance fields: the partition :func:`_preserve_non_derivable_state`
|
|
526
|
+
compares the closure's non-monetary structure without.
|
|
527
|
+
"""
|
|
528
|
+
return frozenset(
|
|
529
|
+
key for key in payload
|
|
530
|
+
if key in _CLOSE_MONEY_EXTRA_KEYS
|
|
531
|
+
or (isinstance(key, str) and key.startswith("total_"))
|
|
532
|
+
)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def close_money_overlay(desired: Mapping, current: Mapping) -> dict:
|
|
536
|
+
"""#1040 (spec §5.2): ``overlay(D, C)`` on whole records.
|
|
537
|
+
|
|
538
|
+
``desired`` with ``current``'s money group: every money key of D is
|
|
539
|
+
replaced by C's, and a money key C lacks is removed.
|
|
540
|
+
"""
|
|
541
|
+
desired_payload = desired.get("payload") or {}
|
|
542
|
+
current_payload = current.get("payload") or {}
|
|
543
|
+
drop = _close_money_keys(desired_payload)
|
|
544
|
+
payload = {key: value for key, value in desired_payload.items()
|
|
545
|
+
if key not in drop}
|
|
546
|
+
for key in _close_money_keys(current_payload):
|
|
547
|
+
payload[key] = current_payload[key]
|
|
548
|
+
return {**desired, "payload": payload}
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
_CLOSE_POPULATION_TOTALS = (
|
|
552
|
+
"total_input_tokens", "total_output_tokens", "total_cache_create_tokens",
|
|
553
|
+
"total_cache_read_tokens",
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def close_population(payload: Mapping) -> tuple:
|
|
558
|
+
"""#1040 (spec §5.2): the card-independent part of a close's money group.
|
|
559
|
+
|
|
560
|
+
The four block token totals, and per ``_models`` / ``_projects`` child its
|
|
561
|
+
model or project path, account key, four token counts and entry count,
|
|
562
|
+
order-independent. Compared exactly.
|
|
563
|
+
"""
|
|
564
|
+
def children(key: str, name: str) -> tuple:
|
|
565
|
+
rows = [
|
|
566
|
+
(child.get(name), child.get("account_key"),
|
|
567
|
+
*(child.get(token) for token in _BLOCK_TOKEN_KEYS),
|
|
568
|
+
child.get("entry_count"))
|
|
569
|
+
for child in payload.get(key) or ()
|
|
570
|
+
if isinstance(child, Mapping)
|
|
571
|
+
]
|
|
572
|
+
return tuple(sorted(rows, key=lambda row: json.dumps(
|
|
573
|
+
row, sort_keys=True, default=repr)))
|
|
574
|
+
|
|
575
|
+
return (
|
|
576
|
+
tuple(payload.get(key) for key in _CLOSE_POPULATION_TOTALS),
|
|
577
|
+
children("_models", "model"),
|
|
578
|
+
children("_projects", "project_path"),
|
|
579
|
+
)
|
|
580
|
+
|
|
581
|
+
|
|
475
582
|
def _preserve_non_derivable_state(
|
|
476
583
|
current_record: Mapping, desired_record: Mapping, *,
|
|
477
584
|
historical_marker: "Mapping | None" = None,
|
|
@@ -495,8 +602,10 @@ def _preserve_non_derivable_state(
|
|
|
495
602
|
marker is stamped with its own basis, ``RECORDED_CLOSE_CORRECTION_BASIS``.
|
|
496
603
|
|
|
497
604
|
#875: the milestones of a frozen or replaced close are decided by
|
|
498
|
-
:func:`_resolve_closed_block_milestones`, not here
|
|
499
|
-
|
|
605
|
+
:func:`_resolve_closed_block_milestones`, not here: an unchanged crossing
|
|
606
|
+
keeps its selected block cost there unless its lineage proves an earlier
|
|
607
|
+
rederive repriced it (#884). This function only carries a milestone's
|
|
608
|
+
``alerted_at`` latch.
|
|
500
609
|
"""
|
|
501
610
|
current_payload = current_record.get("payload") or {}
|
|
502
611
|
desired_payload = desired_record.get("payload") or {}
|
|
@@ -520,6 +629,30 @@ def _preserve_non_derivable_state(
|
|
|
520
629
|
|
|
521
630
|
if cost_structure(current_payload) == cost_structure(desired_payload):
|
|
522
631
|
return current_record
|
|
632
|
+
if (
|
|
633
|
+
current_payload.get("kind") == "five_hour_block_open"
|
|
634
|
+
and desired_payload.get("kind") == "five_hour_block_open"
|
|
635
|
+
and current_record.get("at") == desired_record.get("at")
|
|
636
|
+
):
|
|
637
|
+
# #956: the live writer stamps ``block_start_at`` in the host's zone,
|
|
638
|
+
# and a replay under another zone spells the same boundary with its
|
|
639
|
+
# own offset. An opening is immutable once written; a different
|
|
640
|
+
# spelling of the same instants is not a revision to supersede.
|
|
641
|
+
def opening_structure(payload):
|
|
642
|
+
facts = dict(payload)
|
|
643
|
+
for field in (
|
|
644
|
+
"block_start_at", "five_hour_resets_at",
|
|
645
|
+
"first_observed_at_utc", "created_at_utc",
|
|
646
|
+
"weekly_basis_captured_at_utc",
|
|
647
|
+
):
|
|
648
|
+
instant = _iso_instant(facts.get(field))
|
|
649
|
+
if instant is not None:
|
|
650
|
+
facts[field] = instant.isoformat()
|
|
651
|
+
return facts
|
|
652
|
+
|
|
653
|
+
if opening_structure(current_payload) == opening_structure(
|
|
654
|
+
desired_payload):
|
|
655
|
+
return current_record
|
|
523
656
|
if (
|
|
524
657
|
current_payload.get("kind") == "five_hour_block_close"
|
|
525
658
|
and desired_payload.get("kind") == "five_hour_block_close"
|
|
@@ -533,13 +666,10 @@ def _preserve_non_derivable_state(
|
|
|
533
666
|
# closure's non-monetary evidence first so a changed boundary or final
|
|
534
667
|
# reading still reaches the normal correction path.
|
|
535
668
|
def structure(payload):
|
|
669
|
+
money = _close_money_keys(payload)
|
|
536
670
|
facts = {
|
|
537
671
|
key: value for key, value in payload.items()
|
|
538
|
-
if not
|
|
539
|
-
and key not in {
|
|
540
|
-
"_models", "_projects", "_pricing",
|
|
541
|
-
"pricing_provenance_json",
|
|
542
|
-
}
|
|
672
|
+
if key not in money
|
|
543
673
|
}
|
|
544
674
|
# Live close used the display-zone offset; scratch replay writes
|
|
545
675
|
# UTC. Both name one frozen boundary and must compare as an instant.
|
|
@@ -605,9 +735,9 @@ def _preserve_non_derivable_state(
|
|
|
605
735
|
return keep_drifted()
|
|
606
736
|
# #869 V4: record that this close was CORRECTED. The basis is
|
|
607
737
|
# provenance only: since #875 every frozen close freezes the
|
|
608
|
-
# block cost of its unchanged five-hour crossings
|
|
609
|
-
# its basis. A
|
|
610
|
-
# otherwise.
|
|
738
|
+
# block cost of its unchanged five-hour crossings (save a
|
|
739
|
+
# #884 history-proven restoration), whatever its basis. A
|
|
740
|
+
# genuinely priced close's marker looks the same otherwise.
|
|
611
741
|
return _with_close_marker(desired_record, {
|
|
612
742
|
**desired_pricing, "basis": RECORDED_CLOSE_CORRECTION_BASIS,
|
|
613
743
|
})
|
|
@@ -726,10 +856,967 @@ def _revived_by_this_family(selected) -> bool:
|
|
|
726
856
|
)
|
|
727
857
|
|
|
728
858
|
|
|
859
|
+
# ── #943/#956 spec 4.4: five-hour credit companions and block parents ──────
|
|
860
|
+
#
|
|
861
|
+
# A five-hour credit (`five_hour_credit`, `fhc`) and its exact-boundary
|
|
862
|
+
# companion (`five_hour_credit_boundary`, `fhcb`) share one natural key, and
|
|
863
|
+
# the companion is read only while its `credit_revision` matches the credit,
|
|
864
|
+
# so a plan that acts on one without the other splits them. A five-hour
|
|
865
|
+
# milestone's `block_id` is a DERIVED foreign key, resolved at fold from its
|
|
866
|
+
# `(account_key, five_hour_window_key)`, so it depends on whatever
|
|
867
|
+
# materializes that window's block: the opening fact (`five_hour_block_open`,
|
|
868
|
+
# `fhbo`), the close fact (`five_hour_block_close`, `fhbc`), or the rebuild's
|
|
869
|
+
# legacy candidate rule for an unseeded open block.
|
|
870
|
+
|
|
871
|
+
_CREDIT_PAIR_KINDS = frozenset({"five_hour_credit", "five_hour_credit_boundary"})
|
|
872
|
+
_BLOCK_PARENT_KINDS = frozenset({"five_hour_block_open", "five_hour_block_close"})
|
|
873
|
+
_OPENING_ID_PREFIX = "fhbo:"
|
|
874
|
+
|
|
875
|
+
|
|
876
|
+
def _window_of(payload: Mapping):
|
|
877
|
+
"""``(account_key, int(five_hour_window_key))`` of a payload, or None."""
|
|
878
|
+
window = payload.get("five_hour_window_key")
|
|
879
|
+
if window is None or isinstance(window, bool):
|
|
880
|
+
return None
|
|
881
|
+
try:
|
|
882
|
+
return (payload.get("account_key"), int(window))
|
|
883
|
+
except (TypeError, ValueError):
|
|
884
|
+
return None
|
|
885
|
+
|
|
886
|
+
|
|
887
|
+
def _credit_pair_key(payload: Mapping) -> tuple:
|
|
888
|
+
"""The natural key a credit and its companion share."""
|
|
889
|
+
return (payload.get("account_key"), payload.get("five_hour_window_key"),
|
|
890
|
+
payload.get("effective_reset_at_utc"))
|
|
891
|
+
|
|
892
|
+
|
|
893
|
+
def _action_payload(action, current_events: Mapping[str, Mapping]) -> Mapping:
|
|
894
|
+
"""An action's payload; a tombstone's comes from its current record."""
|
|
895
|
+
if action.disposition == "tombstone":
|
|
896
|
+
return (current_events.get(action.event_id) or {}).get("payload") or {}
|
|
897
|
+
return action.payload or {}
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
def _opening_window(event_id: str):
|
|
901
|
+
"""The window of an `fhbo` id (``fhbo:<account_key>:<window_key>``)."""
|
|
902
|
+
if not event_id.startswith(_OPENING_ID_PREFIX):
|
|
903
|
+
return None
|
|
904
|
+
account_key, _sep, window = event_id[len(_OPENING_ID_PREFIX):].rpartition(":")
|
|
905
|
+
try:
|
|
906
|
+
return (account_key, int(window))
|
|
907
|
+
except ValueError:
|
|
908
|
+
return None
|
|
909
|
+
|
|
910
|
+
|
|
911
|
+
def retired_openings_of(selection) -> frozenset:
|
|
912
|
+
"""The windows whose opening fact the effective selection tombstones.
|
|
913
|
+
|
|
914
|
+
The rebuild never makes such a window a legacy open-block candidate
|
|
915
|
+
(`_cctally_journal.retired_block_openings`, spec 4.6.3)."""
|
|
916
|
+
retired = set()
|
|
917
|
+
for event_id, selected in selection.by_id.items():
|
|
918
|
+
if selected.status != "tombstone":
|
|
919
|
+
continue
|
|
920
|
+
window = _opening_window(str(event_id))
|
|
921
|
+
if window is not None:
|
|
922
|
+
retired.add(window)
|
|
923
|
+
return frozenset(retired)
|
|
924
|
+
|
|
925
|
+
|
|
926
|
+
def five_hour_parent_presence(events: Mapping[str, Mapping],
|
|
927
|
+
retired_openings=frozenset()) -> frozenset:
|
|
928
|
+
"""The `(account_key, five_hour_window_key)` windows a rebuild of the
|
|
929
|
+
effective set ``events`` materializes a block for (spec 4.4, 4.6.3).
|
|
930
|
+
|
|
931
|
+
A window has a block when an opening or close fact names it, and, for
|
|
932
|
+
each account, when it is the window of the account's latest-captured
|
|
933
|
+
five-hour reading (`unixepoch(captured_at_utc) DESC, journal_id DESC`)
|
|
934
|
+
and no block of it exists and its opening was not retired: the legacy
|
|
935
|
+
candidate rule of `_cctally_five_hour._backfill_five_hour_blocks`. A
|
|
936
|
+
reading a five-hour credit's suppression names carries no five-hour
|
|
937
|
+
reading once folded (retired in place, or deleted when held), so it never
|
|
938
|
+
qualifies. A weekly suppression deletes a row that carries no five-hour
|
|
939
|
+
reading, which never qualifies, and a `record-credit` synthetic, which
|
|
940
|
+
is counted here although the fold never sees it (#943 review R-5). The
|
|
941
|
+
two differ only when that synthetic is the account's latest reading
|
|
942
|
+
and lies in another window than its next surviving reading; it copies
|
|
943
|
+
the window of the latest reading (`_resolve_prior_5h`), so that does
|
|
944
|
+
not arise in practice."""
|
|
945
|
+
present = set()
|
|
946
|
+
five_hour_suppressed = set()
|
|
947
|
+
for record in events.values():
|
|
948
|
+
payload = (record or {}).get("payload") or {}
|
|
949
|
+
kind = payload.get("kind")
|
|
950
|
+
if kind in _BLOCK_PARENT_KINDS:
|
|
951
|
+
window = _window_of(payload)
|
|
952
|
+
if window is not None:
|
|
953
|
+
present.add(window)
|
|
954
|
+
elif (kind == "five_hour_credit"
|
|
955
|
+
and payload.get("suppression_table", "weekly_usage_snapshots")
|
|
956
|
+
== "weekly_usage_snapshots"
|
|
957
|
+
and isinstance(payload.get("suppression"), list)):
|
|
958
|
+
five_hour_suppressed.update(
|
|
959
|
+
item for item in payload["suppression"]
|
|
960
|
+
if isinstance(item, str))
|
|
961
|
+
latest: dict = {}
|
|
962
|
+
for event_id, record in events.items():
|
|
963
|
+
payload = (record or {}).get("payload") or {}
|
|
964
|
+
if (payload.get("kind") != "snapshot_accept"
|
|
965
|
+
or event_id in five_hour_suppressed
|
|
966
|
+
or payload.get("five_hour_percent") is None):
|
|
967
|
+
continue
|
|
968
|
+
window = _window_of(payload)
|
|
969
|
+
instant = _iso_instant(payload.get("captured_at_utc"))
|
|
970
|
+
if window is None or instant is None:
|
|
971
|
+
continue
|
|
972
|
+
rank = (instant.timestamp() // 1, str(event_id))
|
|
973
|
+
prior = latest.get(window[0])
|
|
974
|
+
if prior is None or rank > prior[0]:
|
|
975
|
+
latest[window[0]] = (rank, window)
|
|
976
|
+
retired = frozenset(retired_openings or ())
|
|
977
|
+
for _rank, window in latest.values():
|
|
978
|
+
if window not in retired:
|
|
979
|
+
present.add(window)
|
|
980
|
+
return frozenset(present)
|
|
981
|
+
|
|
982
|
+
|
|
983
|
+
def _post_plan_state(current: Mapping[str, Mapping], actions,
|
|
984
|
+
retired_openings) -> tuple[dict, frozenset]:
|
|
985
|
+
"""The effective set and the retired openings after ``actions``."""
|
|
986
|
+
after = dict(current)
|
|
987
|
+
retired = set(retired_openings or ())
|
|
988
|
+
for action in actions:
|
|
989
|
+
window = _opening_window(action.event_id)
|
|
990
|
+
if action.disposition == "tombstone":
|
|
991
|
+
after.pop(action.event_id, None)
|
|
992
|
+
if window is not None:
|
|
993
|
+
retired.add(window)
|
|
994
|
+
else:
|
|
995
|
+
after[action.event_id] = {"payload": dict(action.payload or {})}
|
|
996
|
+
if window is not None:
|
|
997
|
+
retired.discard(window)
|
|
998
|
+
return after, frozenset(retired)
|
|
999
|
+
|
|
1000
|
+
|
|
1001
|
+
def five_hour_parent_report(current: Mapping[str, Mapping], actions,
|
|
1002
|
+
retired_openings=frozenset()):
|
|
1003
|
+
"""Parent validation over the resulting effective graph (spec 4.4).
|
|
1004
|
+
|
|
1005
|
+
Returns ``(orphaned_before, lost)``: the current five-hour milestones
|
|
1006
|
+
whose window resolves no block BEFORE the plan (pre-existing damage,
|
|
1007
|
+
reported, never a reason to refuse), and the milestones that resolve a
|
|
1008
|
+
block before the plan and survive it without one."""
|
|
1009
|
+
before = five_hour_parent_presence(current, retired_openings)
|
|
1010
|
+
after_events, after_retired = _post_plan_state(
|
|
1011
|
+
current, actions, retired_openings)
|
|
1012
|
+
after = five_hour_parent_presence(after_events, after_retired)
|
|
1013
|
+
orphaned_before = set()
|
|
1014
|
+
lost = []
|
|
1015
|
+
for event_id, record in sorted(current.items()):
|
|
1016
|
+
payload = (record or {}).get("payload") or {}
|
|
1017
|
+
if payload.get("kind") != "five_hour_milestone":
|
|
1018
|
+
continue
|
|
1019
|
+
if _window_of(payload) not in before:
|
|
1020
|
+
orphaned_before.add(event_id)
|
|
1021
|
+
continue
|
|
1022
|
+
survivor = after_events.get(event_id)
|
|
1023
|
+
if survivor is None:
|
|
1024
|
+
continue
|
|
1025
|
+
if _window_of(survivor.get("payload") or {}) not in after:
|
|
1026
|
+
lost.append(event_id)
|
|
1027
|
+
return frozenset(orphaned_before), lost
|
|
1028
|
+
|
|
1029
|
+
|
|
1030
|
+
def _companions_follow_kept_credits(current: Mapping[str, Mapping],
|
|
1031
|
+
desired: dict[str, Mapping]) -> None:
|
|
1032
|
+
"""A credit the target keeps as its CURRENT record (a reviewed hold's
|
|
1033
|
+
dependency, :func:`_preserve_reviewed_hold_dependencies`) keeps its
|
|
1034
|
+
current companion with it: the companion the replay derived is bound to
|
|
1035
|
+
the replay's credit content, not the one kept, and a current companion
|
|
1036
|
+
with no desired twin would otherwise be tombstoned beside its credit."""
|
|
1037
|
+
companions = {}
|
|
1038
|
+
for source in (current, desired):
|
|
1039
|
+
for event_id, record in source.items():
|
|
1040
|
+
payload = (record or {}).get("payload") or {}
|
|
1041
|
+
if payload.get("kind") == "five_hour_credit_boundary":
|
|
1042
|
+
companions.setdefault(_credit_pair_key(payload), set()).add(
|
|
1043
|
+
event_id)
|
|
1044
|
+
for event_id, record in current.items():
|
|
1045
|
+
payload = (record or {}).get("payload") or {}
|
|
1046
|
+
if (payload.get("kind") != "five_hour_credit"
|
|
1047
|
+
or desired.get(event_id) is not record):
|
|
1048
|
+
continue
|
|
1049
|
+
for companion_id in companions.get(_credit_pair_key(payload), ()):
|
|
1050
|
+
kept = current.get(companion_id)
|
|
1051
|
+
if kept is None:
|
|
1052
|
+
desired.pop(companion_id, None)
|
|
1053
|
+
else:
|
|
1054
|
+
desired[companion_id] = kept
|
|
1055
|
+
|
|
1056
|
+
|
|
1057
|
+
# ── #884: lineage repair of five-hour milestones an earlier rederive priced ─
|
|
1058
|
+
#
|
|
1059
|
+
# #875 freezes the block cost of an unchanged crossing under a frozen close,
|
|
1060
|
+
# so a later card no longer reprices it. A pre-#875 rederive may already have
|
|
1061
|
+
# repriced one; the freeze would then keep that later card's price for good.
|
|
1062
|
+
# The lineage index reads each milestone's admissible journal history beside
|
|
1063
|
+
# its block's close history, proves which revision last legitimately set the
|
|
1064
|
+
# block cost (its ANCHOR), and lets the resolver restore that amount. Any
|
|
1065
|
+
# ambiguity keeps the current amount and is disclosed (spec §5.1–§5.7).
|
|
1066
|
+
|
|
1067
|
+
#: §5.2: an event's own history is uncertain.
|
|
1068
|
+
LINEAGE_HIDDEN_REVISION = "hidden-revision"
|
|
1069
|
+
LINEAGE_REVISION_GAP = "revision-gap"
|
|
1070
|
+
LINEAGE_DIVERGENT_REVISION = "divergent-revision"
|
|
1071
|
+
LINEAGE_INVALID_PAYLOAD = "invalid-payload"
|
|
1072
|
+
LINEAGE_ORDER_INVERSION = "order-inversion"
|
|
1073
|
+
LINEAGE_SELECTION_MISMATCH = "selection-mismatch"
|
|
1074
|
+
#: §5.3/§5.4: the walk cannot prove the anchor.
|
|
1075
|
+
LINEAGE_CLOSE_LINEAGE_UNCERTAIN = "close-lineage-uncertain"
|
|
1076
|
+
LINEAGE_CLOSE_PAIRING_UNCERTAIN = "close-pairing-uncertain"
|
|
1077
|
+
LINEAGE_REWRITE_WITHOUT_CLOSED_BOUNDARY = "rewrite-without-closed-boundary"
|
|
1078
|
+
LINEAGE_UNPAIRED_CLOSE_CHANGE = "unpaired-close-change"
|
|
1079
|
+
|
|
1080
|
+
_LINEAGE_KINDS = frozenset({"five_hour_milestone", "five_hour_block_close"})
|
|
1081
|
+
|
|
1082
|
+
|
|
1083
|
+
@dataclasses.dataclass(frozen=True)
|
|
1084
|
+
class LineageOccurrence:
|
|
1085
|
+
"""One admissible candidate of a milestone or close, summarized (§5.1).
|
|
1086
|
+
|
|
1087
|
+
``available_at`` is the journal coordinate at which it became effective:
|
|
1088
|
+
a rev-0 event's own sequence, or its batch's commit sequence. Milestone
|
|
1089
|
+
fields are ``None`` for a close and vice versa; a tombstone carries none.
|
|
1090
|
+
``identity`` is :func:`_milestone_identity` without ``alerted_at``.
|
|
1091
|
+
``close_view`` is an active close's canonical JSON encoding of what the
|
|
1092
|
+
close timeline reads (§5.2, §5.3): ``kind``, the block key, ``is_closed``
|
|
1093
|
+
and ``total_cost_usd``, each type-sensitive (``1``, ``1.0`` and ``true``
|
|
1094
|
+
differ); an absent key and an explicit ``null`` encode alike.
|
|
1095
|
+
"""
|
|
1096
|
+
rev: int
|
|
1097
|
+
status: str
|
|
1098
|
+
content_hash: str
|
|
1099
|
+
batch_id: "str | None"
|
|
1100
|
+
available_at: int
|
|
1101
|
+
kind: "str | None"
|
|
1102
|
+
block: "tuple | None"
|
|
1103
|
+
at: object = None
|
|
1104
|
+
identity: "dict | None" = None
|
|
1105
|
+
block_cost_usd: object = None
|
|
1106
|
+
is_closed: object = None
|
|
1107
|
+
total_cost_usd: object = None
|
|
1108
|
+
close_view: "bytes | None" = None
|
|
1109
|
+
|
|
1110
|
+
|
|
1111
|
+
@dataclasses.dataclass(frozen=True)
|
|
1112
|
+
class Restoration:
|
|
1113
|
+
"""One unchanged crossing's lineage answer (§5.4): ``keep``,
|
|
1114
|
+
``withhold`` (fail closed) or ``restore`` to ``value``.
|
|
1115
|
+
|
|
1116
|
+
``reported`` is §5.7's predicate: a withhold counts and is disclosed only
|
|
1117
|
+
when the selected crossing has a repair candidate, the revisions listed in
|
|
1118
|
+
``candidate_revisions``.
|
|
1119
|
+
"""
|
|
1120
|
+
action: str
|
|
1121
|
+
value: object = None
|
|
1122
|
+
reason_codes: tuple = ()
|
|
1123
|
+
candidate_revisions: tuple = ()
|
|
1124
|
+
reported: bool = False
|
|
1125
|
+
|
|
1126
|
+
|
|
1127
|
+
_LINEAGE_KEEP = Restoration("keep")
|
|
1128
|
+
|
|
1129
|
+
|
|
1130
|
+
def _lineage_identity(payload: Mapping) -> dict:
|
|
1131
|
+
"""§5.1: a milestone's non-monetary identity without its alert latch."""
|
|
1132
|
+
return {
|
|
1133
|
+
key: value for key, value in payload.items()
|
|
1134
|
+
if key not in _FIVE_HOUR_MILESTONE_MONETARY_KEYS
|
|
1135
|
+
and key != "alerted_at"
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
|
|
1139
|
+
def _divergence_key(occurrence, *, close: bool):
|
|
1140
|
+
"""§5.2: what two occurrences of one revision must share to agree.
|
|
1141
|
+
|
|
1142
|
+
A milestone's occurrences agree only on the same ``(status,
|
|
1143
|
+
content_hash)``. A close's agree on what §5.3 reads: a tombstone on its
|
|
1144
|
+
status alone, an active occurrence on its status and ``close_view``
|
|
1145
|
+
(compared as bytes, so no value is coerced or assumed hashable). A
|
|
1146
|
+
divergence confined to a close's observation fields, weekly axes,
|
|
1147
|
+
tokens, children or markers changes no close state or event (finding
|
|
1148
|
+
F1, the live writer's rev-0 re-emissions).
|
|
1149
|
+
"""
|
|
1150
|
+
if not close:
|
|
1151
|
+
return (occurrence.status, occurrence.content_hash)
|
|
1152
|
+
if occurrence.status != "active":
|
|
1153
|
+
return (occurrence.status,)
|
|
1154
|
+
return (occurrence.status, occurrence.close_view)
|
|
1155
|
+
|
|
1156
|
+
|
|
1157
|
+
def _history_uncertainty(occurrences, *, hidden: bool, invalid: bool,
|
|
1158
|
+
selected, close: bool = False) -> tuple:
|
|
1159
|
+
"""§5.2: the sorted reason codes of one event's own uncertain history.
|
|
1160
|
+
|
|
1161
|
+
``close`` marks a close id, whose same-revision occurrences are compared
|
|
1162
|
+
on the fields the close timeline reads (:func:`_divergence_key`); every
|
|
1163
|
+
other code, ``selection-mismatch``'s full-hash check included, is the
|
|
1164
|
+
same for both kinds.
|
|
1165
|
+
"""
|
|
1166
|
+
codes = set()
|
|
1167
|
+
if hidden:
|
|
1168
|
+
codes.add(LINEAGE_HIDDEN_REVISION)
|
|
1169
|
+
if invalid:
|
|
1170
|
+
codes.add(LINEAGE_INVALID_PAYLOAD)
|
|
1171
|
+
by_rev: dict[int, list] = {}
|
|
1172
|
+
for occurrence in occurrences:
|
|
1173
|
+
by_rev.setdefault(occurrence.rev, []).append(occurrence)
|
|
1174
|
+
top = max(by_rev)
|
|
1175
|
+
upper = top if selected is None else max(top, selected[0])
|
|
1176
|
+
if any(rev not in by_rev for rev in range(upper + 1)):
|
|
1177
|
+
codes.add(LINEAGE_REVISION_GAP)
|
|
1178
|
+
earliest = {}
|
|
1179
|
+
for rev, group in by_rev.items():
|
|
1180
|
+
if len({_divergence_key(item, close=close) for item in group}) > 1:
|
|
1181
|
+
codes.add(LINEAGE_DIVERGENT_REVISION)
|
|
1182
|
+
earliest[rev] = min(item.available_at for item in group)
|
|
1183
|
+
if any(
|
|
1184
|
+
rev - 1 in earliest and earliest[rev] < earliest[rev - 1]
|
|
1185
|
+
for rev in earliest
|
|
1186
|
+
):
|
|
1187
|
+
codes.add(LINEAGE_ORDER_INVERSION)
|
|
1188
|
+
if (
|
|
1189
|
+
selected is None or selected[0] != top
|
|
1190
|
+
or selected[1] not in {item.content_hash for item in by_rev[top]}
|
|
1191
|
+
):
|
|
1192
|
+
codes.add(LINEAGE_SELECTION_MISMATCH)
|
|
1193
|
+
return tuple(sorted(codes))
|
|
1194
|
+
|
|
1195
|
+
|
|
1196
|
+
def _first_per_revision(occurrences) -> dict:
|
|
1197
|
+
"""``{rev: earliest occurrence}``; ``occurrences`` are sorted."""
|
|
1198
|
+
chain: dict[int, LineageOccurrence] = {}
|
|
1199
|
+
for occurrence in occurrences:
|
|
1200
|
+
chain.setdefault(occurrence.rev, occurrence)
|
|
1201
|
+
return chain
|
|
1202
|
+
|
|
1203
|
+
|
|
1204
|
+
@dataclasses.dataclass(frozen=True)
|
|
1205
|
+
class FiveHourLineage:
|
|
1206
|
+
"""§5.1: the compact lineage of every five-hour milestone and close.
|
|
1207
|
+
|
|
1208
|
+
Holds summaries only, never records or the fold. ``uncertain`` maps each
|
|
1209
|
+
id whose own history is uncertain (§5.2) to its reason codes.
|
|
1210
|
+
``close_ids_ever_in_block`` is a candidate index; membership at a
|
|
1211
|
+
coordinate is decided from the winning payload there (§5.3).
|
|
1212
|
+
"""
|
|
1213
|
+
milestones: Mapping[str, tuple]
|
|
1214
|
+
closes: Mapping[str, tuple]
|
|
1215
|
+
close_ids_ever_in_block: Mapping[tuple, tuple]
|
|
1216
|
+
hidden_ids: frozenset
|
|
1217
|
+
selected: Mapping[str, tuple]
|
|
1218
|
+
uncertain: Mapping[str, tuple]
|
|
1219
|
+
_block_timelines: dict = dataclasses.field(
|
|
1220
|
+
default_factory=dict, compare=False, repr=False)
|
|
1221
|
+
#: #1040: memoized journal end of the summarized occurrences.
|
|
1222
|
+
_coordinate_end: list = dataclasses.field(
|
|
1223
|
+
default_factory=list, compare=False, repr=False)
|
|
1224
|
+
|
|
1225
|
+
def _block_timeline(self, block) -> tuple:
|
|
1226
|
+
"""§5.3: K's close events, ``(coordinates, states, changed,
|
|
1227
|
+
live_writer)``.
|
|
1228
|
+
|
|
1229
|
+
Only called when every close id of the block has a certain history,
|
|
1230
|
+
so each id's revision becomes effective at its earliest occurrence.
|
|
1231
|
+
A state is ``("ambiguous",)``, ``("absent",)``, ``("open",)`` or
|
|
1232
|
+
``("closed", total)``. ``live_writer`` marks a coordinate that only
|
|
1233
|
+
rev-0 ``evt`` occurrences (no correction batch) make effective
|
|
1234
|
+
(§5.4 step 3's first live-writer close).
|
|
1235
|
+
"""
|
|
1236
|
+
cached = self._block_timelines.get(block)
|
|
1237
|
+
if cached is not None:
|
|
1238
|
+
return cached
|
|
1239
|
+
chains = {
|
|
1240
|
+
event_id: _first_per_revision(self.closes[event_id])
|
|
1241
|
+
for event_id in self.close_ids_ever_in_block.get(block, ())
|
|
1242
|
+
}
|
|
1243
|
+
coordinates = sorted({
|
|
1244
|
+
occurrence.available_at
|
|
1245
|
+
for event_id in chains
|
|
1246
|
+
for occurrence in self.closes[event_id]
|
|
1247
|
+
})
|
|
1248
|
+
states = []
|
|
1249
|
+
changed = []
|
|
1250
|
+
live_writer = []
|
|
1251
|
+
for coordinate in coordinates:
|
|
1252
|
+
counted = []
|
|
1253
|
+
moved = False
|
|
1254
|
+
for chain in chains.values():
|
|
1255
|
+
live = [rev for rev, occurrence in chain.items()
|
|
1256
|
+
if occurrence.available_at <= coordinate]
|
|
1257
|
+
if not live:
|
|
1258
|
+
continue
|
|
1259
|
+
winner = chain[max(live)]
|
|
1260
|
+
moved = moved or winner.available_at == coordinate
|
|
1261
|
+
if winner.status == "active" and winner.block == block:
|
|
1262
|
+
counted.append(winner)
|
|
1263
|
+
changed.append(moved)
|
|
1264
|
+
live_writer.append(all(
|
|
1265
|
+
occurrence.batch_id is None and occurrence.rev == 0
|
|
1266
|
+
for event_id in chains
|
|
1267
|
+
for occurrence in self.closes[event_id]
|
|
1268
|
+
if occurrence.available_at == coordinate
|
|
1269
|
+
))
|
|
1270
|
+
if len(counted) > 1:
|
|
1271
|
+
states.append(("ambiguous",))
|
|
1272
|
+
elif not counted:
|
|
1273
|
+
states.append(("absent",))
|
|
1274
|
+
elif counted[0].is_closed != 1:
|
|
1275
|
+
states.append(("open",))
|
|
1276
|
+
else:
|
|
1277
|
+
states.append(("closed", counted[0].total_cost_usd))
|
|
1278
|
+
timeline = (tuple(coordinates), tuple(states), tuple(changed),
|
|
1279
|
+
tuple(live_writer))
|
|
1280
|
+
self._block_timelines[block] = timeline
|
|
1281
|
+
return timeline
|
|
1282
|
+
|
|
1283
|
+
@staticmethod
|
|
1284
|
+
def _closed_with_money(state) -> bool:
|
|
1285
|
+
return state[0] == "closed" and _usable_amount(state[1])
|
|
1286
|
+
|
|
1287
|
+
@classmethod
|
|
1288
|
+
def _classify(cls, before, after, changed: bool) -> str:
|
|
1289
|
+
"""§5.3: one close event, from K's states just before and at it."""
|
|
1290
|
+
if before[0] == "ambiguous" or after[0] == "ambiguous":
|
|
1291
|
+
return "uncertain"
|
|
1292
|
+
if not changed:
|
|
1293
|
+
return "metadata"
|
|
1294
|
+
if before[0] == "closed" and after[0] == "closed":
|
|
1295
|
+
if not (_usable_amount(before[1]) and _usable_amount(after[1])):
|
|
1296
|
+
return "uncertain"
|
|
1297
|
+
if abs(after[1] - before[1]) > _CLOSE_TOTAL_TOLERANCE_USD:
|
|
1298
|
+
return "monetary"
|
|
1299
|
+
return "metadata"
|
|
1300
|
+
if after[0] == "closed":
|
|
1301
|
+
return "monetary" if _usable_amount(after[1]) else "uncertain"
|
|
1302
|
+
if before[0] == "closed":
|
|
1303
|
+
return "monetary" if _usable_amount(before[1]) else "uncertain"
|
|
1304
|
+
if before[0] == after[0] == "absent":
|
|
1305
|
+
return "metadata"
|
|
1306
|
+
return "no-boundary"
|
|
1307
|
+
|
|
1308
|
+
def _walk(self, chain, tail: int, selected: int, block) -> tuple:
|
|
1309
|
+
"""§5.4 steps 1–6 over the coordinate groups from ``tail``'s
|
|
1310
|
+
coordinate inclusive. Returns ``(anchor, reasons)``."""
|
|
1311
|
+
coordinates, states, changed_flags, live_writer = \
|
|
1312
|
+
self._block_timeline(block)
|
|
1313
|
+
start = chain[tail].available_at
|
|
1314
|
+
revisions_at: dict[int, list] = {}
|
|
1315
|
+
for rev in range(tail + 1, selected + 1):
|
|
1316
|
+
revisions_at.setdefault(chain[rev].available_at, []).append(rev)
|
|
1317
|
+
events = {}
|
|
1318
|
+
# §5.4 step 3: an unpaired monetary event is exempt from the pending
|
|
1319
|
+
# mark only as K's FIRST live-writer close, so whether K was ever
|
|
1320
|
+
# closed before it is read over the whole timeline, before t too.
|
|
1321
|
+
first_live_close = {}
|
|
1322
|
+
closed_before = False
|
|
1323
|
+
for index, coordinate in enumerate(coordinates):
|
|
1324
|
+
before = states[index - 1] if index else ("absent",)
|
|
1325
|
+
if coordinate >= start:
|
|
1326
|
+
events[coordinate] = self._classify(
|
|
1327
|
+
before, states[index], changed_flags[index])
|
|
1328
|
+
first_live_close[coordinate] = (
|
|
1329
|
+
live_writer[index] and not closed_before
|
|
1330
|
+
and before[0] in ("absent", "open")
|
|
1331
|
+
and states[index][0] == "closed"
|
|
1332
|
+
)
|
|
1333
|
+
closed_before = closed_before or states[index][0] == "closed"
|
|
1334
|
+
anchor = tail
|
|
1335
|
+
reasons: set = set()
|
|
1336
|
+
# An unpaired monetary close event since the anchor leaves A
|
|
1337
|
+
# unproven for a later price-only rewrite (findings D1, R7-1).
|
|
1338
|
+
marked = False
|
|
1339
|
+
for coordinate in sorted(set(revisions_at) | set(events)):
|
|
1340
|
+
revisions = revisions_at.get(coordinate, [])
|
|
1341
|
+
changed = any(
|
|
1342
|
+
chain[rev].block_cost_usd != chain[rev - 1].block_cost_usd
|
|
1343
|
+
for rev in revisions
|
|
1344
|
+
)
|
|
1345
|
+
event = events.get(coordinate)
|
|
1346
|
+
if event == "monetary":
|
|
1347
|
+
if changed:
|
|
1348
|
+
anchor = max(revisions)
|
|
1349
|
+
reasons.clear()
|
|
1350
|
+
marked = False
|
|
1351
|
+
else:
|
|
1352
|
+
effective = max(
|
|
1353
|
+
rev for rev in range(tail, selected + 1)
|
|
1354
|
+
if chain[rev].available_at <= coordinate
|
|
1355
|
+
)
|
|
1356
|
+
if (chain[effective].block_cost_usd
|
|
1357
|
+
!= chain[anchor].block_cost_usd):
|
|
1358
|
+
reasons.add(LINEAGE_UNPAIRED_CLOSE_CHANGE)
|
|
1359
|
+
if not first_live_close[coordinate]:
|
|
1360
|
+
marked = True
|
|
1361
|
+
elif event == "uncertain":
|
|
1362
|
+
reasons.add(LINEAGE_CLOSE_PAIRING_UNCERTAIN)
|
|
1363
|
+
elif changed:
|
|
1364
|
+
if event == "no-boundary":
|
|
1365
|
+
reasons.add(LINEAGE_REWRITE_WITHOUT_CLOSED_BOUNDARY)
|
|
1366
|
+
else:
|
|
1367
|
+
# A price-only rewrite (no close event, or a metadata
|
|
1368
|
+
# one) is walked past only under a retained closed close
|
|
1369
|
+
# and with no step-3 mark pending.
|
|
1370
|
+
index = bisect.bisect_left(coordinates, coordinate)
|
|
1371
|
+
before = states[index - 1] if index else ("absent",)
|
|
1372
|
+
if not self._closed_with_money(before):
|
|
1373
|
+
reasons.add(LINEAGE_REWRITE_WITHOUT_CLOSED_BOUNDARY)
|
|
1374
|
+
if marked:
|
|
1375
|
+
reasons.add(LINEAGE_UNPAIRED_CLOSE_CHANGE)
|
|
1376
|
+
return anchor, reasons
|
|
1377
|
+
|
|
1378
|
+
def _withhold(self, event_id: str, reasons, current_cost,
|
|
1379
|
+
anchor) -> Restoration:
|
|
1380
|
+
"""A fail-closed answer and §5.7's report predicate."""
|
|
1381
|
+
occurrences = self.milestones[event_id]
|
|
1382
|
+
selected = self.selected.get(event_id)
|
|
1383
|
+
target = None
|
|
1384
|
+
for occurrence in occurrences:
|
|
1385
|
+
if (selected is not None and occurrence.status == "active"
|
|
1386
|
+
and (occurrence.rev, occurrence.content_hash) == selected):
|
|
1387
|
+
target = occurrence
|
|
1388
|
+
break
|
|
1389
|
+
candidates = []
|
|
1390
|
+
if target is not None:
|
|
1391
|
+
by_rev: dict[int, list] = {}
|
|
1392
|
+
for occurrence in occurrences:
|
|
1393
|
+
by_rev.setdefault(occurrence.rev, []).append(occurrence)
|
|
1394
|
+
for rev in range(max(by_rev), -1, -1):
|
|
1395
|
+
if anchor is not None and rev < anchor:
|
|
1396
|
+
break
|
|
1397
|
+
group = by_rev.get(rev)
|
|
1398
|
+
if not group:
|
|
1399
|
+
continue # a gap is not a boundary
|
|
1400
|
+
matching = [
|
|
1401
|
+
occurrence for occurrence in group
|
|
1402
|
+
if occurrence.status == "active"
|
|
1403
|
+
and occurrence.at == target.at
|
|
1404
|
+
and occurrence.identity == target.identity
|
|
1405
|
+
]
|
|
1406
|
+
if not matching:
|
|
1407
|
+
break # a proven boundary
|
|
1408
|
+
if any(
|
|
1409
|
+
_usable_amount(occurrence.block_cost_usd)
|
|
1410
|
+
and occurrence.block_cost_usd != current_cost
|
|
1411
|
+
for occurrence in matching
|
|
1412
|
+
):
|
|
1413
|
+
candidates.append(rev)
|
|
1414
|
+
return Restoration(
|
|
1415
|
+
"withhold",
|
|
1416
|
+
reason_codes=tuple(sorted(set(reasons))),
|
|
1417
|
+
candidate_revisions=tuple(sorted(candidates)),
|
|
1418
|
+
reported=bool(candidates),
|
|
1419
|
+
)
|
|
1420
|
+
|
|
1421
|
+
def restoration(self, event_id: str,
|
|
1422
|
+
current_record: Mapping) -> Restoration:
|
|
1423
|
+
"""§5.4: the decision for one unchanged crossing in a frozen block.
|
|
1424
|
+
|
|
1425
|
+
``current_record`` is the selected record; its ``block_cost_usd`` is
|
|
1426
|
+
the comparison base. In order: ``keep`` without recorded history;
|
|
1427
|
+
``withhold`` when the event's own history is uncertain; ``withhold``
|
|
1428
|
+
when the walk collected a reason; ``keep`` when the anchor's amount
|
|
1429
|
+
equals the current one or is unusable; else ``restore`` to the
|
|
1430
|
+
anchor's parsed ``block_cost_usd``, copied as is.
|
|
1431
|
+
"""
|
|
1432
|
+
occurrences = self.milestones.get(event_id)
|
|
1433
|
+
if not occurrences:
|
|
1434
|
+
return _LINEAGE_KEEP
|
|
1435
|
+
current_cost = (current_record.get("payload") or {}).get(
|
|
1436
|
+
"block_cost_usd")
|
|
1437
|
+
own = self.uncertain.get(event_id)
|
|
1438
|
+
if own:
|
|
1439
|
+
return self._withhold(event_id, own, current_cost, None)
|
|
1440
|
+
chain = _first_per_revision(occurrences)
|
|
1441
|
+
selected = max(chain)
|
|
1442
|
+
if chain[selected].status != "active":
|
|
1443
|
+
return _LINEAGE_KEEP
|
|
1444
|
+
life_start = 0
|
|
1445
|
+
for rev in range(selected, -1, -1):
|
|
1446
|
+
if chain[rev].status != "active":
|
|
1447
|
+
life_start = rev + 1
|
|
1448
|
+
break
|
|
1449
|
+
tail = life_start
|
|
1450
|
+
for rev in range(life_start + 1, selected + 1):
|
|
1451
|
+
if (chain[rev].at != chain[rev - 1].at
|
|
1452
|
+
or chain[rev].identity != chain[rev - 1].identity):
|
|
1453
|
+
tail = rev
|
|
1454
|
+
block = chain[selected].block
|
|
1455
|
+
if any(close_id in self.uncertain
|
|
1456
|
+
for close_id in self.close_ids_ever_in_block.get(block, ())):
|
|
1457
|
+
# §5.3: no close event of K can prove anything; A stays at t.
|
|
1458
|
+
return self._withhold(
|
|
1459
|
+
event_id, {LINEAGE_CLOSE_LINEAGE_UNCERTAIN}, current_cost,
|
|
1460
|
+
tail)
|
|
1461
|
+
anchor, reasons = self._walk(chain, tail, selected, block)
|
|
1462
|
+
if reasons:
|
|
1463
|
+
return self._withhold(event_id, reasons, current_cost, anchor)
|
|
1464
|
+
value = chain[anchor].block_cost_usd
|
|
1465
|
+
if not _usable_amount(value) or value == current_cost:
|
|
1466
|
+
return _LINEAGE_KEEP
|
|
1467
|
+
return Restoration("restore", value=value)
|
|
1468
|
+
|
|
1469
|
+
def extended(self, *, batch_id: "str | None" = None,
|
|
1470
|
+
batch_actions: Sequence[Mapping] = (),
|
|
1471
|
+
plain: Sequence[Mapping] = ()) -> "FiveHourLineage":
|
|
1472
|
+
"""#1040 (spec §5.4): the lineage as it reads after ``batch_actions``
|
|
1473
|
+
(one completed correction batch, completion's action form) and then
|
|
1474
|
+
``plain`` (revision-0 events) are appended at the journal's end.
|
|
1475
|
+
|
|
1476
|
+
Read-only: a new instance with an empty timeline cache. The batch
|
|
1477
|
+
becomes effective at one coordinate after every existing one, each
|
|
1478
|
+
plain event at the following coordinates in order. Every candidate
|
|
1479
|
+
is built and hashed by the selector's own constructors, so each
|
|
1480
|
+
touched id's occurrences, selected pair (its highest revision; a tie
|
|
1481
|
+
keeps the earlier occurrence) and history uncertainty are what
|
|
1482
|
+
:func:`build_five_hour_lineage` reads over a journal that really
|
|
1483
|
+
holds the records: a revision written above an admission-hidden id
|
|
1484
|
+
stays ``hidden-revision``, one above an inert revision leaves a
|
|
1485
|
+
``revision-gap``. The close index gains every block a touched close
|
|
1486
|
+
id's new occurrence names.
|
|
1487
|
+
"""
|
|
1488
|
+
if not self._coordinate_end:
|
|
1489
|
+
self._coordinate_end.append(max(
|
|
1490
|
+
(occurrence.available_at
|
|
1491
|
+
for table in (self.milestones, self.closes)
|
|
1492
|
+
for items in table.values() for occurrence in items),
|
|
1493
|
+
default=-1,
|
|
1494
|
+
))
|
|
1495
|
+
coordinate = self._coordinate_end[0] + 1
|
|
1496
|
+
added: dict[str, list] = {}
|
|
1497
|
+
if batch_actions:
|
|
1498
|
+
name = batch_id or "lineage-extended"
|
|
1499
|
+
records = _lib_journal.make_correction_batch(
|
|
1500
|
+
batch_id=name, family="lineage-extended", at="extended",
|
|
1501
|
+
actions=[dict(action) for action in batch_actions])
|
|
1502
|
+
for record in records[1:-1]:
|
|
1503
|
+
candidate = _lib_journal._candidate_from_correction(
|
|
1504
|
+
record, coordinate)
|
|
1505
|
+
added.setdefault(candidate.event_id, []).append(candidate)
|
|
1506
|
+
coordinate += 1
|
|
1507
|
+
for record in plain:
|
|
1508
|
+
candidate = _lib_journal._candidate_from_evt(record, coordinate)
|
|
1509
|
+
added.setdefault(candidate.event_id, []).append(candidate)
|
|
1510
|
+
coordinate += 1
|
|
1511
|
+
if not added:
|
|
1512
|
+
return dataclasses.replace(
|
|
1513
|
+
self, _block_timelines={}, _coordinate_end=[])
|
|
1514
|
+
milestones = dict(self.milestones)
|
|
1515
|
+
closes = dict(self.closes)
|
|
1516
|
+
selected = dict(self.selected)
|
|
1517
|
+
uncertain = dict(self.uncertain)
|
|
1518
|
+
ever = {block: set(ids)
|
|
1519
|
+
for block, ids in self.close_ids_ever_in_block.items()}
|
|
1520
|
+
for event_id, candidates in added.items():
|
|
1521
|
+
existing = list(self.closes.get(event_id)
|
|
1522
|
+
or self.milestones.get(event_id) or ())
|
|
1523
|
+
identities: dict = {}
|
|
1524
|
+
for occurrence in existing:
|
|
1525
|
+
if occurrence.identity is not None:
|
|
1526
|
+
identities["identity"] = occurrence.identity
|
|
1527
|
+
fresh = []
|
|
1528
|
+
for candidate in candidates:
|
|
1529
|
+
occurrence, _kind, _unhashable = _lineage_occurrence(
|
|
1530
|
+
candidate.record, rev=candidate.rev,
|
|
1531
|
+
status=candidate.status,
|
|
1532
|
+
content_hash=candidate.content_hash,
|
|
1533
|
+
batch_id=candidate.batch_id,
|
|
1534
|
+
available_at=candidate.sequence, identities=identities)
|
|
1535
|
+
fresh.append(occurrence)
|
|
1536
|
+
items = existing + fresh
|
|
1537
|
+
kinds = {occurrence.kind for occurrence in items
|
|
1538
|
+
if occurrence.status == "active"}
|
|
1539
|
+
if not existing and not (kinds & _LINEAGE_KINDS):
|
|
1540
|
+
continue # not a five-hour milestone or close
|
|
1541
|
+
pair = selected.get(event_id)
|
|
1542
|
+
for candidate in candidates:
|
|
1543
|
+
if pair is None or candidate.rev > pair[0]:
|
|
1544
|
+
pair = (candidate.rev, candidate.content_hash)
|
|
1545
|
+
selected[event_id] = pair
|
|
1546
|
+
unhashable = any(
|
|
1547
|
+
occurrence.status == "active"
|
|
1548
|
+
and occurrence.kind == "five_hour_block_close"
|
|
1549
|
+
and occurrence.block is None
|
|
1550
|
+
for occurrence in items)
|
|
1551
|
+
items, codes = _finalize_lineage_id(
|
|
1552
|
+
event_id, items, kinds=kinds, unhashable=unhashable,
|
|
1553
|
+
selected=pair, hidden=event_id in self.hidden_ids)
|
|
1554
|
+
if codes:
|
|
1555
|
+
uncertain[event_id] = codes
|
|
1556
|
+
else:
|
|
1557
|
+
uncertain.pop(event_id, None)
|
|
1558
|
+
if "five_hour_block_close" in kinds:
|
|
1559
|
+
closes[event_id] = items
|
|
1560
|
+
for occurrence in items:
|
|
1561
|
+
if (occurrence.kind == "five_hour_block_close"
|
|
1562
|
+
and occurrence.block is not None):
|
|
1563
|
+
ever.setdefault(occurrence.block, set()).add(event_id)
|
|
1564
|
+
if "five_hour_milestone" in kinds:
|
|
1565
|
+
milestones[event_id] = items
|
|
1566
|
+
return FiveHourLineage(
|
|
1567
|
+
milestones=milestones,
|
|
1568
|
+
closes=closes,
|
|
1569
|
+
close_ids_ever_in_block={
|
|
1570
|
+
block: tuple(sorted(ids)) for block, ids in ever.items()},
|
|
1571
|
+
hidden_ids=self.hidden_ids,
|
|
1572
|
+
selected=selected,
|
|
1573
|
+
uncertain=uncertain,
|
|
1574
|
+
)
|
|
1575
|
+
|
|
1576
|
+
|
|
1577
|
+
def _lineage_occurrence(record, *, rev: int, status: str, content_hash: str,
|
|
1578
|
+
batch_id, available_at: int, identities: dict):
|
|
1579
|
+
"""§5.1: one admissible candidate summarized as a :class:`LineageOccurrence`.
|
|
1580
|
+
|
|
1581
|
+
Returns ``(occurrence, kind, unhashable)``: ``kind`` is the active
|
|
1582
|
+
payload's kind (``None`` for a tombstone or a non-object payload) and
|
|
1583
|
+
``unhashable`` marks a close whose block key indexes no block (§5.2,
|
|
1584
|
+
finding D6). ``identities`` shares one identity object per run of equal
|
|
1585
|
+
milestone identities of an id.
|
|
1586
|
+
"""
|
|
1587
|
+
fields: dict = {}
|
|
1588
|
+
kind = None
|
|
1589
|
+
unhashable = False
|
|
1590
|
+
if status == "active":
|
|
1591
|
+
payload = (record or {}).get("payload")
|
|
1592
|
+
kind = payload.get("kind") if isinstance(payload, dict) else None
|
|
1593
|
+
if kind == "five_hour_milestone":
|
|
1594
|
+
identity = _lineage_identity(payload)
|
|
1595
|
+
prior = identities.get("identity")
|
|
1596
|
+
if prior is not None and prior == identity:
|
|
1597
|
+
identity = prior # one shared object per run of equals
|
|
1598
|
+
identities["identity"] = identity
|
|
1599
|
+
fields = dict(
|
|
1600
|
+
block=_five_hour_block(payload),
|
|
1601
|
+
at=record.get("at"), identity=identity,
|
|
1602
|
+
block_cost_usd=payload.get("block_cost_usd"),
|
|
1603
|
+
)
|
|
1604
|
+
elif kind == "five_hour_block_close":
|
|
1605
|
+
block = _five_hour_block(payload)
|
|
1606
|
+
try:
|
|
1607
|
+
hash(block)
|
|
1608
|
+
except TypeError:
|
|
1609
|
+
# §5.2 (finding D6): a block key that indexes no block
|
|
1610
|
+
# makes the id ``invalid-payload``; the plan goes on.
|
|
1611
|
+
unhashable = True
|
|
1612
|
+
block = None
|
|
1613
|
+
fields = dict(
|
|
1614
|
+
block=block,
|
|
1615
|
+
is_closed=payload.get("is_closed"),
|
|
1616
|
+
total_cost_usd=payload.get("total_cost_usd"),
|
|
1617
|
+
close_view=_canonical_bytes([
|
|
1618
|
+
kind, payload.get("account_key"),
|
|
1619
|
+
payload.get("five_hour_window_key"),
|
|
1620
|
+
payload.get("is_closed"),
|
|
1621
|
+
payload.get("total_cost_usd"),
|
|
1622
|
+
]),
|
|
1623
|
+
)
|
|
1624
|
+
fields.setdefault("block", None)
|
|
1625
|
+
return LineageOccurrence(
|
|
1626
|
+
rev=rev, status=status, content_hash=content_hash, batch_id=batch_id,
|
|
1627
|
+
available_at=available_at, kind=kind, **fields,
|
|
1628
|
+
), kind, unhashable
|
|
1629
|
+
|
|
1630
|
+
|
|
1631
|
+
def _finalize_lineage_id(event_id: str, items: list, *, kinds, unhashable: bool,
|
|
1632
|
+
selected, hidden: bool) -> tuple:
|
|
1633
|
+
"""§5.1/§5.2: one id's occurrences in canonical order and its own
|
|
1634
|
+
history uncertainty. Sorts ``items`` in place; returns
|
|
1635
|
+
``(tuple(items), codes)``."""
|
|
1636
|
+
items.sort(key=lambda item: (
|
|
1637
|
+
item.rev, item.available_at, item.status, item.content_hash,
|
|
1638
|
+
item.batch_id or "",
|
|
1639
|
+
))
|
|
1640
|
+
codes = _history_uncertainty(
|
|
1641
|
+
items, hidden=hidden,
|
|
1642
|
+
invalid=len(kinds) != 1 or unhashable,
|
|
1643
|
+
selected=selected,
|
|
1644
|
+
close=set(kinds) == {"five_hour_block_close"},
|
|
1645
|
+
)
|
|
1646
|
+
return tuple(items), codes
|
|
1647
|
+
|
|
1648
|
+
|
|
1649
|
+
def build_five_hour_lineage(fold, *, selection,
|
|
1650
|
+
physical_revisions) -> FiveHourLineage:
|
|
1651
|
+
"""§5.1: one pass over the selector fold's admissible candidates.
|
|
1652
|
+
|
|
1653
|
+
``fold`` is the :class:`SelectorFold` ``resolve_effective_events`` built
|
|
1654
|
+
(its ``candidates`` hold every rev-0 event and every action of each
|
|
1655
|
+
completed, untainted batch); ``physical_revisions`` maps an id to its
|
|
1656
|
+
highest admission-hidden revision. Every candidate of an id some active
|
|
1657
|
+
candidate places in a five-hour milestone or close is kept, identical
|
|
1658
|
+
duplicates in different batches included.
|
|
1659
|
+
"""
|
|
1660
|
+
wanted = set()
|
|
1661
|
+
for candidate in fold.candidates:
|
|
1662
|
+
if candidate.status == "active" and candidate.record is not None:
|
|
1663
|
+
payload = candidate.record.get("payload")
|
|
1664
|
+
if isinstance(payload, dict) and payload.get("kind") in \
|
|
1665
|
+
_LINEAGE_KINDS:
|
|
1666
|
+
wanted.add(candidate.event_id)
|
|
1667
|
+
commits: dict[str, int] = {}
|
|
1668
|
+
kinds: dict[str, set] = {}
|
|
1669
|
+
occurrences: dict[str, list] = {}
|
|
1670
|
+
identities: dict[str, dict] = {}
|
|
1671
|
+
unhashable_ids: set = set()
|
|
1672
|
+
for candidate in fold.candidates:
|
|
1673
|
+
event_id = candidate.event_id
|
|
1674
|
+
if event_id not in wanted:
|
|
1675
|
+
continue
|
|
1676
|
+
if candidate.batch_id is None:
|
|
1677
|
+
available_at = candidate.sequence
|
|
1678
|
+
else:
|
|
1679
|
+
available_at = commits.get(candidate.batch_id)
|
|
1680
|
+
if available_at is None:
|
|
1681
|
+
available_at = fold.markers[candidate.batch_id]["commit"][3]
|
|
1682
|
+
commits[candidate.batch_id] = available_at
|
|
1683
|
+
occurrence, kind, unhashable = _lineage_occurrence(
|
|
1684
|
+
candidate.record, rev=candidate.rev, status=candidate.status,
|
|
1685
|
+
content_hash=candidate.content_hash, batch_id=candidate.batch_id,
|
|
1686
|
+
available_at=available_at,
|
|
1687
|
+
identities=identities.setdefault(event_id, {}),
|
|
1688
|
+
)
|
|
1689
|
+
if candidate.status == "active":
|
|
1690
|
+
kinds.setdefault(event_id, set()).add(kind)
|
|
1691
|
+
if unhashable:
|
|
1692
|
+
unhashable_ids.add(event_id)
|
|
1693
|
+
occurrences.setdefault(event_id, []).append(occurrence)
|
|
1694
|
+
hidden_ids = frozenset(physical_revisions or ())
|
|
1695
|
+
milestones: dict[str, tuple] = {}
|
|
1696
|
+
closes: dict[str, tuple] = {}
|
|
1697
|
+
close_ids_ever_in_block: dict[tuple, set] = {}
|
|
1698
|
+
selected: dict[str, tuple] = {}
|
|
1699
|
+
uncertain: dict[str, tuple] = {}
|
|
1700
|
+
for event_id, items in occurrences.items():
|
|
1701
|
+
event_kinds = kinds.get(event_id, set())
|
|
1702
|
+
winner = selection.by_id.get(event_id)
|
|
1703
|
+
if winner is not None:
|
|
1704
|
+
selected[event_id] = (winner.rev, winner.content_hash)
|
|
1705
|
+
items, codes = _finalize_lineage_id(
|
|
1706
|
+
event_id, items, kinds=event_kinds,
|
|
1707
|
+
unhashable=event_id in unhashable_ids,
|
|
1708
|
+
selected=selected.get(event_id),
|
|
1709
|
+
hidden=event_id in hidden_ids,
|
|
1710
|
+
)
|
|
1711
|
+
if codes:
|
|
1712
|
+
uncertain[event_id] = codes
|
|
1713
|
+
if "five_hour_block_close" in event_kinds:
|
|
1714
|
+
closes[event_id] = items
|
|
1715
|
+
for item in items:
|
|
1716
|
+
if (item.kind == "five_hour_block_close"
|
|
1717
|
+
and item.block is not None):
|
|
1718
|
+
close_ids_ever_in_block.setdefault(
|
|
1719
|
+
item.block, set()).add(event_id)
|
|
1720
|
+
if "five_hour_milestone" in event_kinds:
|
|
1721
|
+
milestones[event_id] = items
|
|
1722
|
+
return FiveHourLineage(
|
|
1723
|
+
milestones=milestones,
|
|
1724
|
+
closes=closes,
|
|
1725
|
+
close_ids_ever_in_block={
|
|
1726
|
+
block: tuple(sorted(ids))
|
|
1727
|
+
for block, ids in close_ids_ever_in_block.items()
|
|
1728
|
+
},
|
|
1729
|
+
hidden_ids=hidden_ids,
|
|
1730
|
+
selected=selected,
|
|
1731
|
+
uncertain=uncertain,
|
|
1732
|
+
)
|
|
1733
|
+
|
|
1734
|
+
|
|
1735
|
+
@dataclasses.dataclass(frozen=True)
|
|
1736
|
+
class WithheldMilestone:
|
|
1737
|
+
"""One reported fail-closed milestone of a withheld block (§5.7)."""
|
|
1738
|
+
event_id: str
|
|
1739
|
+
current_revision: int
|
|
1740
|
+
candidate_revisions: tuple
|
|
1741
|
+
reason_codes: tuple
|
|
1742
|
+
|
|
1743
|
+
|
|
1744
|
+
@dataclasses.dataclass(frozen=True)
|
|
1745
|
+
class WithheldBlock:
|
|
1746
|
+
"""One five-hour block whose milestone repair was withheld (§5.7)."""
|
|
1747
|
+
account_key: object
|
|
1748
|
+
five_hour_window_key: object
|
|
1749
|
+
close_event_ids: tuple
|
|
1750
|
+
milestones: tuple
|
|
1751
|
+
|
|
1752
|
+
def to_dict(self) -> dict:
|
|
1753
|
+
return {
|
|
1754
|
+
"accountKey": self.account_key,
|
|
1755
|
+
"fiveHourWindowKey": self.five_hour_window_key,
|
|
1756
|
+
"closeEventIds": list(self.close_event_ids),
|
|
1757
|
+
"milestones": [
|
|
1758
|
+
{
|
|
1759
|
+
"eventId": milestone.event_id,
|
|
1760
|
+
"currentRevision": milestone.current_revision,
|
|
1761
|
+
"candidateRevisions": list(milestone.candidate_revisions),
|
|
1762
|
+
"reasonCodes": list(milestone.reason_codes),
|
|
1763
|
+
}
|
|
1764
|
+
for milestone in self.milestones
|
|
1765
|
+
],
|
|
1766
|
+
}
|
|
1767
|
+
|
|
1768
|
+
|
|
1769
|
+
def _withheld_blocks(withheld: Mapping, current: Mapping[str, Mapping],
|
|
1770
|
+
actions, selection) -> tuple:
|
|
1771
|
+
"""§5.7: group reported withholds by block, sorted, with the block's
|
|
1772
|
+
close ids in the plan's final state (retained, acted-on or preserved
|
|
1773
|
+
records; a revival is an acted-on record)."""
|
|
1774
|
+
if not withheld:
|
|
1775
|
+
return ()
|
|
1776
|
+
final = dict(current)
|
|
1777
|
+
for action in actions:
|
|
1778
|
+
if action.disposition == "tombstone":
|
|
1779
|
+
final.pop(action.event_id, None)
|
|
1780
|
+
else:
|
|
1781
|
+
final[action.event_id] = {"payload": action.payload or {}}
|
|
1782
|
+
close_ids: dict[tuple, set] = {}
|
|
1783
|
+
for event_id, record in final.items():
|
|
1784
|
+
payload = (record or {}).get("payload") or {}
|
|
1785
|
+
if payload.get("kind") == "five_hour_block_close":
|
|
1786
|
+
close_ids.setdefault(_five_hour_block(payload), set()).add(
|
|
1787
|
+
event_id)
|
|
1788
|
+
grouped: dict[tuple, list] = {}
|
|
1789
|
+
for event_id, decision in withheld.items():
|
|
1790
|
+
payload = (current.get(event_id) or {}).get("payload") or {}
|
|
1791
|
+
winner = selection.by_id.get(event_id)
|
|
1792
|
+
grouped.setdefault(_five_hour_block(payload), []).append(
|
|
1793
|
+
WithheldMilestone(
|
|
1794
|
+
event_id=event_id,
|
|
1795
|
+
current_revision=None if winner is None else int(winner.rev),
|
|
1796
|
+
candidate_revisions=tuple(sorted(
|
|
1797
|
+
decision.candidate_revisions)),
|
|
1798
|
+
reason_codes=tuple(sorted(set(decision.reason_codes))),
|
|
1799
|
+
))
|
|
1800
|
+
blocks = [
|
|
1801
|
+
WithheldBlock(
|
|
1802
|
+
account_key=block[0], five_hour_window_key=block[1],
|
|
1803
|
+
close_event_ids=tuple(sorted(close_ids.get(block, ()))),
|
|
1804
|
+
milestones=tuple(sorted(items, key=lambda item: item.event_id)),
|
|
1805
|
+
)
|
|
1806
|
+
for block, items in grouped.items()
|
|
1807
|
+
]
|
|
1808
|
+
blocks.sort(key=lambda item: (
|
|
1809
|
+
"" if item.account_key is None else str(item.account_key),
|
|
1810
|
+
str(item.five_hour_window_key),
|
|
1811
|
+
))
|
|
1812
|
+
return tuple(blocks)
|
|
1813
|
+
|
|
1814
|
+
|
|
729
1815
|
def _resolve_closed_block_milestones(
|
|
730
1816
|
current: Mapping[str, Mapping], desired: Mapping[str, Mapping],
|
|
731
1817
|
preserved: Mapping[str, Mapping], frozen_blocks, replaced_blocks, *,
|
|
732
|
-
selection,
|
|
1818
|
+
selection, lineage: "FiveHourLineage | None" = None,
|
|
1819
|
+
withheld: "dict | None" = None,
|
|
733
1820
|
) -> dict[str, dict]:
|
|
734
1821
|
"""#875: target records of the five-hour milestones of closed blocks.
|
|
735
1822
|
|
|
@@ -743,6 +1830,13 @@ def _resolve_closed_block_milestones(
|
|
|
743
1830
|
and non-monetary identity are both unchanged keeps its current
|
|
744
1831
|
``block_cost_usd`` across any card edit, and a crossing whose ``at`` or
|
|
745
1832
|
identity differs is a corrected crossing that takes replay's block cost.
|
|
1833
|
+
|
|
1834
|
+
#884 (spec §5.5): with ``lineage``, such an unchanged crossing instead
|
|
1835
|
+
takes its history-proven anchor amount when an earlier rederive repriced
|
|
1836
|
+
it (:meth:`FiveHourLineage.restoration`). When its history is ambiguous
|
|
1837
|
+
it keeps the current amount, and a reported withhold is recorded in
|
|
1838
|
+
``withheld`` as ``{event_id: Restoration}``. No other role is restored,
|
|
1839
|
+
and restoration never changes a block's frozen or replaced class.
|
|
746
1840
|
A REPLACED close is one this plan adds, or corrects so that it writes the
|
|
747
1841
|
close's money (it closes an open block, or its totals are unusable or
|
|
748
1842
|
differ by more), and every matched milestone under it is a corrected
|
|
@@ -765,13 +1859,15 @@ def _resolve_closed_block_milestones(
|
|
|
765
1859
|
crossing: replay never reproduces preserved ``b:`` predecessors, and a
|
|
766
1860
|
reviewed hold can remove a desired predecessor before the diff. It comes
|
|
767
1861
|
from the desired record for a corrected crossing or an addition. A
|
|
768
|
-
candidate keeps its
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
block cost is
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
1862
|
+
candidate keeps its chosen marginal (the current one for an unchanged
|
|
1863
|
+
crossing, the desired one otherwise) rather than inventing a value when
|
|
1864
|
+
it has no usable predecessor, or when its own threshold or block cost is
|
|
1865
|
+
unusable. Its block cost is already settled and is never reset there, so
|
|
1866
|
+
a restored amount survives (#884 spec §5.6). Only an ``int`` threshold
|
|
1867
|
+
occupies a position, and only a finite amount is usable. A row whose
|
|
1868
|
+
threshold is usable but whose block cost is not still occupies its
|
|
1869
|
+
threshold, so a successor right above it fails closed and never skips to
|
|
1870
|
+
a lower row.
|
|
775
1871
|
|
|
776
1872
|
A second plan is a no-op for a block whose close it takes no action on.
|
|
777
1873
|
That close is frozen in the next plan, kept or preserved, whatever role
|
|
@@ -833,6 +1929,16 @@ def _resolve_closed_block_milestones(
|
|
|
833
1929
|
payload["block_cost_usd"] = current_payload.get(
|
|
834
1930
|
"block_cost_usd")
|
|
835
1931
|
roles[event_id] = "unchanged"
|
|
1932
|
+
if lineage is not None:
|
|
1933
|
+
# #884 (spec §5.5): a proven historical amount replaces
|
|
1934
|
+
# one an earlier rederive repriced; an ambiguous history
|
|
1935
|
+
# keeps the current amount and is disclosed.
|
|
1936
|
+
decision = lineage.restoration(event_id, current_record)
|
|
1937
|
+
if decision.action == "restore":
|
|
1938
|
+
payload["block_cost_usd"] = decision.value
|
|
1939
|
+
elif (decision.action == "withhold" and decision.reported
|
|
1940
|
+
and withheld is not None):
|
|
1941
|
+
withheld[event_id] = decision
|
|
836
1942
|
else:
|
|
837
1943
|
roles[event_id] = "corrected"
|
|
838
1944
|
record = {**carried, "payload": payload}
|
|
@@ -891,7 +1997,8 @@ def _resolve_closed_block_milestones(
|
|
|
891
1997
|
):
|
|
892
1998
|
prior = predecessor_cost(payload)
|
|
893
1999
|
if prior is None:
|
|
894
|
-
|
|
2000
|
+
# #884 (spec §5.6): only the marginal falls back. The block cost
|
|
2001
|
+
# is already the chosen record's, or a restored amount.
|
|
895
2002
|
payload["marginal_cost_usd"] = chosen.get("marginal_cost_usd")
|
|
896
2003
|
continue
|
|
897
2004
|
value = float(payload["block_cost_usd"]) - float(prior)
|
|
@@ -907,6 +2014,769 @@ def _resolve_closed_block_milestones(
|
|
|
907
2014
|
return resolved
|
|
908
2015
|
|
|
909
2016
|
|
|
2017
|
+
# ── #1040: derivation completion keeps closed five-hour money frozen ───────
|
|
2018
|
+
#
|
|
2019
|
+
# Derivation completion journals, for every id the gap acts on (its with-gap
|
|
2020
|
+
# and without-gap candidates differ), the with-gap candidate. For a closed
|
|
2021
|
+
# five-hour block that candidate carries the money the scratch replay
|
|
2022
|
+
# computed at the binary's CURRENT card, so a metadata-only gap repriced the
|
|
2023
|
+
# frozen block (spec §1–§3). The rules below decide what completion appends
|
|
2024
|
+
# for the closes and milestones of every touched block instead (spec §5).
|
|
2025
|
+
|
|
2026
|
+
COMPLETION_FROZEN = "FROZEN"
|
|
2027
|
+
COMPLETION_REPLACED = "REPLACED"
|
|
2028
|
+
COMPLETION_ADDED = "ADDED"
|
|
2029
|
+
COMPLETION_OPEN = "OPEN"
|
|
2030
|
+
|
|
2031
|
+
|
|
2032
|
+
@dataclasses.dataclass(frozen=True)
|
|
2033
|
+
class CompletionClose:
|
|
2034
|
+
"""#1040 (spec §5.2): one close's proposed target and class.
|
|
2035
|
+
|
|
2036
|
+
``target`` is T (``None`` keeps nothing to append, or a tombstone on the
|
|
2037
|
+
OPEN path); ``frozen_form`` is T₀, defined for a close whose retained and
|
|
2038
|
+
with-gap records are both closed, else ``None``.
|
|
2039
|
+
"""
|
|
2040
|
+
block_class: str
|
|
2041
|
+
target: "Mapping | None"
|
|
2042
|
+
frozen_form: "Mapping | None" = None
|
|
2043
|
+
|
|
2044
|
+
|
|
2045
|
+
def _closed_record(record: "Mapping | None") -> bool:
|
|
2046
|
+
return (record is not None
|
|
2047
|
+
and (record.get("payload") or {}).get("is_closed") == 1)
|
|
2048
|
+
|
|
2049
|
+
|
|
2050
|
+
def _total_moves(before: "Mapping", after: "Mapping") -> bool:
|
|
2051
|
+
"""Both totals usable and more than 1e-9 USD apart."""
|
|
2052
|
+
left = (before.get("payload") or {}).get("total_cost_usd")
|
|
2053
|
+
right = (after.get("payload") or {}).get("total_cost_usd")
|
|
2054
|
+
return (_usable_amount(left) and _usable_amount(right)
|
|
2055
|
+
and abs(right - left) > _CLOSE_TOTAL_TOLERANCE_USD)
|
|
2056
|
+
|
|
2057
|
+
|
|
2058
|
+
def _is_missing_card_correction(preserved, current, desired) -> bool:
|
|
2059
|
+
"""§5.2: a preserved result that is neither D nor C and moves C's total.
|
|
2060
|
+
|
|
2061
|
+
``_with_close_marker`` builds a #869 missing-card correction as a new
|
|
2062
|
+
record under the same structure, so identity tells it apart from a kept
|
|
2063
|
+
(``current``) or structurally different (``desired``) result.
|
|
2064
|
+
"""
|
|
2065
|
+
return (preserved is not current and preserved is not desired
|
|
2066
|
+
and _total_moves(current, preserved))
|
|
2067
|
+
|
|
2068
|
+
|
|
2069
|
+
def classify_completion_close(
|
|
2070
|
+
current: "Mapping | None", desired: "Mapping | None",
|
|
2071
|
+
reference: "Mapping | None", *, acted: bool, has_physical: bool,
|
|
2072
|
+
several_close_ids: bool,
|
|
2073
|
+
) -> CompletionClose:
|
|
2074
|
+
"""#1040 (spec §5.2): the proposed close target T and the block class.
|
|
2075
|
+
|
|
2076
|
+
``current`` is C (the selected active close, or ``None``), ``desired`` D
|
|
2077
|
+
and ``reference`` R. The rules apply in order:
|
|
2078
|
+
|
|
2079
|
+
1. not acted on: T = C, FROZEN when C is closed, else OPEN;
|
|
2080
|
+
2. D absent or not closed, or C present and not closed: OPEN (today's
|
|
2081
|
+
path; no closed money exists to keep). OPEN is this close's class:
|
|
2082
|
+
the block is OPEN only when every close of it is (spec §5.2 rule 2,
|
|
2083
|
+
"no closed close in the final state");
|
|
2084
|
+
3. D closed, no C: ADDED (plain revision 0) without a physical revision,
|
|
2085
|
+
else a REPLACED revival;
|
|
2086
|
+
4. C and D closed: T₀ first (``overlay(D, C)`` when preservation returns
|
|
2087
|
+
D, else C), then 4.1 a gap-caused population change, 4.2 a #869
|
|
2088
|
+
missing-card correction P, 4.3 the same-round P₂ = preserve(T₀, D),
|
|
2089
|
+
4.4 FROZEN with T = T₀.
|
|
2090
|
+
|
|
2091
|
+
A block naming several close ids is never REPLACED: rules 4.1–4.3 yield
|
|
2092
|
+
T₀ and FROZEN, and a revival proposes D under FROZEN, whose lineage check
|
|
2093
|
+
decides whether it can land.
|
|
2094
|
+
"""
|
|
2095
|
+
if not acted:
|
|
2096
|
+
return CompletionClose(
|
|
2097
|
+
COMPLETION_FROZEN if _closed_record(current) else COMPLETION_OPEN,
|
|
2098
|
+
current)
|
|
2099
|
+
if not _closed_record(desired) or (
|
|
2100
|
+
current is not None and not _closed_record(current)):
|
|
2101
|
+
return CompletionClose(COMPLETION_OPEN, desired)
|
|
2102
|
+
if current is None:
|
|
2103
|
+
if not has_physical:
|
|
2104
|
+
return CompletionClose(COMPLETION_ADDED, desired)
|
|
2105
|
+
if several_close_ids:
|
|
2106
|
+
return CompletionClose(COMPLETION_FROZEN, desired)
|
|
2107
|
+
return CompletionClose(COMPLETION_REPLACED, desired)
|
|
2108
|
+
preserved = _preserve_non_derivable_state(current, desired)
|
|
2109
|
+
frozen = (close_money_overlay(desired, current)
|
|
2110
|
+
if preserved is desired else current)
|
|
2111
|
+
if several_close_ids:
|
|
2112
|
+
return CompletionClose(COMPLETION_FROZEN, frozen, frozen)
|
|
2113
|
+
current_payload = current.get("payload") or {}
|
|
2114
|
+
desired_payload = desired.get("payload") or {}
|
|
2115
|
+
if (
|
|
2116
|
+
_closed_record(reference)
|
|
2117
|
+
and _usable_amount(current_payload.get("total_cost_usd"))
|
|
2118
|
+
and _usable_amount(desired_payload.get("total_cost_usd"))
|
|
2119
|
+
and close_population(current_payload)
|
|
2120
|
+
== close_population(reference.get("payload") or {})
|
|
2121
|
+
and close_population(desired_payload)
|
|
2122
|
+
!= close_population(current_payload)
|
|
2123
|
+
and _total_moves(current, desired)
|
|
2124
|
+
):
|
|
2125
|
+
return CompletionClose(COMPLETION_REPLACED, desired, frozen)
|
|
2126
|
+
if _is_missing_card_correction(preserved, current, desired):
|
|
2127
|
+
return CompletionClose(COMPLETION_REPLACED, preserved, frozen)
|
|
2128
|
+
if _semantic_event(frozen) != _semantic_event(current):
|
|
2129
|
+
second = _preserve_non_derivable_state(frozen, desired)
|
|
2130
|
+
if _is_missing_card_correction(second, frozen, desired):
|
|
2131
|
+
return CompletionClose(COMPLETION_REPLACED, second, frozen)
|
|
2132
|
+
return CompletionClose(COMPLETION_FROZEN, frozen, frozen)
|
|
2133
|
+
|
|
2134
|
+
#: The milestone fields a FROZEN or ADDED block's acted crossing keeps from
|
|
2135
|
+
#: its retained record (spec §5.3): its money and the block tokens it was
|
|
2136
|
+
#: priced from.
|
|
2137
|
+
_COMPLETION_MILESTONE_KEPT_KEYS = (
|
|
2138
|
+
"block_cost_usd", "marginal_cost_usd",
|
|
2139
|
+
*(f"block_{key}" for key in _BLOCK_TOKEN_KEYS),
|
|
2140
|
+
)
|
|
2141
|
+
|
|
2142
|
+
COMPLETION_TREATMENTS = ("frozen", "replaced", "marginal", "none", "main")
|
|
2143
|
+
|
|
2144
|
+
|
|
2145
|
+
def _milestone_overlay(desired: Mapping, current: Mapping) -> dict:
|
|
2146
|
+
"""D with C's money and block tokens (a key C lacks is removed)."""
|
|
2147
|
+
payload = dict(desired.get("payload") or {})
|
|
2148
|
+
current_payload = current.get("payload") or {}
|
|
2149
|
+
for key in _COMPLETION_MILESTONE_KEPT_KEYS:
|
|
2150
|
+
if key in current_payload:
|
|
2151
|
+
payload[key] = current_payload[key]
|
|
2152
|
+
else:
|
|
2153
|
+
payload.pop(key, None)
|
|
2154
|
+
return {**desired, "payload": payload}
|
|
2155
|
+
|
|
2156
|
+
|
|
2157
|
+
def completion_block_milestones(
|
|
2158
|
+
*, treatment: str, block_class: str, current: Mapping[str, Mapping],
|
|
2159
|
+
with_gap: Mapping[str, Mapping], acted: frozenset, retired: frozenset,
|
|
2160
|
+
lineage: FiveHourLineage, selection,
|
|
2161
|
+
fixed: "Mapping[str, Mapping] | None" = None,
|
|
2162
|
+
) -> dict[str, dict]:
|
|
2163
|
+
"""#1040 (spec §5.3, §5.4): one block's milestone targets.
|
|
2164
|
+
|
|
2165
|
+
``current`` holds the block's selected active milestones (the whole
|
|
2166
|
+
journal's, so predecessors before a trusted frontier are included) and
|
|
2167
|
+
``with_gap`` the block's with-gap milestone candidates. ``retired`` are
|
|
2168
|
+
the acted milestones with a retained record and no with-gap one: today's
|
|
2169
|
+
path tombstones them, so they are no predecessor in any final chain.
|
|
2170
|
+
``fixed`` are records of ids the planner does not own that today's path
|
|
2171
|
+
leaves in this block: they stand in the final chain as given and are
|
|
2172
|
+
never returned. Returns every resolved record that differs semantically
|
|
2173
|
+
from its retained record or has none.
|
|
2174
|
+
|
|
2175
|
+
A **required addition** is an acted with-gap id absent from
|
|
2176
|
+
``selection.by_id`` (spec §5.4): no admitted decision, active or
|
|
2177
|
+
tombstoned, answers it. An acted with-gap id without C that the
|
|
2178
|
+
selection does know (a tombstoned milestone, revived) is an optional
|
|
2179
|
+
rewrite. ``treatment``:
|
|
2180
|
+
|
|
2181
|
+
* ``"frozen"``: an acted crossing with C takes D with C's money and
|
|
2182
|
+
block tokens; when that is a corrected crossing (its ``at`` or
|
|
2183
|
+
identity differs) its block cost is the lineage's restoration value
|
|
2184
|
+
when :meth:`FiveHourLineage.restoration` says ``restore``. An acted id
|
|
2185
|
+
without C takes D (required or revived), every other id C.
|
|
2186
|
+
* ``"replaced"`` (REPLACED blocks only): an id the with-gap replay
|
|
2187
|
+
carries takes D, acted or not; a current id it does not carry takes C.
|
|
2188
|
+
* ``"marginal"``: every existing id keeps C; required additions take D;
|
|
2189
|
+
only the marginals are resolved over the final chain.
|
|
2190
|
+
* ``"none"``: the required additions alone, their marginals resolved
|
|
2191
|
+
over the chain whose existing ids are C.
|
|
2192
|
+
* ``"main"``: each acted id's record as today's completion path appends
|
|
2193
|
+
it (``_preserve_non_derivable_state(C, D)``, or D), nothing else and
|
|
2194
|
+
no marginal resolution.
|
|
2195
|
+
|
|
2196
|
+
The kernel is :func:`_resolve_closed_block_milestones` with the block
|
|
2197
|
+
FROZEN (``replaced`` treatment: REPLACED) and no lineage.
|
|
2198
|
+
"""
|
|
2199
|
+
if treatment not in COMPLETION_TREATMENTS:
|
|
2200
|
+
raise ValueError(f"unknown milestone treatment {treatment!r}")
|
|
2201
|
+
if treatment == "replaced" and block_class != COMPLETION_REPLACED:
|
|
2202
|
+
raise ValueError("the replaced treatment needs a REPLACED block")
|
|
2203
|
+
fixed = dict(fixed or {})
|
|
2204
|
+
if treatment == "main":
|
|
2205
|
+
out = {}
|
|
2206
|
+
for event_id in sorted(acted & set(with_gap)):
|
|
2207
|
+
desired = with_gap[event_id]
|
|
2208
|
+
existing = current.get(event_id)
|
|
2209
|
+
record = (desired if existing is None
|
|
2210
|
+
else _preserve_non_derivable_state(existing, desired))
|
|
2211
|
+
if existing is None or (_semantic_event(record)
|
|
2212
|
+
!= _semantic_event(existing)):
|
|
2213
|
+
out[event_id] = dict(record)
|
|
2214
|
+
return out
|
|
2215
|
+
desired: dict[str, Mapping] = {}
|
|
2216
|
+
for event_id, existing in current.items():
|
|
2217
|
+
if event_id in retired:
|
|
2218
|
+
continue
|
|
2219
|
+
candidate = with_gap.get(event_id)
|
|
2220
|
+
if treatment == "replaced" and candidate is not None:
|
|
2221
|
+
desired[event_id] = candidate
|
|
2222
|
+
elif (treatment == "frozen" and candidate is not None
|
|
2223
|
+
and event_id in acted):
|
|
2224
|
+
record = _milestone_overlay(candidate, existing)
|
|
2225
|
+
existing_payload = existing.get("payload") or {}
|
|
2226
|
+
if (record.get("at") != existing.get("at")
|
|
2227
|
+
or _lineage_identity(record["payload"])
|
|
2228
|
+
!= _lineage_identity(existing_payload)):
|
|
2229
|
+
decision = lineage.restoration(event_id, existing)
|
|
2230
|
+
if decision.action == "restore":
|
|
2231
|
+
record["payload"]["block_cost_usd"] = decision.value
|
|
2232
|
+
desired[event_id] = record
|
|
2233
|
+
else:
|
|
2234
|
+
desired[event_id] = existing
|
|
2235
|
+
required = set()
|
|
2236
|
+
for event_id, candidate in with_gap.items():
|
|
2237
|
+
if event_id in current or event_id not in acted:
|
|
2238
|
+
continue
|
|
2239
|
+
if event_id not in selection.by_id:
|
|
2240
|
+
required.add(event_id)
|
|
2241
|
+
elif treatment in ("marginal", "none"):
|
|
2242
|
+
continue # an optional revival: only F, K and R rewrite
|
|
2243
|
+
desired[event_id] = candidate
|
|
2244
|
+
desired.update(fixed)
|
|
2245
|
+
if not desired:
|
|
2246
|
+
return {}
|
|
2247
|
+
block = _five_hour_block(
|
|
2248
|
+
(next(iter(desired.values())).get("payload") or {}))
|
|
2249
|
+
replaced = treatment == "replaced"
|
|
2250
|
+
resolved = _resolve_closed_block_milestones(
|
|
2251
|
+
# A fixed record is today's: it stands in the chain at its own money.
|
|
2252
|
+
{event_id: record for event_id, record in current.items()
|
|
2253
|
+
if event_id not in fixed},
|
|
2254
|
+
desired, {},
|
|
2255
|
+
frozenset() if replaced else frozenset({block}),
|
|
2256
|
+
frozenset({block}) if replaced else frozenset(),
|
|
2257
|
+
selection=selection, lineage=None,
|
|
2258
|
+
)
|
|
2259
|
+
out = {}
|
|
2260
|
+
for event_id in sorted(resolved):
|
|
2261
|
+
if event_id in fixed:
|
|
2262
|
+
continue
|
|
2263
|
+
if treatment == "none" and event_id not in required:
|
|
2264
|
+
continue
|
|
2265
|
+
existing = current.get(event_id)
|
|
2266
|
+
record = resolved[event_id]
|
|
2267
|
+
if existing is None or (_semantic_event(record)
|
|
2268
|
+
!= _semantic_event(existing)):
|
|
2269
|
+
out[event_id] = record
|
|
2270
|
+
return out
|
|
2271
|
+
|
|
2272
|
+
@dataclasses.dataclass(frozen=True)
|
|
2273
|
+
class BlockDecision:
|
|
2274
|
+
"""#1040 (spec §5.4): one touched block's lineage-checked decision.
|
|
2275
|
+
|
|
2276
|
+
``landed`` names the candidate whose records land (``"class"``, ``"F"``,
|
|
2277
|
+
``"P"``, ``"K"``, ``"M"`` or ``"A"``); ``violations`` maps every
|
|
2278
|
+
candidate the decision simulated to its violation keys.
|
|
2279
|
+
"""
|
|
2280
|
+
block: tuple
|
|
2281
|
+
block_class: str
|
|
2282
|
+
component: tuple
|
|
2283
|
+
landed: str
|
|
2284
|
+
violations: Mapping[str, frozenset]
|
|
2285
|
+
|
|
2286
|
+
|
|
2287
|
+
@dataclasses.dataclass(frozen=True)
|
|
2288
|
+
class CompletionFiveHourPlan:
|
|
2289
|
+
"""#1040: what derivation completion appends for touched blocks.
|
|
2290
|
+
|
|
2291
|
+
``targets`` maps an id to the record to append (each differs from its
|
|
2292
|
+
retained record or has none); ``handled`` are the five-hour ids whose
|
|
2293
|
+
every record lies in a decided block, retirements and the acted closes
|
|
2294
|
+
left to today's path excluded, which today's per-id path must not append
|
|
2295
|
+
again (``targets`` is a subset); ``decisions`` are sorted by block.
|
|
2296
|
+
"""
|
|
2297
|
+
targets: Mapping[str, Mapping]
|
|
2298
|
+
handled: frozenset
|
|
2299
|
+
decisions: tuple
|
|
2300
|
+
|
|
2301
|
+
|
|
2302
|
+
#: §5.4: the candidates of each block class, in decision order.
|
|
2303
|
+
_COMPLETION_CANDIDATE_ORDER = {
|
|
2304
|
+
COMPLETION_FROZEN: ("class", "K", "M", "A"),
|
|
2305
|
+
COMPLETION_REPLACED: ("class", "F", "K", "M", "A"),
|
|
2306
|
+
COMPLETION_ADDED: ("class", "P", "M", "A"),
|
|
2307
|
+
}
|
|
2308
|
+
#: §5.4: each later candidate's milestone treatment.
|
|
2309
|
+
_COMPLETION_CANDIDATE_TREATMENT = {
|
|
2310
|
+
"F": "frozen", "P": "main", "K": "frozen", "M": "marginal", "A": "none",
|
|
2311
|
+
}
|
|
2312
|
+
_RESTORATION_RANK = {"keep": 0, "restore": 1, "withhold": 2}
|
|
2313
|
+
|
|
2314
|
+
|
|
2315
|
+
def _selected_milestone_view(lineage: FiveHourLineage, event_id: str) -> dict:
|
|
2316
|
+
"""The selected active occurrence's block cost, as the record
|
|
2317
|
+
:meth:`FiveHourLineage.restoration` reads (``{}`` when there is none)."""
|
|
2318
|
+
pair = lineage.selected.get(event_id)
|
|
2319
|
+
for occurrence in lineage.milestones.get(event_id, ()):
|
|
2320
|
+
if (pair is not None and occurrence.status == "active"
|
|
2321
|
+
and (occurrence.rev, occurrence.content_hash) == pair):
|
|
2322
|
+
return {"payload": {"block_cost_usd": occurrence.block_cost_usd}}
|
|
2323
|
+
return {"payload": {}}
|
|
2324
|
+
|
|
2325
|
+
|
|
2326
|
+
def _violation_keys(before: FiveHourLineage, after: FiveHourLineage,
|
|
2327
|
+
milestone_ids, records: Mapping[str, Mapping]) -> frozenset:
|
|
2328
|
+
"""#1040 (spec §5.4): how a candidate makes #884 decisions worse.
|
|
2329
|
+
|
|
2330
|
+
For each milestone, its decision in ``after`` against its decision in
|
|
2331
|
+
``before``; a milestone ``before`` does not know counts as ``keep`` with
|
|
2332
|
+
no reason. ``records`` are the candidate's records by id; a milestone it
|
|
2333
|
+
does not name, and every ``before`` decision, reads the lineage's
|
|
2334
|
+
selected occurrence. Keys: ``(id, "action", a)`` when the action rises
|
|
2335
|
+
to ``a`` (``keep`` < ``restore`` < ``withhold``), ``(id, "reason",
|
|
2336
|
+
code)`` per gained reason code, and ``(id, "moved")`` when both are
|
|
2337
|
+
``restore`` to different values.
|
|
2338
|
+
"""
|
|
2339
|
+
keys = set()
|
|
2340
|
+
for event_id in sorted(milestone_ids):
|
|
2341
|
+
old = before.restoration(
|
|
2342
|
+
event_id, _selected_milestone_view(before, event_id))
|
|
2343
|
+
record = records.get(event_id)
|
|
2344
|
+
new = after.restoration(
|
|
2345
|
+
event_id, record if record is not None
|
|
2346
|
+
else _selected_milestone_view(after, event_id))
|
|
2347
|
+
if _RESTORATION_RANK[new.action] > _RESTORATION_RANK[old.action]:
|
|
2348
|
+
keys.add((event_id, "action", new.action))
|
|
2349
|
+
for code in set(new.reason_codes) - set(old.reason_codes):
|
|
2350
|
+
keys.add((event_id, "reason", code))
|
|
2351
|
+
if (old.action == new.action == "restore"
|
|
2352
|
+
and old.value != new.value):
|
|
2353
|
+
keys.add((event_id, "moved"))
|
|
2354
|
+
return frozenset(keys)
|
|
2355
|
+
|
|
2356
|
+
|
|
2357
|
+
def _hashable_block(payload: Mapping):
|
|
2358
|
+
block = _five_hour_block(payload)
|
|
2359
|
+
try:
|
|
2360
|
+
hash(block)
|
|
2361
|
+
except TypeError:
|
|
2362
|
+
return None
|
|
2363
|
+
return block
|
|
2364
|
+
|
|
2365
|
+
|
|
2366
|
+
def plan_completion_five_hour(
|
|
2367
|
+
*, current: Mapping[str, Mapping], with_gap: Mapping[str, Mapping],
|
|
2368
|
+
without_gap: Mapping[str, Mapping], acted: frozenset,
|
|
2369
|
+
physical: Mapping[str, int], selection, lineage: FiveHourLineage,
|
|
2370
|
+
retirements: Sequence[Mapping],
|
|
2371
|
+
) -> CompletionFiveHourPlan:
|
|
2372
|
+
"""#1040 (spec §5.1–§5.5): completion's five-hour targets, pure.
|
|
2373
|
+
|
|
2374
|
+
``current`` holds the selected active five-hour records of the whole
|
|
2375
|
+
journal; ``with_gap`` / ``without_gap`` are the round's candidates by
|
|
2376
|
+
id (any kind) and ``acted`` the ids whose two candidates differ;
|
|
2377
|
+
``physical`` maps an id to its highest physical revision (selected,
|
|
2378
|
+
admission-excluded or inert); ``retirements`` are the tombstone actions
|
|
2379
|
+
today's path appends, which land whatever is chosen and so ride along in
|
|
2380
|
+
every simulation.
|
|
2381
|
+
|
|
2382
|
+
A block is touched when an acted close or milestone names it. Each
|
|
2383
|
+
touched block's closes are classified (:func:`classify_completion_close`).
|
|
2384
|
+
A block is OPEN, and keeps today's path, when no close it decides is
|
|
2385
|
+
closed in the final state; in a block that stays decided an acted close
|
|
2386
|
+
of rule 2 still keeps today's path. The planner owns an id only when
|
|
2387
|
+
every block its retained, with-gap and without-gap records name is
|
|
2388
|
+
decided (``handled``); it never targets another id, and every
|
|
2389
|
+
simulation carries today's record of each acted id it leaves to that
|
|
2390
|
+
path, like the retirements. Every decided block builds its whole
|
|
2391
|
+
close-and-milestone candidates, each carrying the block's required
|
|
2392
|
+
additions (records whose id the selection does not know), and the
|
|
2393
|
+
candidates of a component of blocks that share a close or milestone id
|
|
2394
|
+
are simulated through :meth:`FiveHourLineage.extended` with the append
|
|
2395
|
+
policy's coordinates. A component with one touched block lands the
|
|
2396
|
+
first candidate whose violations are contained in its additions' alone
|
|
2397
|
+
and strictly exceed no other candidate's; a class proposal without
|
|
2398
|
+
violations lands at once. Several touched blocks choose likewise with
|
|
2399
|
+
each other's additions carried, and their union lands only when it adds
|
|
2400
|
+
nothing beyond all their additions together; otherwise each lands its
|
|
2401
|
+
additions alone.
|
|
2402
|
+
"""
|
|
2403
|
+
def payload_of(record):
|
|
2404
|
+
return (record or {}).get("payload") or {}
|
|
2405
|
+
|
|
2406
|
+
def five_hour(record):
|
|
2407
|
+
return (record is not None
|
|
2408
|
+
and payload_of(record).get("kind") in _LINEAGE_KINDS)
|
|
2409
|
+
|
|
2410
|
+
def by_block(records):
|
|
2411
|
+
out: dict = {}
|
|
2412
|
+
for event_id, record in records.items():
|
|
2413
|
+
if five_hour(record):
|
|
2414
|
+
block = _hashable_block(payload_of(record))
|
|
2415
|
+
if block is not None:
|
|
2416
|
+
out.setdefault(block, {})[event_id] = record
|
|
2417
|
+
return out
|
|
2418
|
+
|
|
2419
|
+
current_by_block = by_block(current)
|
|
2420
|
+
with_by_block = by_block(with_gap)
|
|
2421
|
+
without_by_block = by_block(without_gap)
|
|
2422
|
+
touched = set()
|
|
2423
|
+
for event_id in acted:
|
|
2424
|
+
for record in (with_gap.get(event_id), current.get(event_id)):
|
|
2425
|
+
if five_hour(record):
|
|
2426
|
+
block = _hashable_block(payload_of(record))
|
|
2427
|
+
if block is not None:
|
|
2428
|
+
touched.add(block)
|
|
2429
|
+
|
|
2430
|
+
# Every block an id's retained, with-gap or without-gap record names
|
|
2431
|
+
# (an unusable block key stands for a block nothing decides).
|
|
2432
|
+
unusable_block = object()
|
|
2433
|
+
blocks_of_id: dict = {}
|
|
2434
|
+
for table in (current, with_gap, without_gap):
|
|
2435
|
+
for event_id, record in table.items():
|
|
2436
|
+
if five_hour(record):
|
|
2437
|
+
block = _hashable_block(payload_of(record))
|
|
2438
|
+
blocks_of_id.setdefault(event_id, set()).add(
|
|
2439
|
+
unusable_block if block is None else block)
|
|
2440
|
+
|
|
2441
|
+
def todays_record(event_id):
|
|
2442
|
+
"""The record today's per-id path leaves active for ``event_id``."""
|
|
2443
|
+
existing = current.get(event_id)
|
|
2444
|
+
if event_id not in acted:
|
|
2445
|
+
return existing
|
|
2446
|
+
desired = with_gap.get(event_id)
|
|
2447
|
+
if desired is None:
|
|
2448
|
+
return None # a retirement tombstone, or nothing
|
|
2449
|
+
if existing is None:
|
|
2450
|
+
return desired
|
|
2451
|
+
return _preserve_non_derivable_state(existing, desired)
|
|
2452
|
+
|
|
2453
|
+
def classify_blocks(not_owned):
|
|
2454
|
+
"""§5.2 per touched block: its class and the closes the planner
|
|
2455
|
+
decides. A block is OPEN only when no close it decides is closed in
|
|
2456
|
+
the final state; an acted close of rule 2 (and an acted close the
|
|
2457
|
+
planner does not own) keeps today's path."""
|
|
2458
|
+
classes, owned_closes, todays_closes = {}, {}, set()
|
|
2459
|
+
for block in sorted(touched, key=repr):
|
|
2460
|
+
close_ids = sorted({
|
|
2461
|
+
event_id for records in (
|
|
2462
|
+
current_by_block.get(block, {}),
|
|
2463
|
+
with_by_block.get(block, {}),
|
|
2464
|
+
without_by_block.get(block, {}))
|
|
2465
|
+
for event_id, record in records.items()
|
|
2466
|
+
if payload_of(record).get("kind") == "five_hour_block_close"
|
|
2467
|
+
})
|
|
2468
|
+
several = len(close_ids) > 1
|
|
2469
|
+
closes = {}
|
|
2470
|
+
for close_id in close_ids:
|
|
2471
|
+
close = classify_completion_close(
|
|
2472
|
+
current.get(close_id), with_gap.get(close_id),
|
|
2473
|
+
without_gap.get(close_id), acted=close_id in acted,
|
|
2474
|
+
has_physical=close_id in physical,
|
|
2475
|
+
several_close_ids=several)
|
|
2476
|
+
if close_id in acted and (
|
|
2477
|
+
close.block_class == COMPLETION_OPEN
|
|
2478
|
+
or close_id in not_owned):
|
|
2479
|
+
todays_closes.add(close_id)
|
|
2480
|
+
elif close.block_class != COMPLETION_OPEN:
|
|
2481
|
+
closes[close_id] = close
|
|
2482
|
+
if not closes:
|
|
2483
|
+
classes[block] = COMPLETION_OPEN
|
|
2484
|
+
continue
|
|
2485
|
+
kinds = {close.block_class for close in closes.values()}
|
|
2486
|
+
if len(closes) == 1:
|
|
2487
|
+
classes[block] = next(iter(kinds))
|
|
2488
|
+
elif kinds == {COMPLETION_ADDED}:
|
|
2489
|
+
classes[block] = COMPLETION_ADDED
|
|
2490
|
+
else:
|
|
2491
|
+
classes[block] = COMPLETION_FROZEN
|
|
2492
|
+
owned_closes[block] = closes
|
|
2493
|
+
return classes, owned_closes, todays_closes
|
|
2494
|
+
|
|
2495
|
+
# An id whose records reach a block the planner does not decide stays
|
|
2496
|
+
# on today's path (the planner never writes it); deciding fewer blocks
|
|
2497
|
+
# only adds such ids, so this settles.
|
|
2498
|
+
not_owned: set = set()
|
|
2499
|
+
while True:
|
|
2500
|
+
classes, owned_closes, todays_closes = classify_blocks(not_owned)
|
|
2501
|
+
decided_blocks = set(owned_closes)
|
|
2502
|
+
reaching = {
|
|
2503
|
+
event_id for event_id, blocks in blocks_of_id.items()
|
|
2504
|
+
if blocks & decided_blocks and not blocks <= decided_blocks}
|
|
2505
|
+
if reaching <= not_owned:
|
|
2506
|
+
break
|
|
2507
|
+
not_owned |= reaching
|
|
2508
|
+
not_owned |= todays_closes
|
|
2509
|
+
|
|
2510
|
+
candidates: dict = {}
|
|
2511
|
+
retired_ids: set = set()
|
|
2512
|
+
for block in sorted(owned_closes, key=repr):
|
|
2513
|
+
closes = owned_closes[block]
|
|
2514
|
+
block_class = classes[block]
|
|
2515
|
+
cur = current_by_block.get(block, {})
|
|
2516
|
+
wg = with_by_block.get(block, {})
|
|
2517
|
+
milestones_current = {
|
|
2518
|
+
event_id: record for event_id, record in cur.items()
|
|
2519
|
+
if payload_of(record).get("kind") == "five_hour_milestone"}
|
|
2520
|
+
milestones_with = {
|
|
2521
|
+
event_id: record for event_id, record in wg.items()
|
|
2522
|
+
if payload_of(record).get("kind") == "five_hour_milestone"
|
|
2523
|
+
and event_id not in not_owned}
|
|
2524
|
+
retired = frozenset(
|
|
2525
|
+
event_id for event_id in milestones_current
|
|
2526
|
+
if event_id in acted and with_gap.get(event_id) is None)
|
|
2527
|
+
retired_ids.update(retired)
|
|
2528
|
+
# Milestones of this block the planner does not own: today's record
|
|
2529
|
+
# stands in the chain when it stays here, else it leaves the chain.
|
|
2530
|
+
fixed: dict = {}
|
|
2531
|
+
leaving = set()
|
|
2532
|
+
for event_id in sorted(not_owned):
|
|
2533
|
+
if block not in blocks_of_id.get(event_id, ()):
|
|
2534
|
+
continue
|
|
2535
|
+
record = todays_record(event_id)
|
|
2536
|
+
if (record is not None and payload_of(record).get("kind")
|
|
2537
|
+
== "five_hour_milestone"
|
|
2538
|
+
and _hashable_block(payload_of(record)) == block):
|
|
2539
|
+
fixed[event_id] = record
|
|
2540
|
+
elif event_id in milestones_current:
|
|
2541
|
+
leaving.add(event_id)
|
|
2542
|
+
retired_chain = retired | frozenset(leaving)
|
|
2543
|
+
|
|
2544
|
+
def candidate(name, *, closes=closes, block_class=block_class,
|
|
2545
|
+
milestones_current=milestones_current,
|
|
2546
|
+
milestones_with=milestones_with,
|
|
2547
|
+
retired_chain=retired_chain, fixed=fixed):
|
|
2548
|
+
records: dict = {}
|
|
2549
|
+
for close_id, close in closes.items():
|
|
2550
|
+
existing = current.get(close_id)
|
|
2551
|
+
desired = with_gap.get(close_id)
|
|
2552
|
+
if name == "class":
|
|
2553
|
+
target = close.target
|
|
2554
|
+
elif name == "F":
|
|
2555
|
+
target = (close.frozen_form if existing is not None
|
|
2556
|
+
else desired if close_id in acted else None)
|
|
2557
|
+
elif name == "P":
|
|
2558
|
+
target = desired if close_id in acted else None
|
|
2559
|
+
else:
|
|
2560
|
+
target = None
|
|
2561
|
+
if (close_id in acted and desired is not None
|
|
2562
|
+
and close_id not in selection.by_id):
|
|
2563
|
+
target = desired # a required addition
|
|
2564
|
+
if target is not None and (
|
|
2565
|
+
existing is None
|
|
2566
|
+
or _semantic_event(target) != _semantic_event(existing)):
|
|
2567
|
+
records[close_id] = dict(target)
|
|
2568
|
+
treatment = (
|
|
2569
|
+
("replaced" if block_class == COMPLETION_REPLACED
|
|
2570
|
+
else "frozen")
|
|
2571
|
+
if name == "class" else _COMPLETION_CANDIDATE_TREATMENT[name])
|
|
2572
|
+
records.update(completion_block_milestones(
|
|
2573
|
+
treatment=treatment, block_class=block_class,
|
|
2574
|
+
current=milestones_current, with_gap=milestones_with,
|
|
2575
|
+
acted=acted, retired=retired_chain, lineage=lineage,
|
|
2576
|
+
selection=selection, fixed=fixed))
|
|
2577
|
+
return records
|
|
2578
|
+
|
|
2579
|
+
candidates[block] = {
|
|
2580
|
+
name: candidate(name)
|
|
2581
|
+
for name in _COMPLETION_CANDIDATE_ORDER[block_class]}
|
|
2582
|
+
|
|
2583
|
+
decided = sorted(candidates, key=repr)
|
|
2584
|
+
# The planner owns an id only when every block its records name is
|
|
2585
|
+
# decided; today's path keeps every other id (spec §5.1).
|
|
2586
|
+
handled = {
|
|
2587
|
+
event_id for event_id, blocks in blocks_of_id.items()
|
|
2588
|
+
if blocks <= decided_blocks and event_id not in not_owned
|
|
2589
|
+
and event_id not in retired_ids}
|
|
2590
|
+
if not decided:
|
|
2591
|
+
return CompletionFiveHourPlan(
|
|
2592
|
+
targets={}, handled=frozenset(handled), decisions=())
|
|
2593
|
+
# Today's records of the acted ids the planner leaves to that path in a
|
|
2594
|
+
# decided block's reach: they land whatever is chosen, so every
|
|
2595
|
+
# candidate's simulation carries them (like the retirements).
|
|
2596
|
+
todays_records: dict = {}
|
|
2597
|
+
for event_id in sorted(not_owned & acted):
|
|
2598
|
+
if not blocks_of_id.get(event_id, set()) & decided_blocks:
|
|
2599
|
+
continue
|
|
2600
|
+
record = todays_record(event_id)
|
|
2601
|
+
existing = current.get(event_id)
|
|
2602
|
+
if record is not None and (
|
|
2603
|
+
existing is None
|
|
2604
|
+
or _semantic_event(record) != _semantic_event(existing)):
|
|
2605
|
+
todays_records[event_id] = dict(record)
|
|
2606
|
+
|
|
2607
|
+
# §5.4 components: blocks sharing a close or milestone id, over the
|
|
2608
|
+
# lineage's occurrences and every candidate's records.
|
|
2609
|
+
id_blocks: dict = {}
|
|
2610
|
+
block_ids: dict = {}
|
|
2611
|
+
|
|
2612
|
+
def link(event_id, block):
|
|
2613
|
+
if block is None:
|
|
2614
|
+
return
|
|
2615
|
+
id_blocks.setdefault(event_id, set()).add(block)
|
|
2616
|
+
block_ids.setdefault(block, set()).add(event_id)
|
|
2617
|
+
|
|
2618
|
+
for table in (lineage.milestones, lineage.closes):
|
|
2619
|
+
for event_id, items in table.items():
|
|
2620
|
+
for occurrence in items:
|
|
2621
|
+
if occurrence.status == "active" and occurrence.block is not None:
|
|
2622
|
+
try:
|
|
2623
|
+
hash(occurrence.block)
|
|
2624
|
+
except TypeError:
|
|
2625
|
+
continue
|
|
2626
|
+
link(event_id, occurrence.block)
|
|
2627
|
+
for block in decided:
|
|
2628
|
+
for records in candidates[block].values():
|
|
2629
|
+
for event_id, record in records.items():
|
|
2630
|
+
link(event_id, _hashable_block(payload_of(record)))
|
|
2631
|
+
for event_id, record in todays_records.items():
|
|
2632
|
+
link(event_id, _hashable_block(payload_of(record)))
|
|
2633
|
+
components: dict = {}
|
|
2634
|
+
for start in decided:
|
|
2635
|
+
if start in components:
|
|
2636
|
+
continue
|
|
2637
|
+
component: set = set()
|
|
2638
|
+
frontier = [start]
|
|
2639
|
+
while frontier:
|
|
2640
|
+
block = frontier.pop()
|
|
2641
|
+
if block in component:
|
|
2642
|
+
continue
|
|
2643
|
+
component.add(block)
|
|
2644
|
+
for event_id in block_ids.get(block, ()):
|
|
2645
|
+
frontier.extend(id_blocks.get(event_id, ()) - component)
|
|
2646
|
+
frozen_component = tuple(sorted(component, key=repr))
|
|
2647
|
+
for block in component:
|
|
2648
|
+
components[block] = frozen_component
|
|
2649
|
+
|
|
2650
|
+
order_index = {event_id: index for index, event_id in enumerate(with_gap)}
|
|
2651
|
+
retirement_blocks = []
|
|
2652
|
+
for action in retirements:
|
|
2653
|
+
record = current.get(action.get("id"))
|
|
2654
|
+
retirement_blocks.append(
|
|
2655
|
+
(action, _hashable_block(payload_of(record))
|
|
2656
|
+
if five_hour(record) else None))
|
|
2657
|
+
|
|
2658
|
+
def landed_records(choices):
|
|
2659
|
+
out: dict = {}
|
|
2660
|
+
for block, name in choices.items():
|
|
2661
|
+
out.update(candidates[block][name])
|
|
2662
|
+
return out
|
|
2663
|
+
|
|
2664
|
+
targets: dict = {}
|
|
2665
|
+
decisions: list = []
|
|
2666
|
+
done: set = set()
|
|
2667
|
+
for start in decided:
|
|
2668
|
+
component = components[start]
|
|
2669
|
+
if start in done:
|
|
2670
|
+
continue
|
|
2671
|
+
members = set(component)
|
|
2672
|
+
touched_here = [block for block in component if block in candidates]
|
|
2673
|
+
done.update(touched_here)
|
|
2674
|
+
milestone_ids = {
|
|
2675
|
+
event_id for block in component
|
|
2676
|
+
for event_id in block_ids.get(block, ())
|
|
2677
|
+
if event_id in lineage.milestones
|
|
2678
|
+
or any(payload_of(records.get(event_id)).get("kind")
|
|
2679
|
+
== "five_hour_milestone"
|
|
2680
|
+
for records in candidates.get(block, {}).values())
|
|
2681
|
+
}
|
|
2682
|
+
retired_actions = [dict(action) for action, block in retirement_blocks
|
|
2683
|
+
if block in members]
|
|
2684
|
+
todays_here = {
|
|
2685
|
+
event_id: record for event_id, record in todays_records.items()
|
|
2686
|
+
if blocks_of_id.get(event_id, set()) & members
|
|
2687
|
+
or _hashable_block(payload_of(record)) in members}
|
|
2688
|
+
milestone_ids |= {
|
|
2689
|
+
event_id for event_id, record in todays_here.items()
|
|
2690
|
+
if payload_of(record).get("kind") == "five_hour_milestone"}
|
|
2691
|
+
cache: dict = {}
|
|
2692
|
+
|
|
2693
|
+
def violations(records, *, retired_actions=retired_actions,
|
|
2694
|
+
milestone_ids=milestone_ids, cache=cache,
|
|
2695
|
+
todays_here=todays_here):
|
|
2696
|
+
records = {**todays_here, **records}
|
|
2697
|
+
key = _canonical_bytes([
|
|
2698
|
+
[event_id, _semantic_event(records[event_id])]
|
|
2699
|
+
for event_id in sorted(records)])
|
|
2700
|
+
if key in cache:
|
|
2701
|
+
return cache[key]
|
|
2702
|
+
actions, plain = [], []
|
|
2703
|
+
for event_id in sorted(records):
|
|
2704
|
+
record = records[event_id]
|
|
2705
|
+
existing = current.get(event_id)
|
|
2706
|
+
if existing is not None and (_semantic_event(existing)
|
|
2707
|
+
== _semantic_event(record)):
|
|
2708
|
+
continue
|
|
2709
|
+
if event_id in physical:
|
|
2710
|
+
actions.append({
|
|
2711
|
+
"action": "replace", "id": event_id,
|
|
2712
|
+
"rev": physical[event_id] + 1,
|
|
2713
|
+
"at": str(record.get("at")),
|
|
2714
|
+
"payload": dict(record.get("payload") or {}),
|
|
2715
|
+
})
|
|
2716
|
+
else:
|
|
2717
|
+
plain.append(record)
|
|
2718
|
+
actions.extend(retired_actions)
|
|
2719
|
+
if not actions and not plain:
|
|
2720
|
+
# Nothing would be appended: every decision stays as it is.
|
|
2721
|
+
cache[key] = frozenset()
|
|
2722
|
+
return cache[key]
|
|
2723
|
+
actions.sort(key=lambda action: action["id"])
|
|
2724
|
+
plain.sort(key=lambda record: order_index.get(
|
|
2725
|
+
record.get("id"), len(order_index)))
|
|
2726
|
+
after = lineage.extended(batch_actions=actions, plain=plain)
|
|
2727
|
+
result = _violation_keys(lineage, after, milestone_ids, records)
|
|
2728
|
+
cache[key] = result
|
|
2729
|
+
return result
|
|
2730
|
+
|
|
2731
|
+
def choose(block, carried):
|
|
2732
|
+
order = _COMPLETION_CANDIDATE_ORDER[classes[block]]
|
|
2733
|
+
own = candidates[block]
|
|
2734
|
+
found = {"class": violations({**carried, **own["class"]})}
|
|
2735
|
+
if not found["class"]:
|
|
2736
|
+
return "class", found
|
|
2737
|
+
for name in order[1:]:
|
|
2738
|
+
found[name] = violations({**carried, **own[name]})
|
|
2739
|
+
ceiling = found["A"]
|
|
2740
|
+
for name in order:
|
|
2741
|
+
if found[name] <= ceiling and not any(
|
|
2742
|
+
found[other] < found[name] for other in order):
|
|
2743
|
+
return name, found
|
|
2744
|
+
return "A", found # unreachable: A always qualifies
|
|
2745
|
+
|
|
2746
|
+
if len(touched_here) == 1:
|
|
2747
|
+
[block] = touched_here
|
|
2748
|
+
name, found = choose(block, {})
|
|
2749
|
+
choices = {block: name}
|
|
2750
|
+
found_by_block = {block: found}
|
|
2751
|
+
else:
|
|
2752
|
+
choices, found_by_block = {}, {}
|
|
2753
|
+
for block in touched_here:
|
|
2754
|
+
carried: dict = {}
|
|
2755
|
+
for other in touched_here:
|
|
2756
|
+
if other != block:
|
|
2757
|
+
carried.update(candidates[other]["A"])
|
|
2758
|
+
choices[block], found_by_block[block] = choose(block, carried)
|
|
2759
|
+
alone = landed_records({block: "A" for block in touched_here})
|
|
2760
|
+
if not violations(landed_records(choices)) <= violations(alone):
|
|
2761
|
+
choices = {block: "A" for block in touched_here}
|
|
2762
|
+
targets.update(landed_records(choices))
|
|
2763
|
+
for block in touched_here:
|
|
2764
|
+
decisions.append(BlockDecision(
|
|
2765
|
+
block=block, block_class=classes[block], component=component,
|
|
2766
|
+
landed=choices[block],
|
|
2767
|
+
violations=dict(found_by_block[block])))
|
|
2768
|
+
decisions.sort(key=lambda decision: repr(decision.block))
|
|
2769
|
+
if not set(targets) <= handled:
|
|
2770
|
+
# Both the planner and today's path would write these ids.
|
|
2771
|
+
raise RuntimeError(
|
|
2772
|
+
"#1040 planner targets outside the ids it owns: "
|
|
2773
|
+
f"{sorted(set(targets) - handled)}")
|
|
2774
|
+
return CompletionFiveHourPlan(
|
|
2775
|
+
targets={event_id: targets[event_id] for event_id in sorted(targets)},
|
|
2776
|
+
handled=frozenset(handled),
|
|
2777
|
+
decisions=tuple(decisions),
|
|
2778
|
+
)
|
|
2779
|
+
|
|
910
2780
|
_SNAPSHOT_IDENTITY_FIELDS = (
|
|
911
2781
|
"account_key", "week_start_date", "week_start_at", "week_end_at",
|
|
912
2782
|
"weekly_percent", "five_hour_percent", "five_hour_window_key",
|
|
@@ -1876,6 +3746,19 @@ class RederivePlan:
|
|
|
1876
3746
|
#: (`_retraction_scope`). Not part of the plan body: the retractions are
|
|
1877
3747
|
#: already in the config fingerprint through the decision op's payload.
|
|
1878
3748
|
retraction_event_ids: frozenset = frozenset()
|
|
3749
|
+
#: #956 spec 4.6.3: the windows whose opening fact the selection this plan
|
|
3750
|
+
#: was built over tombstones. Not part of the plan body: it is a property
|
|
3751
|
+
#: of the journal prefix the plan already names. A filtered plan
|
|
3752
|
+
#: (:func:`causal_delta_plan`) evaluates parent presence with it.
|
|
3753
|
+
retired_openings: frozenset = frozenset()
|
|
3754
|
+
#: #943/#956 spec 4.4: the current five-hour milestones whose window
|
|
3755
|
+
#: resolves no block before the plan (pre-existing damage). Reported,
|
|
3756
|
+
#: never a reason to refuse, and not part of the plan body.
|
|
3757
|
+
orphaned_milestone_ids: frozenset = frozenset()
|
|
3758
|
+
#: #884 spec §5.7: the five-hour blocks whose milestone repair the
|
|
3759
|
+
#: lineage withheld (:class:`WithheldBlock`). A diagnostic, outside the
|
|
3760
|
+
#: plan body; its count reaches ``uncertain_cost_fact_count``.
|
|
3761
|
+
withheld_milestone_repairs: tuple = ()
|
|
1879
3762
|
|
|
1880
3763
|
def _body(self) -> dict:
|
|
1881
3764
|
body = {
|
|
@@ -1940,6 +3823,7 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
1940
3823
|
uncertain_cost_fact_ids=(),
|
|
1941
3824
|
historical_close_markers=None,
|
|
1942
3825
|
physical_revisions=None,
|
|
3826
|
+
milestone_lineage=None,
|
|
1943
3827
|
enforce_guard=True) -> RederivePlan:
|
|
1944
3828
|
"""Diff current effective events against one scratch-derived desired set.
|
|
1945
3829
|
|
|
@@ -1972,7 +3856,15 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
1972
3856
|
preserved history retained or revived, or corrected without moving its
|
|
1973
3857
|
money) or replaced (added, or corrected in a way that writes its money)
|
|
1974
3858
|
are decided by :func:`_resolve_closed_block_milestones` before the diff,
|
|
1975
|
-
the same way closes are decided before milestones.
|
|
3859
|
+
the same way closes are decided before milestones.
|
|
3860
|
+
|
|
3861
|
+
#884: ``milestone_lineage`` (:func:`build_five_hour_lineage`, built from
|
|
3862
|
+
the same selection's fold) lets an unchanged crossing of a frozen block
|
|
3863
|
+
regain the amount its history proves an earlier rederive repriced, and
|
|
3864
|
+
fails closed otherwise: each reported withheld block counts once in
|
|
3865
|
+
``uncertain_cost_fact_count`` and is listed in
|
|
3866
|
+
``withheld_milestone_repairs``. ``None`` reproduces #875's plan exactly;
|
|
3867
|
+
a missing lineage is a compatibility default, never evidence."""
|
|
1976
3868
|
conflicted_event_ids = frozenset(conflicted_event_ids or ())
|
|
1977
3869
|
historical_close_markers = dict(historical_close_markers or {})
|
|
1978
3870
|
# #825/#832 fix round A (Codex fix-round-1 Q2): the highest revision of
|
|
@@ -2009,6 +3901,8 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
2009
3901
|
current, desired, reviewed_weekly_hold_ids, historical_close_markers,
|
|
2010
3902
|
unsupported_resets=unsupported_resets,
|
|
2011
3903
|
)
|
|
3904
|
+
# #943 spec 4.4: an `fhcb` follows its `fhc` by natural key.
|
|
3905
|
+
_companions_follow_kept_credits(current, desired)
|
|
2012
3906
|
retained_event_count = sum(
|
|
2013
3907
|
1 for selected in selection.by_id.values()
|
|
2014
3908
|
if selected.status == "active"
|
|
@@ -2091,9 +3985,11 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
2091
3985
|
# and a replaced one is frozen. The causal block closure applies the same
|
|
2092
3986
|
# rule (:func:`_replaced_block_closure`).
|
|
2093
3987
|
replaced_close_blocks -= frozen_close_blocks
|
|
3988
|
+
withheld: dict = {}
|
|
2094
3989
|
milestone_decisions = _resolve_closed_block_milestones(
|
|
2095
3990
|
current, desired, preserved, frozen_close_blocks,
|
|
2096
3991
|
replaced_close_blocks, selection=selection,
|
|
3992
|
+
lineage=milestone_lineage, withheld=withheld,
|
|
2097
3993
|
)
|
|
2098
3994
|
|
|
2099
3995
|
for event_id in sorted(set(current) | set(desired) | set(preserved)):
|
|
@@ -2186,6 +4082,19 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
2186
4082
|
kind=kind,
|
|
2187
4083
|
))
|
|
2188
4084
|
|
|
4085
|
+
# #943/#956 spec 4.4: parent validation over the resulting effective
|
|
4086
|
+
# graph. Every five-hour milestone that resolves a block before the plan
|
|
4087
|
+
# must still resolve one after it. The diff above already tombstones a
|
|
4088
|
+
# current milestone the desired replay lacks, so a survivor without its
|
|
4089
|
+
# parent is one the replay itself keeps (or preserved history): refuse
|
|
4090
|
+
# rather than leave it parentless.
|
|
4091
|
+
retired_openings = retired_openings_of(selection)
|
|
4092
|
+
orphaned_before, lost = five_hour_parent_report(
|
|
4093
|
+
current, actions, retired_openings)
|
|
4094
|
+
if lost:
|
|
4095
|
+
raise RederiveConflict(
|
|
4096
|
+
"a five-hour block removal would orphan a retained milestone: "
|
|
4097
|
+
+ ", ".join(lost[:10]))
|
|
2189
4098
|
# #869 F7/Q9: a close counts only while its cost stays unprovable in the
|
|
2190
4099
|
# state this plan leaves behind — not a provably priced close, not one this
|
|
2191
4100
|
# plan corrects, and not one an earlier correction already proved.
|
|
@@ -2207,6 +4116,18 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
2207
4116
|
set(uncertain_cost_fact_ids or ()) | uncertain_closes
|
|
2208
4117
|
| drifted_close_ids
|
|
2209
4118
|
)
|
|
4119
|
+
# #884 spec §5.7: a withheld block counts once, by its smallest close id,
|
|
4120
|
+
# and never beside a close of its own that already counts.
|
|
4121
|
+
withheld_blocks = _withheld_blocks(withheld, current, actions, selection)
|
|
4122
|
+
for block in withheld_blocks:
|
|
4123
|
+
if any(close_id in uncertain_ids for close_id in block.close_event_ids):
|
|
4124
|
+
continue
|
|
4125
|
+
# The synthetic key is defensive only: a frozen block always has a
|
|
4126
|
+
# close in the plan's final state, so it is unreachable.
|
|
4127
|
+
uncertain_ids.add(
|
|
4128
|
+
min(block.close_event_ids) if block.close_event_ids
|
|
4129
|
+
else ("withheld-block", block.account_key,
|
|
4130
|
+
block.five_hour_window_key))
|
|
2210
4131
|
plan = RederivePlan(
|
|
2211
4132
|
family=FAMILY,
|
|
2212
4133
|
journal_high_water=journal_high_water,
|
|
@@ -2219,6 +4140,9 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
2219
4140
|
uncertain_cost_fact_count=len(uncertain_ids),
|
|
2220
4141
|
action_counts_by_event_kind=counts_by_kind,
|
|
2221
4142
|
retraction_event_ids=retraction_event_ids,
|
|
4143
|
+
retired_openings=retired_openings,
|
|
4144
|
+
orphaned_milestone_ids=orphaned_before,
|
|
4145
|
+
withheld_milestone_repairs=withheld_blocks,
|
|
2222
4146
|
)
|
|
2223
4147
|
if enforce_guard:
|
|
2224
4148
|
enforce_week_reset_add_burst_guard(plan)
|
|
@@ -2502,6 +4426,89 @@ def _retained_reference_closure(actions, reviewed_by_id, current_events):
|
|
|
2502
4426
|
return actions
|
|
2503
4427
|
|
|
2504
4428
|
|
|
4429
|
+
def _five_hour_parent_closure(included: list, reviewed_actions,
|
|
4430
|
+
current_events: Mapping[str, Mapping],
|
|
4431
|
+
retired_openings=frozenset()) -> list:
|
|
4432
|
+
"""#943/#956 spec 4.4: the reviewed actions an included action needs.
|
|
4433
|
+
|
|
4434
|
+
* Derived-FK parent closure: an included five-hour milestone add or
|
|
4435
|
+
supersede resolves its `block_id` from its window at fold, so when the
|
|
4436
|
+
graph the included actions leave behind materializes no block for
|
|
4437
|
+
that window, it keeps the reviewed plan's parent-materializing action
|
|
4438
|
+
for the window (an opening or close add or supersede), even when that
|
|
4439
|
+
action is common to the baseline. A window whose block the resulting
|
|
4440
|
+
graph already materializes (a current close or opening the plan keeps)
|
|
4441
|
+
needs no action: pulling a baseline-common close correction in there
|
|
4442
|
+
would rewrite a close the #875 frozen-block rule keeps as it stands.
|
|
4443
|
+
* A credit and its boundary companion travel together: an included
|
|
4444
|
+
action on either brings the reviewed action on the other.
|
|
4445
|
+
* A seed never outlives a removed block: an included close tombstone
|
|
4446
|
+
brings the reviewed opening tombstone of its window (the reviewed plan
|
|
4447
|
+
tombstones the opening only when its replay has no block there).
|
|
4448
|
+
|
|
4449
|
+
Only reviewed actions are ever added, each once."""
|
|
4450
|
+
parents: dict = {}
|
|
4451
|
+
opening_tombstones: dict = {}
|
|
4452
|
+
pairs: dict = {}
|
|
4453
|
+
for action in reviewed_actions:
|
|
4454
|
+
payload = _action_payload(action, current_events)
|
|
4455
|
+
if action.kind in _BLOCK_PARENT_KINDS:
|
|
4456
|
+
window = _window_of(payload)
|
|
4457
|
+
if action.disposition in {"add", "supersede"}:
|
|
4458
|
+
parents.setdefault(window, []).append(action)
|
|
4459
|
+
elif action.kind == "five_hour_block_open":
|
|
4460
|
+
opening_tombstones.setdefault(window, []).append(action)
|
|
4461
|
+
elif action.kind in _CREDIT_PAIR_KINDS:
|
|
4462
|
+
pairs.setdefault(_credit_pair_key(payload), []).append(action)
|
|
4463
|
+
expanded = list(included)
|
|
4464
|
+
included_ids = {action.event_id for action in included}
|
|
4465
|
+
present = None
|
|
4466
|
+
for action in included:
|
|
4467
|
+
payload = _action_payload(action, current_events)
|
|
4468
|
+
needed = []
|
|
4469
|
+
if (action.kind == "five_hour_milestone"
|
|
4470
|
+
and action.disposition != "tombstone"):
|
|
4471
|
+
if present is None:
|
|
4472
|
+
after_events, after_retired = _post_plan_state(
|
|
4473
|
+
current_events, included, retired_openings)
|
|
4474
|
+
present = five_hour_parent_presence(
|
|
4475
|
+
after_events, after_retired)
|
|
4476
|
+
if _window_of(payload) not in present:
|
|
4477
|
+
needed = parents.get(_window_of(payload), [])
|
|
4478
|
+
elif action.kind in _CREDIT_PAIR_KINDS:
|
|
4479
|
+
needed = pairs.get(_credit_pair_key(payload), [])
|
|
4480
|
+
elif (action.kind == "five_hour_block_close"
|
|
4481
|
+
and action.disposition == "tombstone"):
|
|
4482
|
+
needed = opening_tombstones.get(_window_of(payload), [])
|
|
4483
|
+
for dependency in needed:
|
|
4484
|
+
if dependency.event_id not in included_ids:
|
|
4485
|
+
expanded.append(dependency)
|
|
4486
|
+
included_ids.add(dependency.event_id)
|
|
4487
|
+
return expanded
|
|
4488
|
+
|
|
4489
|
+
|
|
4490
|
+
def _parent_validation_completion(included: list, reviewed_by_id,
|
|
4491
|
+
current_events: Mapping[str, Mapping],
|
|
4492
|
+
retired_openings) -> list:
|
|
4493
|
+
"""#943/#956 spec 4.4: a filtered plan whose block removal would orphan
|
|
4494
|
+
an unchanged, retained milestone takes that milestone's reviewed
|
|
4495
|
+
tombstone (the desired replay lacks it). A survivor with no reviewed
|
|
4496
|
+
tombstone is left for the final validation, which refuses."""
|
|
4497
|
+
_orphaned_before, lost = five_hour_parent_report(
|
|
4498
|
+
current_events, included, retired_openings)
|
|
4499
|
+
if not lost:
|
|
4500
|
+
return included
|
|
4501
|
+
expanded = list(included)
|
|
4502
|
+
included_ids = {action.event_id for action in included}
|
|
4503
|
+
for event_id in lost:
|
|
4504
|
+
reviewed = reviewed_by_id.get(event_id)
|
|
4505
|
+
if (reviewed is not None and reviewed.disposition == "tombstone"
|
|
4506
|
+
and event_id not in included_ids):
|
|
4507
|
+
expanded.append(reviewed)
|
|
4508
|
+
included_ids.add(event_id)
|
|
4509
|
+
return expanded
|
|
4510
|
+
|
|
4511
|
+
|
|
2505
4512
|
def causal_delta_plan(
|
|
2506
4513
|
baseline: RederivePlan, reviewed: RederivePlan, *,
|
|
2507
4514
|
current_events: Mapping[str, Mapping],
|
|
@@ -2568,8 +4575,23 @@ def causal_delta_plan(
|
|
|
2568
4575
|
if retraction_scope:
|
|
2569
4576
|
actions = _retained_reference_closure(
|
|
2570
4577
|
actions, reviewed_by_id, current_events)
|
|
4578
|
+
# #943/#956 spec 4.4: over the final filtered set, so a parent or a
|
|
4579
|
+
# companion the B/R filter dropped as baseline-common returns.
|
|
4580
|
+
actions = _five_hour_parent_closure(
|
|
4581
|
+
actions, reviewed.actions, current_events,
|
|
4582
|
+
reviewed.retired_openings)
|
|
4583
|
+
actions = _parent_validation_completion(
|
|
4584
|
+
actions, reviewed_by_id, current_events,
|
|
4585
|
+
reviewed.retired_openings)
|
|
2571
4586
|
if {action.event_id for action in actions} == previous_ids:
|
|
2572
4587
|
break
|
|
4588
|
+
# Parent validation over the resulting effective graph (spec 4.4).
|
|
4589
|
+
orphaned_before, lost = five_hour_parent_report(
|
|
4590
|
+
current_events, actions, reviewed.retired_openings)
|
|
4591
|
+
if lost:
|
|
4592
|
+
raise RederiveConflict(
|
|
4593
|
+
"a five-hour block removal would orphan a retained milestone: "
|
|
4594
|
+
+ ", ".join(lost[:10]))
|
|
2573
4595
|
included_ids = {action.event_id for action in actions}
|
|
2574
4596
|
actions = tuple(
|
|
2575
4597
|
action for action in reviewed.actions
|
|
@@ -2599,6 +4621,9 @@ def causal_delta_plan(
|
|
|
2599
4621
|
uncertain_cost_fact_count=reviewed.uncertain_cost_fact_count,
|
|
2600
4622
|
action_counts_by_event_kind=counts_by_kind,
|
|
2601
4623
|
retraction_event_ids=reviewed.retraction_event_ids,
|
|
4624
|
+
retired_openings=reviewed.retired_openings,
|
|
4625
|
+
orphaned_milestone_ids=orphaned_before,
|
|
4626
|
+
withheld_milestone_repairs=reviewed.withheld_milestone_repairs,
|
|
2602
4627
|
)
|
|
2603
4628
|
|
|
2604
4629
|
|