cctally 1.116.1 → 1.117.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +47 -0
  2. package/README.md +4 -6
  3. package/bin/_cctally_account.py +57 -3
  4. package/bin/_cctally_cache.py +5720 -885
  5. package/bin/_cctally_claude_attribution.py +997 -0
  6. package/bin/_cctally_completion.py +101 -28
  7. package/bin/_cctally_core.py +235 -12
  8. package/bin/_cctally_dashboard.py +263 -17
  9. package/bin/_cctally_dashboard_conversation.py +107 -21
  10. package/bin/_cctally_dashboard_envelope.py +44 -0
  11. package/bin/_cctally_db.py +928 -80
  12. package/bin/_cctally_diagnosis_sources.py +78 -13
  13. package/bin/_cctally_doctor.py +211 -86
  14. package/bin/_cctally_five_hour.py +446 -22
  15. package/bin/_cctally_forecast.py +302 -6
  16. package/bin/_cctally_journal.py +2418 -158
  17. package/bin/_cctally_milestone_history.py +379 -91
  18. package/bin/_cctally_oauth_samples.py +185 -7
  19. package/bin/_cctally_parser.py +38 -0
  20. package/bin/_cctally_project.py +34 -8
  21. package/bin/_cctally_quarantine.py +821 -343
  22. package/bin/_cctally_quota.py +69 -4
  23. package/bin/_cctally_quota_calibration.py +677 -3
  24. package/bin/_cctally_quota_model.py +927 -127
  25. package/bin/_cctally_record.py +922 -445
  26. package/bin/_cctally_rederive.py +119 -15
  27. package/bin/_cctally_statusline.py +1048 -176
  28. package/bin/_cctally_store.py +544 -110
  29. package/bin/_cctally_tui.py +455 -42
  30. package/bin/_cctally_update.py +77 -14
  31. package/bin/_lib_account_protocol.py +227 -19
  32. package/bin/_lib_claude_attribution.py +421 -0
  33. package/bin/_lib_clone_copy.py +30 -0
  34. package/bin/_lib_codex_conversation_query.py +45 -9
  35. package/bin/_lib_codex_conversation_watch.py +72 -13
  36. package/bin/_lib_codex_rollout_container.py +234 -19
  37. package/bin/_lib_codex_worker_lifecycle.py +374 -0
  38. package/bin/_lib_credit.py +22 -13
  39. package/bin/_lib_dashboard_sources.py +257 -18
  40. package/bin/_lib_detector_access_registry.py +1335 -166
  41. package/bin/_lib_doctor.py +276 -92
  42. package/bin/_lib_ingest_frontier.py +269 -36
  43. package/bin/_lib_journal.py +273 -22
  44. package/bin/_lib_jsonl.py +12 -0
  45. package/bin/_lib_meter_rate_change.py +72 -0
  46. package/bin/_lib_outcome_quarantine.py +917 -554
  47. package/bin/_lib_provider_activity.py +24 -6
  48. package/bin/_lib_quota_calibration.py +57 -3
  49. package/bin/_lib_quota_copy.py +7 -2
  50. package/bin/_lib_quota_lineage.py +613 -0
  51. package/bin/_lib_quota_model.py +464 -33
  52. package/bin/_lib_rate_change_delivery.py +16 -0
  53. package/bin/_lib_reclaim_planner.py +6 -4
  54. package/bin/_lib_rederive.py +2046 -21
  55. package/bin/_lib_selector_state.py +131 -3
  56. package/bin/_lib_snapshot_cache.py +8 -0
  57. package/bin/_lib_stale_proof.py +113 -4
  58. package/bin/_lib_statusline.py +72 -3
  59. package/bin/_lib_statusline_candidates.py +246 -42
  60. package/bin/_lib_tick_stats.py +10 -5
  61. package/bin/_lib_write_io.py +419 -6
  62. package/bin/cctally +28 -11
  63. package/dashboard/static/assets/{ConversationsView-Dflb_myA.js → ConversationsView-oqNsWzWV.js} +1 -1
  64. package/dashboard/static/assets/{DoctorModal-DFX5CfIs.js → DoctorModal-B0DnU9d9.js} +1 -1
  65. package/dashboard/static/assets/ModalRoot-CYua2wP_.js +1 -0
  66. package/dashboard/static/assets/{ProjectsDrillPanel-C3PJy1xx.js → ProjectsDrillPanel-DSxXQSa-.js} +1 -1
  67. package/dashboard/static/assets/{SourceDetailModal-BczsWKhD.js → SourceDetailModal-BleDpQqG.js} +1 -1
  68. package/dashboard/static/assets/{UpdateModal-Os-Lo5cs.js → UpdateModal-CMqaAkSK.js} +1 -1
  69. package/dashboard/static/assets/{index-CYZ3NLqF.js → index-BjQV-0-D.js} +3 -3
  70. package/dashboard/static/assets/index-D34eYhAc.css +1 -0
  71. package/dashboard/static/assets/{outlineNavigation-k-zIif6W.js → outlineNavigation-CO-VWBm7.js} +1 -1
  72. package/dashboard/static/assets/{useKeymap-CW7Up0Ay.js → useKeymap-CUgEnnua.js} +1 -1
  73. package/dashboard/static/dashboard.html +3 -3
  74. package/package.json +5 -1
  75. package/dashboard/static/assets/ModalRoot-HlxoagIa.js +0 -1
  76. package/dashboard/static/assets/index-BKITk1CT.css +0 -1
@@ -12,14 +12,16 @@ No filesystem, SQLite, provider, alert, or journal operation belongs here.
12
12
 
13
13
  from __future__ import annotations
14
14
 
15
+ import bisect
15
16
  import dataclasses
16
17
  import datetime as dt
17
18
  import hashlib
18
19
  import json
19
20
  import math
20
- from collections.abc import Iterable, Mapping
21
+ from collections.abc import Iterable, Mapping, Sequence
21
22
 
22
23
  import _lib_accounts
24
+ import _lib_journal
23
25
  from _lib_cost_provenance import (
24
26
  HISTORICAL_CLOSE_BASIS, RECORDED_CLOSE_CORRECTION_BASIS,
25
27
  )
@@ -82,6 +84,12 @@ _EVT_CLASSIFICATIONS = {
82
84
  "rederived", "Claude weekly reset or credit decision"),
83
85
  "five_hour_credit": KindClassification(
84
86
  "rederived", "Claude five-hour credit decision"),
87
+ # #943 spec 4.2.2: the exact boundary companion of a five-hour credit.
88
+ "five_hour_credit_boundary": KindClassification(
89
+ "rederived", "Claude five-hour credit boundary"),
90
+ # #956 spec 4.6.1: every five-hour block creation's opening fact.
91
+ "five_hour_block_open": KindClassification(
92
+ "rederived", "Claude five-hour block opening"),
85
93
  "five_hour_block_close": KindClassification(
86
94
  "rederived", "closed Claude five-hour projection"),
87
95
  "percent_milestone": KindClassification(
@@ -115,6 +123,12 @@ _EVT_CLASSIFICATIONS = {
115
123
  "meter_rate_change": KindClassification(
116
124
  "retained",
117
125
  "the metering-rate latch is replayed, never re-derived"),
126
+ # #1015 spec 5.7: the terminal supersession of one metering-rate
127
+ # transition identity. `retained` like its latch: a self-sufficient
128
+ # payload replayed by its own natural-key fold, never re-derived.
129
+ "meter_rate_change_supersession": KindClassification(
130
+ "retained",
131
+ "a transition's supersession is replayed, never re-derived"),
118
132
  }
119
133
 
120
134
  _OP_CLASSIFICATIONS = {
@@ -145,6 +159,13 @@ _OP_CLASSIFICATIONS = {
145
159
  "retained", "operator Codex window attribution is outside claude-usage"),
146
160
  "codex_window_attribution_retract": KindClassification(
147
161
  "retained", "operator Codex window retraction is outside claude-usage"),
162
+ # #944 spec 5.10: the durable Claude cost attribution decision. `retained`
163
+ # like `codex_file_account`: it is replayed by the Claude cache leg into
164
+ # `claude_cost_decisions`, and re-derivation reads the decided
165
+ # `session_entries.account_key` it materialized rather than deriving it.
166
+ "claude_cost_account": KindClassification(
167
+ "retained", "Claude cost attribution decisions are replayed by the "
168
+ "cache leg, never re-derived"),
148
169
  # #825/#832 spec 5.1. The versioned reviewed decision is replayed into
149
170
  # scratch exactly like its previous-release twin.
150
171
  "claude_weekly_observation_decision_v2": KindClassification(
@@ -171,6 +192,25 @@ _OP_CLASSIFICATIONS = {
171
192
  "re-derived"),
172
193
  "journal_protocol_resolution_v3": KindClassification(
173
194
  "retained", "journal-repair acknowledgements are audit records"),
195
+ # #1019/#1014 spec 4.7 C2, C5: the revision-3 boundary and the v4
196
+ # acknowledgement, likewise.
197
+ "account_bound_revision_v3": KindClassification(
198
+ "retained", "the revision-3 journal fence is replayed, never "
199
+ "re-derived"),
200
+ "journal_protocol_resolution_v4": KindClassification(
201
+ "retained", "journal-repair acknowledgements are audit records"),
202
+ # #948 spec 6.2: the outcome quarantine's recovery ops are preserved
203
+ # machinery. The fold replays them in journal order; nothing in the
204
+ # claude-usage family derives them.
205
+ "legacy_spool_inventory_completion": KindClassification(
206
+ "retained", "inventory-episode completions are replayed by the "
207
+ "quarantine fold, never re-derived"),
208
+ "outcome_authority_completion": KindClassification(
209
+ "retained", "authority completions are replayed by the quarantine "
210
+ "fold, never re-derived"),
211
+ "outcome_axis_release": KindClassification(
212
+ "retained", "axis releases are replayed by the quarantine fold, "
213
+ "never re-derived"),
174
214
  }
175
215
 
176
216
 
@@ -472,6 +512,73 @@ def _with_close_marker(record: Mapping, marker: Mapping) -> dict:
472
512
  return corrected
473
513
 
474
514
 
515
+ #: The non-``total_`` members of a close's money group (#1040 spec §5.2).
516
+ _CLOSE_MONEY_EXTRA_KEYS = frozenset({
517
+ "_models", "_projects", "_pricing", "pricing_provenance_json",
518
+ })
519
+
520
+
521
+ def _close_money_keys(payload: Mapping) -> frozenset:
522
+ """#1040 (spec §5.2): the keys of a close payload's money group.
523
+
524
+ Every key starting with ``total_`` plus the child sets and both pricing
525
+ provenance fields: the partition :func:`_preserve_non_derivable_state`
526
+ compares the closure's non-monetary structure without.
527
+ """
528
+ return frozenset(
529
+ key for key in payload
530
+ if key in _CLOSE_MONEY_EXTRA_KEYS
531
+ or (isinstance(key, str) and key.startswith("total_"))
532
+ )
533
+
534
+
535
+ def close_money_overlay(desired: Mapping, current: Mapping) -> dict:
536
+ """#1040 (spec §5.2): ``overlay(D, C)`` on whole records.
537
+
538
+ ``desired`` with ``current``'s money group: every money key of D is
539
+ replaced by C's, and a money key C lacks is removed.
540
+ """
541
+ desired_payload = desired.get("payload") or {}
542
+ current_payload = current.get("payload") or {}
543
+ drop = _close_money_keys(desired_payload)
544
+ payload = {key: value for key, value in desired_payload.items()
545
+ if key not in drop}
546
+ for key in _close_money_keys(current_payload):
547
+ payload[key] = current_payload[key]
548
+ return {**desired, "payload": payload}
549
+
550
+
551
+ _CLOSE_POPULATION_TOTALS = (
552
+ "total_input_tokens", "total_output_tokens", "total_cache_create_tokens",
553
+ "total_cache_read_tokens",
554
+ )
555
+
556
+
557
+ def close_population(payload: Mapping) -> tuple:
558
+ """#1040 (spec §5.2): the card-independent part of a close's money group.
559
+
560
+ The four block token totals, and per ``_models`` / ``_projects`` child its
561
+ model or project path, account key, four token counts and entry count,
562
+ order-independent. Compared exactly.
563
+ """
564
+ def children(key: str, name: str) -> tuple:
565
+ rows = [
566
+ (child.get(name), child.get("account_key"),
567
+ *(child.get(token) for token in _BLOCK_TOKEN_KEYS),
568
+ child.get("entry_count"))
569
+ for child in payload.get(key) or ()
570
+ if isinstance(child, Mapping)
571
+ ]
572
+ return tuple(sorted(rows, key=lambda row: json.dumps(
573
+ row, sort_keys=True, default=repr)))
574
+
575
+ return (
576
+ tuple(payload.get(key) for key in _CLOSE_POPULATION_TOTALS),
577
+ children("_models", "model"),
578
+ children("_projects", "project_path"),
579
+ )
580
+
581
+
475
582
  def _preserve_non_derivable_state(
476
583
  current_record: Mapping, desired_record: Mapping, *,
477
584
  historical_marker: "Mapping | None" = None,
@@ -495,8 +602,10 @@ def _preserve_non_derivable_state(
495
602
  marker is stamped with its own basis, ``RECORDED_CLOSE_CORRECTION_BASIS``.
496
603
 
497
604
  #875: the milestones of a frozen or replaced close are decided by
498
- :func:`_resolve_closed_block_milestones`, not here. This function only
499
- carries a milestone's ``alerted_at`` latch.
605
+ :func:`_resolve_closed_block_milestones`, not here: an unchanged crossing
606
+ keeps its selected block cost there unless its lineage proves an earlier
607
+ rederive repriced it (#884). This function only carries a milestone's
608
+ ``alerted_at`` latch.
500
609
  """
501
610
  current_payload = current_record.get("payload") or {}
502
611
  desired_payload = desired_record.get("payload") or {}
@@ -520,6 +629,30 @@ def _preserve_non_derivable_state(
520
629
 
521
630
  if cost_structure(current_payload) == cost_structure(desired_payload):
522
631
  return current_record
632
+ if (
633
+ current_payload.get("kind") == "five_hour_block_open"
634
+ and desired_payload.get("kind") == "five_hour_block_open"
635
+ and current_record.get("at") == desired_record.get("at")
636
+ ):
637
+ # #956: the live writer stamps ``block_start_at`` in the host's zone,
638
+ # and a replay under another zone spells the same boundary with its
639
+ # own offset. An opening is immutable once written; a different
640
+ # spelling of the same instants is not a revision to supersede.
641
+ def opening_structure(payload):
642
+ facts = dict(payload)
643
+ for field in (
644
+ "block_start_at", "five_hour_resets_at",
645
+ "first_observed_at_utc", "created_at_utc",
646
+ "weekly_basis_captured_at_utc",
647
+ ):
648
+ instant = _iso_instant(facts.get(field))
649
+ if instant is not None:
650
+ facts[field] = instant.isoformat()
651
+ return facts
652
+
653
+ if opening_structure(current_payload) == opening_structure(
654
+ desired_payload):
655
+ return current_record
523
656
  if (
524
657
  current_payload.get("kind") == "five_hour_block_close"
525
658
  and desired_payload.get("kind") == "five_hour_block_close"
@@ -533,13 +666,10 @@ def _preserve_non_derivable_state(
533
666
  # closure's non-monetary evidence first so a changed boundary or final
534
667
  # reading still reaches the normal correction path.
535
668
  def structure(payload):
669
+ money = _close_money_keys(payload)
536
670
  facts = {
537
671
  key: value for key, value in payload.items()
538
- if not key.startswith("total_")
539
- and key not in {
540
- "_models", "_projects", "_pricing",
541
- "pricing_provenance_json",
542
- }
672
+ if key not in money
543
673
  }
544
674
  # Live close used the display-zone offset; scratch replay writes
545
675
  # UTC. Both name one frozen boundary and must compare as an instant.
@@ -605,9 +735,9 @@ def _preserve_non_derivable_state(
605
735
  return keep_drifted()
606
736
  # #869 V4: record that this close was CORRECTED. The basis is
607
737
  # provenance only: since #875 every frozen close freezes the
608
- # block cost of its unchanged five-hour crossings, whatever
609
- # its basis. A genuinely priced close's marker looks the same
610
- # otherwise.
738
+ # block cost of its unchanged five-hour crossings (save a
739
+ # #884 history-proven restoration), whatever its basis. A
740
+ # genuinely priced close's marker looks the same otherwise.
611
741
  return _with_close_marker(desired_record, {
612
742
  **desired_pricing, "basis": RECORDED_CLOSE_CORRECTION_BASIS,
613
743
  })
@@ -726,10 +856,967 @@ def _revived_by_this_family(selected) -> bool:
726
856
  )
727
857
 
728
858
 
859
+ # ── #943/#956 spec 4.4: five-hour credit companions and block parents ──────
860
+ #
861
+ # A five-hour credit (`five_hour_credit`, `fhc`) and its exact-boundary
862
+ # companion (`five_hour_credit_boundary`, `fhcb`) share one natural key, and
863
+ # the companion is read only while its `credit_revision` matches the credit,
864
+ # so a plan that acts on one without the other splits them. A five-hour
865
+ # milestone's `block_id` is a DERIVED foreign key, resolved at fold from its
866
+ # `(account_key, five_hour_window_key)`, so it depends on whatever
867
+ # materializes that window's block: the opening fact (`five_hour_block_open`,
868
+ # `fhbo`), the close fact (`five_hour_block_close`, `fhbc`), or the rebuild's
869
+ # legacy candidate rule for an unseeded open block.
870
+
871
+ _CREDIT_PAIR_KINDS = frozenset({"five_hour_credit", "five_hour_credit_boundary"})
872
+ _BLOCK_PARENT_KINDS = frozenset({"five_hour_block_open", "five_hour_block_close"})
873
+ _OPENING_ID_PREFIX = "fhbo:"
874
+
875
+
876
+ def _window_of(payload: Mapping):
877
+ """``(account_key, int(five_hour_window_key))`` of a payload, or None."""
878
+ window = payload.get("five_hour_window_key")
879
+ if window is None or isinstance(window, bool):
880
+ return None
881
+ try:
882
+ return (payload.get("account_key"), int(window))
883
+ except (TypeError, ValueError):
884
+ return None
885
+
886
+
887
+ def _credit_pair_key(payload: Mapping) -> tuple:
888
+ """The natural key a credit and its companion share."""
889
+ return (payload.get("account_key"), payload.get("five_hour_window_key"),
890
+ payload.get("effective_reset_at_utc"))
891
+
892
+
893
+ def _action_payload(action, current_events: Mapping[str, Mapping]) -> Mapping:
894
+ """An action's payload; a tombstone's comes from its current record."""
895
+ if action.disposition == "tombstone":
896
+ return (current_events.get(action.event_id) or {}).get("payload") or {}
897
+ return action.payload or {}
898
+
899
+
900
+ def _opening_window(event_id: str):
901
+ """The window of an `fhbo` id (``fhbo:<account_key>:<window_key>``)."""
902
+ if not event_id.startswith(_OPENING_ID_PREFIX):
903
+ return None
904
+ account_key, _sep, window = event_id[len(_OPENING_ID_PREFIX):].rpartition(":")
905
+ try:
906
+ return (account_key, int(window))
907
+ except ValueError:
908
+ return None
909
+
910
+
911
+ def retired_openings_of(selection) -> frozenset:
912
+ """The windows whose opening fact the effective selection tombstones.
913
+
914
+ The rebuild never makes such a window a legacy open-block candidate
915
+ (`_cctally_journal.retired_block_openings`, spec 4.6.3)."""
916
+ retired = set()
917
+ for event_id, selected in selection.by_id.items():
918
+ if selected.status != "tombstone":
919
+ continue
920
+ window = _opening_window(str(event_id))
921
+ if window is not None:
922
+ retired.add(window)
923
+ return frozenset(retired)
924
+
925
+
926
+ def five_hour_parent_presence(events: Mapping[str, Mapping],
927
+ retired_openings=frozenset()) -> frozenset:
928
+ """The `(account_key, five_hour_window_key)` windows a rebuild of the
929
+ effective set ``events`` materializes a block for (spec 4.4, 4.6.3).
930
+
931
+ A window has a block when an opening or close fact names it, and, for
932
+ each account, when it is the window of the account's latest-captured
933
+ five-hour reading (`unixepoch(captured_at_utc) DESC, journal_id DESC`)
934
+ and no block of it exists and its opening was not retired: the legacy
935
+ candidate rule of `_cctally_five_hour._backfill_five_hour_blocks`. A
936
+ reading a five-hour credit's suppression names carries no five-hour
937
+ reading once folded (retired in place, or deleted when held), so it never
938
+ qualifies. A weekly suppression deletes a row that carries no five-hour
939
+ reading, which never qualifies, and a `record-credit` synthetic, which
940
+ is counted here although the fold never sees it (#943 review R-5). The
941
+ two differ only when that synthetic is the account's latest reading
942
+ and lies in another window than its next surviving reading; it copies
943
+ the window of the latest reading (`_resolve_prior_5h`), so that does
944
+ not arise in practice."""
945
+ present = set()
946
+ five_hour_suppressed = set()
947
+ for record in events.values():
948
+ payload = (record or {}).get("payload") or {}
949
+ kind = payload.get("kind")
950
+ if kind in _BLOCK_PARENT_KINDS:
951
+ window = _window_of(payload)
952
+ if window is not None:
953
+ present.add(window)
954
+ elif (kind == "five_hour_credit"
955
+ and payload.get("suppression_table", "weekly_usage_snapshots")
956
+ == "weekly_usage_snapshots"
957
+ and isinstance(payload.get("suppression"), list)):
958
+ five_hour_suppressed.update(
959
+ item for item in payload["suppression"]
960
+ if isinstance(item, str))
961
+ latest: dict = {}
962
+ for event_id, record in events.items():
963
+ payload = (record or {}).get("payload") or {}
964
+ if (payload.get("kind") != "snapshot_accept"
965
+ or event_id in five_hour_suppressed
966
+ or payload.get("five_hour_percent") is None):
967
+ continue
968
+ window = _window_of(payload)
969
+ instant = _iso_instant(payload.get("captured_at_utc"))
970
+ if window is None or instant is None:
971
+ continue
972
+ rank = (instant.timestamp() // 1, str(event_id))
973
+ prior = latest.get(window[0])
974
+ if prior is None or rank > prior[0]:
975
+ latest[window[0]] = (rank, window)
976
+ retired = frozenset(retired_openings or ())
977
+ for _rank, window in latest.values():
978
+ if window not in retired:
979
+ present.add(window)
980
+ return frozenset(present)
981
+
982
+
983
+ def _post_plan_state(current: Mapping[str, Mapping], actions,
984
+ retired_openings) -> tuple[dict, frozenset]:
985
+ """The effective set and the retired openings after ``actions``."""
986
+ after = dict(current)
987
+ retired = set(retired_openings or ())
988
+ for action in actions:
989
+ window = _opening_window(action.event_id)
990
+ if action.disposition == "tombstone":
991
+ after.pop(action.event_id, None)
992
+ if window is not None:
993
+ retired.add(window)
994
+ else:
995
+ after[action.event_id] = {"payload": dict(action.payload or {})}
996
+ if window is not None:
997
+ retired.discard(window)
998
+ return after, frozenset(retired)
999
+
1000
+
1001
+ def five_hour_parent_report(current: Mapping[str, Mapping], actions,
1002
+ retired_openings=frozenset()):
1003
+ """Parent validation over the resulting effective graph (spec 4.4).
1004
+
1005
+ Returns ``(orphaned_before, lost)``: the current five-hour milestones
1006
+ whose window resolves no block BEFORE the plan (pre-existing damage,
1007
+ reported, never a reason to refuse), and the milestones that resolve a
1008
+ block before the plan and survive it without one."""
1009
+ before = five_hour_parent_presence(current, retired_openings)
1010
+ after_events, after_retired = _post_plan_state(
1011
+ current, actions, retired_openings)
1012
+ after = five_hour_parent_presence(after_events, after_retired)
1013
+ orphaned_before = set()
1014
+ lost = []
1015
+ for event_id, record in sorted(current.items()):
1016
+ payload = (record or {}).get("payload") or {}
1017
+ if payload.get("kind") != "five_hour_milestone":
1018
+ continue
1019
+ if _window_of(payload) not in before:
1020
+ orphaned_before.add(event_id)
1021
+ continue
1022
+ survivor = after_events.get(event_id)
1023
+ if survivor is None:
1024
+ continue
1025
+ if _window_of(survivor.get("payload") or {}) not in after:
1026
+ lost.append(event_id)
1027
+ return frozenset(orphaned_before), lost
1028
+
1029
+
1030
+ def _companions_follow_kept_credits(current: Mapping[str, Mapping],
1031
+ desired: dict[str, Mapping]) -> None:
1032
+ """A credit the target keeps as its CURRENT record (a reviewed hold's
1033
+ dependency, :func:`_preserve_reviewed_hold_dependencies`) keeps its
1034
+ current companion with it: the companion the replay derived is bound to
1035
+ the replay's credit content, not the one kept, and a current companion
1036
+ with no desired twin would otherwise be tombstoned beside its credit."""
1037
+ companions = {}
1038
+ for source in (current, desired):
1039
+ for event_id, record in source.items():
1040
+ payload = (record or {}).get("payload") or {}
1041
+ if payload.get("kind") == "five_hour_credit_boundary":
1042
+ companions.setdefault(_credit_pair_key(payload), set()).add(
1043
+ event_id)
1044
+ for event_id, record in current.items():
1045
+ payload = (record or {}).get("payload") or {}
1046
+ if (payload.get("kind") != "five_hour_credit"
1047
+ or desired.get(event_id) is not record):
1048
+ continue
1049
+ for companion_id in companions.get(_credit_pair_key(payload), ()):
1050
+ kept = current.get(companion_id)
1051
+ if kept is None:
1052
+ desired.pop(companion_id, None)
1053
+ else:
1054
+ desired[companion_id] = kept
1055
+
1056
+
1057
+ # ── #884: lineage repair of five-hour milestones an earlier rederive priced ─
1058
+ #
1059
+ # #875 freezes the block cost of an unchanged crossing under a frozen close,
1060
+ # so a later card no longer reprices it. A pre-#875 rederive may already have
1061
+ # repriced one; the freeze would then keep that later card's price for good.
1062
+ # The lineage index reads each milestone's admissible journal history beside
1063
+ # its block's close history, proves which revision last legitimately set the
1064
+ # block cost (its ANCHOR), and lets the resolver restore that amount. Any
1065
+ # ambiguity keeps the current amount and is disclosed (spec §5.1–§5.7).
1066
+
1067
+ #: §5.2: an event's own history is uncertain.
1068
+ LINEAGE_HIDDEN_REVISION = "hidden-revision"
1069
+ LINEAGE_REVISION_GAP = "revision-gap"
1070
+ LINEAGE_DIVERGENT_REVISION = "divergent-revision"
1071
+ LINEAGE_INVALID_PAYLOAD = "invalid-payload"
1072
+ LINEAGE_ORDER_INVERSION = "order-inversion"
1073
+ LINEAGE_SELECTION_MISMATCH = "selection-mismatch"
1074
+ #: §5.3/§5.4: the walk cannot prove the anchor.
1075
+ LINEAGE_CLOSE_LINEAGE_UNCERTAIN = "close-lineage-uncertain"
1076
+ LINEAGE_CLOSE_PAIRING_UNCERTAIN = "close-pairing-uncertain"
1077
+ LINEAGE_REWRITE_WITHOUT_CLOSED_BOUNDARY = "rewrite-without-closed-boundary"
1078
+ LINEAGE_UNPAIRED_CLOSE_CHANGE = "unpaired-close-change"
1079
+
1080
+ _LINEAGE_KINDS = frozenset({"five_hour_milestone", "five_hour_block_close"})
1081
+
1082
+
1083
+ @dataclasses.dataclass(frozen=True)
1084
+ class LineageOccurrence:
1085
+ """One admissible candidate of a milestone or close, summarized (§5.1).
1086
+
1087
+ ``available_at`` is the journal coordinate at which it became effective:
1088
+ a rev-0 event's own sequence, or its batch's commit sequence. Milestone
1089
+ fields are ``None`` for a close and vice versa; a tombstone carries none.
1090
+ ``identity`` is :func:`_milestone_identity` without ``alerted_at``.
1091
+ ``close_view`` is an active close's canonical JSON encoding of what the
1092
+ close timeline reads (§5.2, §5.3): ``kind``, the block key, ``is_closed``
1093
+ and ``total_cost_usd``, each type-sensitive (``1``, ``1.0`` and ``true``
1094
+ differ); an absent key and an explicit ``null`` encode alike.
1095
+ """
1096
+ rev: int
1097
+ status: str
1098
+ content_hash: str
1099
+ batch_id: "str | None"
1100
+ available_at: int
1101
+ kind: "str | None"
1102
+ block: "tuple | None"
1103
+ at: object = None
1104
+ identity: "dict | None" = None
1105
+ block_cost_usd: object = None
1106
+ is_closed: object = None
1107
+ total_cost_usd: object = None
1108
+ close_view: "bytes | None" = None
1109
+
1110
+
1111
+ @dataclasses.dataclass(frozen=True)
1112
+ class Restoration:
1113
+ """One unchanged crossing's lineage answer (§5.4): ``keep``,
1114
+ ``withhold`` (fail closed) or ``restore`` to ``value``.
1115
+
1116
+ ``reported`` is §5.7's predicate: a withhold counts and is disclosed only
1117
+ when the selected crossing has a repair candidate, the revisions listed in
1118
+ ``candidate_revisions``.
1119
+ """
1120
+ action: str
1121
+ value: object = None
1122
+ reason_codes: tuple = ()
1123
+ candidate_revisions: tuple = ()
1124
+ reported: bool = False
1125
+
1126
+
1127
+ _LINEAGE_KEEP = Restoration("keep")
1128
+
1129
+
1130
+ def _lineage_identity(payload: Mapping) -> dict:
1131
+ """§5.1: a milestone's non-monetary identity without its alert latch."""
1132
+ return {
1133
+ key: value for key, value in payload.items()
1134
+ if key not in _FIVE_HOUR_MILESTONE_MONETARY_KEYS
1135
+ and key != "alerted_at"
1136
+ }
1137
+
1138
+
1139
+ def _divergence_key(occurrence, *, close: bool):
1140
+ """§5.2: what two occurrences of one revision must share to agree.
1141
+
1142
+ A milestone's occurrences agree only on the same ``(status,
1143
+ content_hash)``. A close's agree on what §5.3 reads: a tombstone on its
1144
+ status alone, an active occurrence on its status and ``close_view``
1145
+ (compared as bytes, so no value is coerced or assumed hashable). A
1146
+ divergence confined to a close's observation fields, weekly axes,
1147
+ tokens, children or markers changes no close state or event (finding
1148
+ F1, the live writer's rev-0 re-emissions).
1149
+ """
1150
+ if not close:
1151
+ return (occurrence.status, occurrence.content_hash)
1152
+ if occurrence.status != "active":
1153
+ return (occurrence.status,)
1154
+ return (occurrence.status, occurrence.close_view)
1155
+
1156
+
1157
+ def _history_uncertainty(occurrences, *, hidden: bool, invalid: bool,
1158
+ selected, close: bool = False) -> tuple:
1159
+ """§5.2: the sorted reason codes of one event's own uncertain history.
1160
+
1161
+ ``close`` marks a close id, whose same-revision occurrences are compared
1162
+ on the fields the close timeline reads (:func:`_divergence_key`); every
1163
+ other code, ``selection-mismatch``'s full-hash check included, is the
1164
+ same for both kinds.
1165
+ """
1166
+ codes = set()
1167
+ if hidden:
1168
+ codes.add(LINEAGE_HIDDEN_REVISION)
1169
+ if invalid:
1170
+ codes.add(LINEAGE_INVALID_PAYLOAD)
1171
+ by_rev: dict[int, list] = {}
1172
+ for occurrence in occurrences:
1173
+ by_rev.setdefault(occurrence.rev, []).append(occurrence)
1174
+ top = max(by_rev)
1175
+ upper = top if selected is None else max(top, selected[0])
1176
+ if any(rev not in by_rev for rev in range(upper + 1)):
1177
+ codes.add(LINEAGE_REVISION_GAP)
1178
+ earliest = {}
1179
+ for rev, group in by_rev.items():
1180
+ if len({_divergence_key(item, close=close) for item in group}) > 1:
1181
+ codes.add(LINEAGE_DIVERGENT_REVISION)
1182
+ earliest[rev] = min(item.available_at for item in group)
1183
+ if any(
1184
+ rev - 1 in earliest and earliest[rev] < earliest[rev - 1]
1185
+ for rev in earliest
1186
+ ):
1187
+ codes.add(LINEAGE_ORDER_INVERSION)
1188
+ if (
1189
+ selected is None or selected[0] != top
1190
+ or selected[1] not in {item.content_hash for item in by_rev[top]}
1191
+ ):
1192
+ codes.add(LINEAGE_SELECTION_MISMATCH)
1193
+ return tuple(sorted(codes))
1194
+
1195
+
1196
+ def _first_per_revision(occurrences) -> dict:
1197
+ """``{rev: earliest occurrence}``; ``occurrences`` are sorted."""
1198
+ chain: dict[int, LineageOccurrence] = {}
1199
+ for occurrence in occurrences:
1200
+ chain.setdefault(occurrence.rev, occurrence)
1201
+ return chain
1202
+
1203
+
1204
+ @dataclasses.dataclass(frozen=True)
1205
+ class FiveHourLineage:
1206
+ """§5.1: the compact lineage of every five-hour milestone and close.
1207
+
1208
+ Holds summaries only, never records or the fold. ``uncertain`` maps each
1209
+ id whose own history is uncertain (§5.2) to its reason codes.
1210
+ ``close_ids_ever_in_block`` is a candidate index; membership at a
1211
+ coordinate is decided from the winning payload there (§5.3).
1212
+ """
1213
+ milestones: Mapping[str, tuple]
1214
+ closes: Mapping[str, tuple]
1215
+ close_ids_ever_in_block: Mapping[tuple, tuple]
1216
+ hidden_ids: frozenset
1217
+ selected: Mapping[str, tuple]
1218
+ uncertain: Mapping[str, tuple]
1219
+ _block_timelines: dict = dataclasses.field(
1220
+ default_factory=dict, compare=False, repr=False)
1221
+ #: #1040: memoized journal end of the summarized occurrences.
1222
+ _coordinate_end: list = dataclasses.field(
1223
+ default_factory=list, compare=False, repr=False)
1224
+
1225
+ def _block_timeline(self, block) -> tuple:
1226
+ """§5.3: K's close events, ``(coordinates, states, changed,
1227
+ live_writer)``.
1228
+
1229
+ Only called when every close id of the block has a certain history,
1230
+ so each id's revision becomes effective at its earliest occurrence.
1231
+ A state is ``("ambiguous",)``, ``("absent",)``, ``("open",)`` or
1232
+ ``("closed", total)``. ``live_writer`` marks a coordinate that only
1233
+ rev-0 ``evt`` occurrences (no correction batch) make effective
1234
+ (§5.4 step 3's first live-writer close).
1235
+ """
1236
+ cached = self._block_timelines.get(block)
1237
+ if cached is not None:
1238
+ return cached
1239
+ chains = {
1240
+ event_id: _first_per_revision(self.closes[event_id])
1241
+ for event_id in self.close_ids_ever_in_block.get(block, ())
1242
+ }
1243
+ coordinates = sorted({
1244
+ occurrence.available_at
1245
+ for event_id in chains
1246
+ for occurrence in self.closes[event_id]
1247
+ })
1248
+ states = []
1249
+ changed = []
1250
+ live_writer = []
1251
+ for coordinate in coordinates:
1252
+ counted = []
1253
+ moved = False
1254
+ for chain in chains.values():
1255
+ live = [rev for rev, occurrence in chain.items()
1256
+ if occurrence.available_at <= coordinate]
1257
+ if not live:
1258
+ continue
1259
+ winner = chain[max(live)]
1260
+ moved = moved or winner.available_at == coordinate
1261
+ if winner.status == "active" and winner.block == block:
1262
+ counted.append(winner)
1263
+ changed.append(moved)
1264
+ live_writer.append(all(
1265
+ occurrence.batch_id is None and occurrence.rev == 0
1266
+ for event_id in chains
1267
+ for occurrence in self.closes[event_id]
1268
+ if occurrence.available_at == coordinate
1269
+ ))
1270
+ if len(counted) > 1:
1271
+ states.append(("ambiguous",))
1272
+ elif not counted:
1273
+ states.append(("absent",))
1274
+ elif counted[0].is_closed != 1:
1275
+ states.append(("open",))
1276
+ else:
1277
+ states.append(("closed", counted[0].total_cost_usd))
1278
+ timeline = (tuple(coordinates), tuple(states), tuple(changed),
1279
+ tuple(live_writer))
1280
+ self._block_timelines[block] = timeline
1281
+ return timeline
1282
+
1283
+ @staticmethod
1284
+ def _closed_with_money(state) -> bool:
1285
+ return state[0] == "closed" and _usable_amount(state[1])
1286
+
1287
+ @classmethod
1288
+ def _classify(cls, before, after, changed: bool) -> str:
1289
+ """§5.3: one close event, from K's states just before and at it."""
1290
+ if before[0] == "ambiguous" or after[0] == "ambiguous":
1291
+ return "uncertain"
1292
+ if not changed:
1293
+ return "metadata"
1294
+ if before[0] == "closed" and after[0] == "closed":
1295
+ if not (_usable_amount(before[1]) and _usable_amount(after[1])):
1296
+ return "uncertain"
1297
+ if abs(after[1] - before[1]) > _CLOSE_TOTAL_TOLERANCE_USD:
1298
+ return "monetary"
1299
+ return "metadata"
1300
+ if after[0] == "closed":
1301
+ return "monetary" if _usable_amount(after[1]) else "uncertain"
1302
+ if before[0] == "closed":
1303
+ return "monetary" if _usable_amount(before[1]) else "uncertain"
1304
+ if before[0] == after[0] == "absent":
1305
+ return "metadata"
1306
+ return "no-boundary"
1307
+
1308
+ def _walk(self, chain, tail: int, selected: int, block) -> tuple:
1309
+ """§5.4 steps 1–6 over the coordinate groups from ``tail``'s
1310
+ coordinate inclusive. Returns ``(anchor, reasons)``."""
1311
+ coordinates, states, changed_flags, live_writer = \
1312
+ self._block_timeline(block)
1313
+ start = chain[tail].available_at
1314
+ revisions_at: dict[int, list] = {}
1315
+ for rev in range(tail + 1, selected + 1):
1316
+ revisions_at.setdefault(chain[rev].available_at, []).append(rev)
1317
+ events = {}
1318
+ # §5.4 step 3: an unpaired monetary event is exempt from the pending
1319
+ # mark only as K's FIRST live-writer close, so whether K was ever
1320
+ # closed before it is read over the whole timeline, before t too.
1321
+ first_live_close = {}
1322
+ closed_before = False
1323
+ for index, coordinate in enumerate(coordinates):
1324
+ before = states[index - 1] if index else ("absent",)
1325
+ if coordinate >= start:
1326
+ events[coordinate] = self._classify(
1327
+ before, states[index], changed_flags[index])
1328
+ first_live_close[coordinate] = (
1329
+ live_writer[index] and not closed_before
1330
+ and before[0] in ("absent", "open")
1331
+ and states[index][0] == "closed"
1332
+ )
1333
+ closed_before = closed_before or states[index][0] == "closed"
1334
+ anchor = tail
1335
+ reasons: set = set()
1336
+ # An unpaired monetary close event since the anchor leaves A
1337
+ # unproven for a later price-only rewrite (findings D1, R7-1).
1338
+ marked = False
1339
+ for coordinate in sorted(set(revisions_at) | set(events)):
1340
+ revisions = revisions_at.get(coordinate, [])
1341
+ changed = any(
1342
+ chain[rev].block_cost_usd != chain[rev - 1].block_cost_usd
1343
+ for rev in revisions
1344
+ )
1345
+ event = events.get(coordinate)
1346
+ if event == "monetary":
1347
+ if changed:
1348
+ anchor = max(revisions)
1349
+ reasons.clear()
1350
+ marked = False
1351
+ else:
1352
+ effective = max(
1353
+ rev for rev in range(tail, selected + 1)
1354
+ if chain[rev].available_at <= coordinate
1355
+ )
1356
+ if (chain[effective].block_cost_usd
1357
+ != chain[anchor].block_cost_usd):
1358
+ reasons.add(LINEAGE_UNPAIRED_CLOSE_CHANGE)
1359
+ if not first_live_close[coordinate]:
1360
+ marked = True
1361
+ elif event == "uncertain":
1362
+ reasons.add(LINEAGE_CLOSE_PAIRING_UNCERTAIN)
1363
+ elif changed:
1364
+ if event == "no-boundary":
1365
+ reasons.add(LINEAGE_REWRITE_WITHOUT_CLOSED_BOUNDARY)
1366
+ else:
1367
+ # A price-only rewrite (no close event, or a metadata
1368
+ # one) is walked past only under a retained closed close
1369
+ # and with no step-3 mark pending.
1370
+ index = bisect.bisect_left(coordinates, coordinate)
1371
+ before = states[index - 1] if index else ("absent",)
1372
+ if not self._closed_with_money(before):
1373
+ reasons.add(LINEAGE_REWRITE_WITHOUT_CLOSED_BOUNDARY)
1374
+ if marked:
1375
+ reasons.add(LINEAGE_UNPAIRED_CLOSE_CHANGE)
1376
+ return anchor, reasons
1377
+
1378
+ def _withhold(self, event_id: str, reasons, current_cost,
1379
+ anchor) -> Restoration:
1380
+ """A fail-closed answer and §5.7's report predicate."""
1381
+ occurrences = self.milestones[event_id]
1382
+ selected = self.selected.get(event_id)
1383
+ target = None
1384
+ for occurrence in occurrences:
1385
+ if (selected is not None and occurrence.status == "active"
1386
+ and (occurrence.rev, occurrence.content_hash) == selected):
1387
+ target = occurrence
1388
+ break
1389
+ candidates = []
1390
+ if target is not None:
1391
+ by_rev: dict[int, list] = {}
1392
+ for occurrence in occurrences:
1393
+ by_rev.setdefault(occurrence.rev, []).append(occurrence)
1394
+ for rev in range(max(by_rev), -1, -1):
1395
+ if anchor is not None and rev < anchor:
1396
+ break
1397
+ group = by_rev.get(rev)
1398
+ if not group:
1399
+ continue # a gap is not a boundary
1400
+ matching = [
1401
+ occurrence for occurrence in group
1402
+ if occurrence.status == "active"
1403
+ and occurrence.at == target.at
1404
+ and occurrence.identity == target.identity
1405
+ ]
1406
+ if not matching:
1407
+ break # a proven boundary
1408
+ if any(
1409
+ _usable_amount(occurrence.block_cost_usd)
1410
+ and occurrence.block_cost_usd != current_cost
1411
+ for occurrence in matching
1412
+ ):
1413
+ candidates.append(rev)
1414
+ return Restoration(
1415
+ "withhold",
1416
+ reason_codes=tuple(sorted(set(reasons))),
1417
+ candidate_revisions=tuple(sorted(candidates)),
1418
+ reported=bool(candidates),
1419
+ )
1420
+
1421
+ def restoration(self, event_id: str,
1422
+ current_record: Mapping) -> Restoration:
1423
+ """§5.4: the decision for one unchanged crossing in a frozen block.
1424
+
1425
+ ``current_record`` is the selected record; its ``block_cost_usd`` is
1426
+ the comparison base. In order: ``keep`` without recorded history;
1427
+ ``withhold`` when the event's own history is uncertain; ``withhold``
1428
+ when the walk collected a reason; ``keep`` when the anchor's amount
1429
+ equals the current one or is unusable; else ``restore`` to the
1430
+ anchor's parsed ``block_cost_usd``, copied as is.
1431
+ """
1432
+ occurrences = self.milestones.get(event_id)
1433
+ if not occurrences:
1434
+ return _LINEAGE_KEEP
1435
+ current_cost = (current_record.get("payload") or {}).get(
1436
+ "block_cost_usd")
1437
+ own = self.uncertain.get(event_id)
1438
+ if own:
1439
+ return self._withhold(event_id, own, current_cost, None)
1440
+ chain = _first_per_revision(occurrences)
1441
+ selected = max(chain)
1442
+ if chain[selected].status != "active":
1443
+ return _LINEAGE_KEEP
1444
+ life_start = 0
1445
+ for rev in range(selected, -1, -1):
1446
+ if chain[rev].status != "active":
1447
+ life_start = rev + 1
1448
+ break
1449
+ tail = life_start
1450
+ for rev in range(life_start + 1, selected + 1):
1451
+ if (chain[rev].at != chain[rev - 1].at
1452
+ or chain[rev].identity != chain[rev - 1].identity):
1453
+ tail = rev
1454
+ block = chain[selected].block
1455
+ if any(close_id in self.uncertain
1456
+ for close_id in self.close_ids_ever_in_block.get(block, ())):
1457
+ # §5.3: no close event of K can prove anything; A stays at t.
1458
+ return self._withhold(
1459
+ event_id, {LINEAGE_CLOSE_LINEAGE_UNCERTAIN}, current_cost,
1460
+ tail)
1461
+ anchor, reasons = self._walk(chain, tail, selected, block)
1462
+ if reasons:
1463
+ return self._withhold(event_id, reasons, current_cost, anchor)
1464
+ value = chain[anchor].block_cost_usd
1465
+ if not _usable_amount(value) or value == current_cost:
1466
+ return _LINEAGE_KEEP
1467
+ return Restoration("restore", value=value)
1468
+
1469
+ def extended(self, *, batch_id: "str | None" = None,
1470
+ batch_actions: Sequence[Mapping] = (),
1471
+ plain: Sequence[Mapping] = ()) -> "FiveHourLineage":
1472
+ """#1040 (spec §5.4): the lineage as it reads after ``batch_actions``
1473
+ (one completed correction batch, completion's action form) and then
1474
+ ``plain`` (revision-0 events) are appended at the journal's end.
1475
+
1476
+ Read-only: a new instance with an empty timeline cache. The batch
1477
+ becomes effective at one coordinate after every existing one, each
1478
+ plain event at the following coordinates in order. Every candidate
1479
+ is built and hashed by the selector's own constructors, so each
1480
+ touched id's occurrences, selected pair (its highest revision; a tie
1481
+ keeps the earlier occurrence) and history uncertainty are what
1482
+ :func:`build_five_hour_lineage` reads over a journal that really
1483
+ holds the records: a revision written above an admission-hidden id
1484
+ stays ``hidden-revision``, one above an inert revision leaves a
1485
+ ``revision-gap``. The close index gains every block a touched close
1486
+ id's new occurrence names.
1487
+ """
1488
+ if not self._coordinate_end:
1489
+ self._coordinate_end.append(max(
1490
+ (occurrence.available_at
1491
+ for table in (self.milestones, self.closes)
1492
+ for items in table.values() for occurrence in items),
1493
+ default=-1,
1494
+ ))
1495
+ coordinate = self._coordinate_end[0] + 1
1496
+ added: dict[str, list] = {}
1497
+ if batch_actions:
1498
+ name = batch_id or "lineage-extended"
1499
+ records = _lib_journal.make_correction_batch(
1500
+ batch_id=name, family="lineage-extended", at="extended",
1501
+ actions=[dict(action) for action in batch_actions])
1502
+ for record in records[1:-1]:
1503
+ candidate = _lib_journal._candidate_from_correction(
1504
+ record, coordinate)
1505
+ added.setdefault(candidate.event_id, []).append(candidate)
1506
+ coordinate += 1
1507
+ for record in plain:
1508
+ candidate = _lib_journal._candidate_from_evt(record, coordinate)
1509
+ added.setdefault(candidate.event_id, []).append(candidate)
1510
+ coordinate += 1
1511
+ if not added:
1512
+ return dataclasses.replace(
1513
+ self, _block_timelines={}, _coordinate_end=[])
1514
+ milestones = dict(self.milestones)
1515
+ closes = dict(self.closes)
1516
+ selected = dict(self.selected)
1517
+ uncertain = dict(self.uncertain)
1518
+ ever = {block: set(ids)
1519
+ for block, ids in self.close_ids_ever_in_block.items()}
1520
+ for event_id, candidates in added.items():
1521
+ existing = list(self.closes.get(event_id)
1522
+ or self.milestones.get(event_id) or ())
1523
+ identities: dict = {}
1524
+ for occurrence in existing:
1525
+ if occurrence.identity is not None:
1526
+ identities["identity"] = occurrence.identity
1527
+ fresh = []
1528
+ for candidate in candidates:
1529
+ occurrence, _kind, _unhashable = _lineage_occurrence(
1530
+ candidate.record, rev=candidate.rev,
1531
+ status=candidate.status,
1532
+ content_hash=candidate.content_hash,
1533
+ batch_id=candidate.batch_id,
1534
+ available_at=candidate.sequence, identities=identities)
1535
+ fresh.append(occurrence)
1536
+ items = existing + fresh
1537
+ kinds = {occurrence.kind for occurrence in items
1538
+ if occurrence.status == "active"}
1539
+ if not existing and not (kinds & _LINEAGE_KINDS):
1540
+ continue # not a five-hour milestone or close
1541
+ pair = selected.get(event_id)
1542
+ for candidate in candidates:
1543
+ if pair is None or candidate.rev > pair[0]:
1544
+ pair = (candidate.rev, candidate.content_hash)
1545
+ selected[event_id] = pair
1546
+ unhashable = any(
1547
+ occurrence.status == "active"
1548
+ and occurrence.kind == "five_hour_block_close"
1549
+ and occurrence.block is None
1550
+ for occurrence in items)
1551
+ items, codes = _finalize_lineage_id(
1552
+ event_id, items, kinds=kinds, unhashable=unhashable,
1553
+ selected=pair, hidden=event_id in self.hidden_ids)
1554
+ if codes:
1555
+ uncertain[event_id] = codes
1556
+ else:
1557
+ uncertain.pop(event_id, None)
1558
+ if "five_hour_block_close" in kinds:
1559
+ closes[event_id] = items
1560
+ for occurrence in items:
1561
+ if (occurrence.kind == "five_hour_block_close"
1562
+ and occurrence.block is not None):
1563
+ ever.setdefault(occurrence.block, set()).add(event_id)
1564
+ if "five_hour_milestone" in kinds:
1565
+ milestones[event_id] = items
1566
+ return FiveHourLineage(
1567
+ milestones=milestones,
1568
+ closes=closes,
1569
+ close_ids_ever_in_block={
1570
+ block: tuple(sorted(ids)) for block, ids in ever.items()},
1571
+ hidden_ids=self.hidden_ids,
1572
+ selected=selected,
1573
+ uncertain=uncertain,
1574
+ )
1575
+
1576
+
1577
+ def _lineage_occurrence(record, *, rev: int, status: str, content_hash: str,
1578
+ batch_id, available_at: int, identities: dict):
1579
+ """§5.1: one admissible candidate summarized as a :class:`LineageOccurrence`.
1580
+
1581
+ Returns ``(occurrence, kind, unhashable)``: ``kind`` is the active
1582
+ payload's kind (``None`` for a tombstone or a non-object payload) and
1583
+ ``unhashable`` marks a close whose block key indexes no block (§5.2,
1584
+ finding D6). ``identities`` shares one identity object per run of equal
1585
+ milestone identities of an id.
1586
+ """
1587
+ fields: dict = {}
1588
+ kind = None
1589
+ unhashable = False
1590
+ if status == "active":
1591
+ payload = (record or {}).get("payload")
1592
+ kind = payload.get("kind") if isinstance(payload, dict) else None
1593
+ if kind == "five_hour_milestone":
1594
+ identity = _lineage_identity(payload)
1595
+ prior = identities.get("identity")
1596
+ if prior is not None and prior == identity:
1597
+ identity = prior # one shared object per run of equals
1598
+ identities["identity"] = identity
1599
+ fields = dict(
1600
+ block=_five_hour_block(payload),
1601
+ at=record.get("at"), identity=identity,
1602
+ block_cost_usd=payload.get("block_cost_usd"),
1603
+ )
1604
+ elif kind == "five_hour_block_close":
1605
+ block = _five_hour_block(payload)
1606
+ try:
1607
+ hash(block)
1608
+ except TypeError:
1609
+ # §5.2 (finding D6): a block key that indexes no block
1610
+ # makes the id ``invalid-payload``; the plan goes on.
1611
+ unhashable = True
1612
+ block = None
1613
+ fields = dict(
1614
+ block=block,
1615
+ is_closed=payload.get("is_closed"),
1616
+ total_cost_usd=payload.get("total_cost_usd"),
1617
+ close_view=_canonical_bytes([
1618
+ kind, payload.get("account_key"),
1619
+ payload.get("five_hour_window_key"),
1620
+ payload.get("is_closed"),
1621
+ payload.get("total_cost_usd"),
1622
+ ]),
1623
+ )
1624
+ fields.setdefault("block", None)
1625
+ return LineageOccurrence(
1626
+ rev=rev, status=status, content_hash=content_hash, batch_id=batch_id,
1627
+ available_at=available_at, kind=kind, **fields,
1628
+ ), kind, unhashable
1629
+
1630
+
1631
+ def _finalize_lineage_id(event_id: str, items: list, *, kinds, unhashable: bool,
1632
+ selected, hidden: bool) -> tuple:
1633
+ """§5.1/§5.2: one id's occurrences in canonical order and its own
1634
+ history uncertainty. Sorts ``items`` in place; returns
1635
+ ``(tuple(items), codes)``."""
1636
+ items.sort(key=lambda item: (
1637
+ item.rev, item.available_at, item.status, item.content_hash,
1638
+ item.batch_id or "",
1639
+ ))
1640
+ codes = _history_uncertainty(
1641
+ items, hidden=hidden,
1642
+ invalid=len(kinds) != 1 or unhashable,
1643
+ selected=selected,
1644
+ close=set(kinds) == {"five_hour_block_close"},
1645
+ )
1646
+ return tuple(items), codes
1647
+
1648
+
1649
+ def build_five_hour_lineage(fold, *, selection,
1650
+ physical_revisions) -> FiveHourLineage:
1651
+ """§5.1: one pass over the selector fold's admissible candidates.
1652
+
1653
+ ``fold`` is the :class:`SelectorFold` ``resolve_effective_events`` built
1654
+ (its ``candidates`` hold every rev-0 event and every action of each
1655
+ completed, untainted batch); ``physical_revisions`` maps an id to its
1656
+ highest admission-hidden revision. Every candidate of an id some active
1657
+ candidate places in a five-hour milestone or close is kept, identical
1658
+ duplicates in different batches included.
1659
+ """
1660
+ wanted = set()
1661
+ for candidate in fold.candidates:
1662
+ if candidate.status == "active" and candidate.record is not None:
1663
+ payload = candidate.record.get("payload")
1664
+ if isinstance(payload, dict) and payload.get("kind") in \
1665
+ _LINEAGE_KINDS:
1666
+ wanted.add(candidate.event_id)
1667
+ commits: dict[str, int] = {}
1668
+ kinds: dict[str, set] = {}
1669
+ occurrences: dict[str, list] = {}
1670
+ identities: dict[str, dict] = {}
1671
+ unhashable_ids: set = set()
1672
+ for candidate in fold.candidates:
1673
+ event_id = candidate.event_id
1674
+ if event_id not in wanted:
1675
+ continue
1676
+ if candidate.batch_id is None:
1677
+ available_at = candidate.sequence
1678
+ else:
1679
+ available_at = commits.get(candidate.batch_id)
1680
+ if available_at is None:
1681
+ available_at = fold.markers[candidate.batch_id]["commit"][3]
1682
+ commits[candidate.batch_id] = available_at
1683
+ occurrence, kind, unhashable = _lineage_occurrence(
1684
+ candidate.record, rev=candidate.rev, status=candidate.status,
1685
+ content_hash=candidate.content_hash, batch_id=candidate.batch_id,
1686
+ available_at=available_at,
1687
+ identities=identities.setdefault(event_id, {}),
1688
+ )
1689
+ if candidate.status == "active":
1690
+ kinds.setdefault(event_id, set()).add(kind)
1691
+ if unhashable:
1692
+ unhashable_ids.add(event_id)
1693
+ occurrences.setdefault(event_id, []).append(occurrence)
1694
+ hidden_ids = frozenset(physical_revisions or ())
1695
+ milestones: dict[str, tuple] = {}
1696
+ closes: dict[str, tuple] = {}
1697
+ close_ids_ever_in_block: dict[tuple, set] = {}
1698
+ selected: dict[str, tuple] = {}
1699
+ uncertain: dict[str, tuple] = {}
1700
+ for event_id, items in occurrences.items():
1701
+ event_kinds = kinds.get(event_id, set())
1702
+ winner = selection.by_id.get(event_id)
1703
+ if winner is not None:
1704
+ selected[event_id] = (winner.rev, winner.content_hash)
1705
+ items, codes = _finalize_lineage_id(
1706
+ event_id, items, kinds=event_kinds,
1707
+ unhashable=event_id in unhashable_ids,
1708
+ selected=selected.get(event_id),
1709
+ hidden=event_id in hidden_ids,
1710
+ )
1711
+ if codes:
1712
+ uncertain[event_id] = codes
1713
+ if "five_hour_block_close" in event_kinds:
1714
+ closes[event_id] = items
1715
+ for item in items:
1716
+ if (item.kind == "five_hour_block_close"
1717
+ and item.block is not None):
1718
+ close_ids_ever_in_block.setdefault(
1719
+ item.block, set()).add(event_id)
1720
+ if "five_hour_milestone" in event_kinds:
1721
+ milestones[event_id] = items
1722
+ return FiveHourLineage(
1723
+ milestones=milestones,
1724
+ closes=closes,
1725
+ close_ids_ever_in_block={
1726
+ block: tuple(sorted(ids))
1727
+ for block, ids in close_ids_ever_in_block.items()
1728
+ },
1729
+ hidden_ids=hidden_ids,
1730
+ selected=selected,
1731
+ uncertain=uncertain,
1732
+ )
1733
+
1734
+
1735
+ @dataclasses.dataclass(frozen=True)
1736
+ class WithheldMilestone:
1737
+ """One reported fail-closed milestone of a withheld block (§5.7)."""
1738
+ event_id: str
1739
+ current_revision: int
1740
+ candidate_revisions: tuple
1741
+ reason_codes: tuple
1742
+
1743
+
1744
+ @dataclasses.dataclass(frozen=True)
1745
+ class WithheldBlock:
1746
+ """One five-hour block whose milestone repair was withheld (§5.7)."""
1747
+ account_key: object
1748
+ five_hour_window_key: object
1749
+ close_event_ids: tuple
1750
+ milestones: tuple
1751
+
1752
+ def to_dict(self) -> dict:
1753
+ return {
1754
+ "accountKey": self.account_key,
1755
+ "fiveHourWindowKey": self.five_hour_window_key,
1756
+ "closeEventIds": list(self.close_event_ids),
1757
+ "milestones": [
1758
+ {
1759
+ "eventId": milestone.event_id,
1760
+ "currentRevision": milestone.current_revision,
1761
+ "candidateRevisions": list(milestone.candidate_revisions),
1762
+ "reasonCodes": list(milestone.reason_codes),
1763
+ }
1764
+ for milestone in self.milestones
1765
+ ],
1766
+ }
1767
+
1768
+
1769
+ def _withheld_blocks(withheld: Mapping, current: Mapping[str, Mapping],
1770
+ actions, selection) -> tuple:
1771
+ """§5.7: group reported withholds by block, sorted, with the block's
1772
+ close ids in the plan's final state (retained, acted-on or preserved
1773
+ records; a revival is an acted-on record)."""
1774
+ if not withheld:
1775
+ return ()
1776
+ final = dict(current)
1777
+ for action in actions:
1778
+ if action.disposition == "tombstone":
1779
+ final.pop(action.event_id, None)
1780
+ else:
1781
+ final[action.event_id] = {"payload": action.payload or {}}
1782
+ close_ids: dict[tuple, set] = {}
1783
+ for event_id, record in final.items():
1784
+ payload = (record or {}).get("payload") or {}
1785
+ if payload.get("kind") == "five_hour_block_close":
1786
+ close_ids.setdefault(_five_hour_block(payload), set()).add(
1787
+ event_id)
1788
+ grouped: dict[tuple, list] = {}
1789
+ for event_id, decision in withheld.items():
1790
+ payload = (current.get(event_id) or {}).get("payload") or {}
1791
+ winner = selection.by_id.get(event_id)
1792
+ grouped.setdefault(_five_hour_block(payload), []).append(
1793
+ WithheldMilestone(
1794
+ event_id=event_id,
1795
+ current_revision=None if winner is None else int(winner.rev),
1796
+ candidate_revisions=tuple(sorted(
1797
+ decision.candidate_revisions)),
1798
+ reason_codes=tuple(sorted(set(decision.reason_codes))),
1799
+ ))
1800
+ blocks = [
1801
+ WithheldBlock(
1802
+ account_key=block[0], five_hour_window_key=block[1],
1803
+ close_event_ids=tuple(sorted(close_ids.get(block, ()))),
1804
+ milestones=tuple(sorted(items, key=lambda item: item.event_id)),
1805
+ )
1806
+ for block, items in grouped.items()
1807
+ ]
1808
+ blocks.sort(key=lambda item: (
1809
+ "" if item.account_key is None else str(item.account_key),
1810
+ str(item.five_hour_window_key),
1811
+ ))
1812
+ return tuple(blocks)
1813
+
1814
+
729
1815
  def _resolve_closed_block_milestones(
730
1816
  current: Mapping[str, Mapping], desired: Mapping[str, Mapping],
731
1817
  preserved: Mapping[str, Mapping], frozen_blocks, replaced_blocks, *,
732
- selection,
1818
+ selection, lineage: "FiveHourLineage | None" = None,
1819
+ withheld: "dict | None" = None,
733
1820
  ) -> dict[str, dict]:
734
1821
  """#875: target records of the five-hour milestones of closed blocks.
735
1822
 
@@ -743,6 +1830,13 @@ def _resolve_closed_block_milestones(
743
1830
  and non-monetary identity are both unchanged keeps its current
744
1831
  ``block_cost_usd`` across any card edit, and a crossing whose ``at`` or
745
1832
  identity differs is a corrected crossing that takes replay's block cost.
1833
+
1834
+ #884 (spec §5.5): with ``lineage``, such an unchanged crossing instead
1835
+ takes its history-proven anchor amount when an earlier rederive repriced
1836
+ it (:meth:`FiveHourLineage.restoration`). When its history is ambiguous
1837
+ it keeps the current amount, and a reported withhold is recorded in
1838
+ ``withheld`` as ``{event_id: Restoration}``. No other role is restored,
1839
+ and restoration never changes a block's frozen or replaced class.
746
1840
  A REPLACED close is one this plan adds, or corrects so that it writes the
747
1841
  close's money (it closes an open block, or its totals are unusable or
748
1842
  differ by more), and every matched milestone under it is a corrected
@@ -765,13 +1859,15 @@ def _resolve_closed_block_milestones(
765
1859
  crossing: replay never reproduces preserved ``b:`` predecessors, and a
766
1860
  reviewed hold can remove a desired predecessor before the diff. It comes
767
1861
  from the desired record for a corrected crossing or an addition. A
768
- candidate keeps its whole chosen monetary pair (the current pair for an
769
- unchanged crossing, the desired pair otherwise) rather than inventing a
770
- value when it has no usable predecessor, or when its own threshold or
771
- block cost is unusable. Only an ``int`` threshold occupies a position,
772
- and only a finite amount is usable. A row whose threshold is usable but
773
- whose block cost is not still occupies its threshold, so a successor
774
- right above it fails closed and never skips to a lower row.
1862
+ candidate keeps its chosen marginal (the current one for an unchanged
1863
+ crossing, the desired one otherwise) rather than inventing a value when
1864
+ it has no usable predecessor, or when its own threshold or block cost is
1865
+ unusable. Its block cost is already settled and is never reset there, so
1866
+ a restored amount survives (#884 spec §5.6). Only an ``int`` threshold
1867
+ occupies a position, and only a finite amount is usable. A row whose
1868
+ threshold is usable but whose block cost is not still occupies its
1869
+ threshold, so a successor right above it fails closed and never skips to
1870
+ a lower row.
775
1871
 
776
1872
  A second plan is a no-op for a block whose close it takes no action on.
777
1873
  That close is frozen in the next plan, kept or preserved, whatever role
@@ -833,6 +1929,16 @@ def _resolve_closed_block_milestones(
833
1929
  payload["block_cost_usd"] = current_payload.get(
834
1930
  "block_cost_usd")
835
1931
  roles[event_id] = "unchanged"
1932
+ if lineage is not None:
1933
+ # #884 (spec §5.5): a proven historical amount replaces
1934
+ # one an earlier rederive repriced; an ambiguous history
1935
+ # keeps the current amount and is disclosed.
1936
+ decision = lineage.restoration(event_id, current_record)
1937
+ if decision.action == "restore":
1938
+ payload["block_cost_usd"] = decision.value
1939
+ elif (decision.action == "withhold" and decision.reported
1940
+ and withheld is not None):
1941
+ withheld[event_id] = decision
836
1942
  else:
837
1943
  roles[event_id] = "corrected"
838
1944
  record = {**carried, "payload": payload}
@@ -891,7 +1997,8 @@ def _resolve_closed_block_milestones(
891
1997
  ):
892
1998
  prior = predecessor_cost(payload)
893
1999
  if prior is None:
894
- payload["block_cost_usd"] = chosen.get("block_cost_usd")
2000
+ # #884 (spec §5.6): only the marginal falls back. The block cost
2001
+ # is already the chosen record's, or a restored amount.
895
2002
  payload["marginal_cost_usd"] = chosen.get("marginal_cost_usd")
896
2003
  continue
897
2004
  value = float(payload["block_cost_usd"]) - float(prior)
@@ -907,6 +2014,769 @@ def _resolve_closed_block_milestones(
907
2014
  return resolved
908
2015
 
909
2016
 
2017
+ # ── #1040: derivation completion keeps closed five-hour money frozen ───────
2018
+ #
2019
+ # Derivation completion journals, for every id the gap acts on (its with-gap
2020
+ # and without-gap candidates differ), the with-gap candidate. For a closed
2021
+ # five-hour block that candidate carries the money the scratch replay
2022
+ # computed at the binary's CURRENT card, so a metadata-only gap repriced the
2023
+ # frozen block (spec §1–§3). The rules below decide what completion appends
2024
+ # for the closes and milestones of every touched block instead (spec §5).
2025
+
2026
+ COMPLETION_FROZEN = "FROZEN"
2027
+ COMPLETION_REPLACED = "REPLACED"
2028
+ COMPLETION_ADDED = "ADDED"
2029
+ COMPLETION_OPEN = "OPEN"
2030
+
2031
+
2032
+ @dataclasses.dataclass(frozen=True)
2033
+ class CompletionClose:
2034
+ """#1040 (spec §5.2): one close's proposed target and class.
2035
+
2036
+ ``target`` is T (``None`` keeps nothing to append, or a tombstone on the
2037
+ OPEN path); ``frozen_form`` is T₀, defined for a close whose retained and
2038
+ with-gap records are both closed, else ``None``.
2039
+ """
2040
+ block_class: str
2041
+ target: "Mapping | None"
2042
+ frozen_form: "Mapping | None" = None
2043
+
2044
+
2045
+ def _closed_record(record: "Mapping | None") -> bool:
2046
+ return (record is not None
2047
+ and (record.get("payload") or {}).get("is_closed") == 1)
2048
+
2049
+
2050
+ def _total_moves(before: "Mapping", after: "Mapping") -> bool:
2051
+ """Both totals usable and more than 1e-9 USD apart."""
2052
+ left = (before.get("payload") or {}).get("total_cost_usd")
2053
+ right = (after.get("payload") or {}).get("total_cost_usd")
2054
+ return (_usable_amount(left) and _usable_amount(right)
2055
+ and abs(right - left) > _CLOSE_TOTAL_TOLERANCE_USD)
2056
+
2057
+
2058
+ def _is_missing_card_correction(preserved, current, desired) -> bool:
2059
+ """§5.2: a preserved result that is neither D nor C and moves C's total.
2060
+
2061
+ ``_with_close_marker`` builds a #869 missing-card correction as a new
2062
+ record under the same structure, so identity tells it apart from a kept
2063
+ (``current``) or structurally different (``desired``) result.
2064
+ """
2065
+ return (preserved is not current and preserved is not desired
2066
+ and _total_moves(current, preserved))
2067
+
2068
+
2069
+ def classify_completion_close(
2070
+ current: "Mapping | None", desired: "Mapping | None",
2071
+ reference: "Mapping | None", *, acted: bool, has_physical: bool,
2072
+ several_close_ids: bool,
2073
+ ) -> CompletionClose:
2074
+ """#1040 (spec §5.2): the proposed close target T and the block class.
2075
+
2076
+ ``current`` is C (the selected active close, or ``None``), ``desired`` D
2077
+ and ``reference`` R. The rules apply in order:
2078
+
2079
+ 1. not acted on: T = C, FROZEN when C is closed, else OPEN;
2080
+ 2. D absent or not closed, or C present and not closed: OPEN (today's
2081
+ path; no closed money exists to keep). OPEN is this close's class:
2082
+ the block is OPEN only when every close of it is (spec §5.2 rule 2,
2083
+ "no closed close in the final state");
2084
+ 3. D closed, no C: ADDED (plain revision 0) without a physical revision,
2085
+ else a REPLACED revival;
2086
+ 4. C and D closed: T₀ first (``overlay(D, C)`` when preservation returns
2087
+ D, else C), then 4.1 a gap-caused population change, 4.2 a #869
2088
+ missing-card correction P, 4.3 the same-round P₂ = preserve(T₀, D),
2089
+ 4.4 FROZEN with T = T₀.
2090
+
2091
+ A block naming several close ids is never REPLACED: rules 4.1–4.3 yield
2092
+ T₀ and FROZEN, and a revival proposes D under FROZEN, whose lineage check
2093
+ decides whether it can land.
2094
+ """
2095
+ if not acted:
2096
+ return CompletionClose(
2097
+ COMPLETION_FROZEN if _closed_record(current) else COMPLETION_OPEN,
2098
+ current)
2099
+ if not _closed_record(desired) or (
2100
+ current is not None and not _closed_record(current)):
2101
+ return CompletionClose(COMPLETION_OPEN, desired)
2102
+ if current is None:
2103
+ if not has_physical:
2104
+ return CompletionClose(COMPLETION_ADDED, desired)
2105
+ if several_close_ids:
2106
+ return CompletionClose(COMPLETION_FROZEN, desired)
2107
+ return CompletionClose(COMPLETION_REPLACED, desired)
2108
+ preserved = _preserve_non_derivable_state(current, desired)
2109
+ frozen = (close_money_overlay(desired, current)
2110
+ if preserved is desired else current)
2111
+ if several_close_ids:
2112
+ return CompletionClose(COMPLETION_FROZEN, frozen, frozen)
2113
+ current_payload = current.get("payload") or {}
2114
+ desired_payload = desired.get("payload") or {}
2115
+ if (
2116
+ _closed_record(reference)
2117
+ and _usable_amount(current_payload.get("total_cost_usd"))
2118
+ and _usable_amount(desired_payload.get("total_cost_usd"))
2119
+ and close_population(current_payload)
2120
+ == close_population(reference.get("payload") or {})
2121
+ and close_population(desired_payload)
2122
+ != close_population(current_payload)
2123
+ and _total_moves(current, desired)
2124
+ ):
2125
+ return CompletionClose(COMPLETION_REPLACED, desired, frozen)
2126
+ if _is_missing_card_correction(preserved, current, desired):
2127
+ return CompletionClose(COMPLETION_REPLACED, preserved, frozen)
2128
+ if _semantic_event(frozen) != _semantic_event(current):
2129
+ second = _preserve_non_derivable_state(frozen, desired)
2130
+ if _is_missing_card_correction(second, frozen, desired):
2131
+ return CompletionClose(COMPLETION_REPLACED, second, frozen)
2132
+ return CompletionClose(COMPLETION_FROZEN, frozen, frozen)
2133
+
2134
+ #: The milestone fields a FROZEN or ADDED block's acted crossing keeps from
2135
+ #: its retained record (spec §5.3): its money and the block tokens it was
2136
+ #: priced from.
2137
+ _COMPLETION_MILESTONE_KEPT_KEYS = (
2138
+ "block_cost_usd", "marginal_cost_usd",
2139
+ *(f"block_{key}" for key in _BLOCK_TOKEN_KEYS),
2140
+ )
2141
+
2142
+ COMPLETION_TREATMENTS = ("frozen", "replaced", "marginal", "none", "main")
2143
+
2144
+
2145
+ def _milestone_overlay(desired: Mapping, current: Mapping) -> dict:
2146
+ """D with C's money and block tokens (a key C lacks is removed)."""
2147
+ payload = dict(desired.get("payload") or {})
2148
+ current_payload = current.get("payload") or {}
2149
+ for key in _COMPLETION_MILESTONE_KEPT_KEYS:
2150
+ if key in current_payload:
2151
+ payload[key] = current_payload[key]
2152
+ else:
2153
+ payload.pop(key, None)
2154
+ return {**desired, "payload": payload}
2155
+
2156
+
2157
+ def completion_block_milestones(
2158
+ *, treatment: str, block_class: str, current: Mapping[str, Mapping],
2159
+ with_gap: Mapping[str, Mapping], acted: frozenset, retired: frozenset,
2160
+ lineage: FiveHourLineage, selection,
2161
+ fixed: "Mapping[str, Mapping] | None" = None,
2162
+ ) -> dict[str, dict]:
2163
+ """#1040 (spec §5.3, §5.4): one block's milestone targets.
2164
+
2165
+ ``current`` holds the block's selected active milestones (the whole
2166
+ journal's, so predecessors before a trusted frontier are included) and
2167
+ ``with_gap`` the block's with-gap milestone candidates. ``retired`` are
2168
+ the acted milestones with a retained record and no with-gap one: today's
2169
+ path tombstones them, so they are no predecessor in any final chain.
2170
+ ``fixed`` are records of ids the planner does not own that today's path
2171
+ leaves in this block: they stand in the final chain as given and are
2172
+ never returned. Returns every resolved record that differs semantically
2173
+ from its retained record or has none.
2174
+
2175
+ A **required addition** is an acted with-gap id absent from
2176
+ ``selection.by_id`` (spec §5.4): no admitted decision, active or
2177
+ tombstoned, answers it. An acted with-gap id without C that the
2178
+ selection does know (a tombstoned milestone, revived) is an optional
2179
+ rewrite. ``treatment``:
2180
+
2181
+ * ``"frozen"``: an acted crossing with C takes D with C's money and
2182
+ block tokens; when that is a corrected crossing (its ``at`` or
2183
+ identity differs) its block cost is the lineage's restoration value
2184
+ when :meth:`FiveHourLineage.restoration` says ``restore``. An acted id
2185
+ without C takes D (required or revived), every other id C.
2186
+ * ``"replaced"`` (REPLACED blocks only): an id the with-gap replay
2187
+ carries takes D, acted or not; a current id it does not carry takes C.
2188
+ * ``"marginal"``: every existing id keeps C; required additions take D;
2189
+ only the marginals are resolved over the final chain.
2190
+ * ``"none"``: the required additions alone, their marginals resolved
2191
+ over the chain whose existing ids are C.
2192
+ * ``"main"``: each acted id's record as today's completion path appends
2193
+ it (``_preserve_non_derivable_state(C, D)``, or D), nothing else and
2194
+ no marginal resolution.
2195
+
2196
+ The kernel is :func:`_resolve_closed_block_milestones` with the block
2197
+ FROZEN (``replaced`` treatment: REPLACED) and no lineage.
2198
+ """
2199
+ if treatment not in COMPLETION_TREATMENTS:
2200
+ raise ValueError(f"unknown milestone treatment {treatment!r}")
2201
+ if treatment == "replaced" and block_class != COMPLETION_REPLACED:
2202
+ raise ValueError("the replaced treatment needs a REPLACED block")
2203
+ fixed = dict(fixed or {})
2204
+ if treatment == "main":
2205
+ out = {}
2206
+ for event_id in sorted(acted & set(with_gap)):
2207
+ desired = with_gap[event_id]
2208
+ existing = current.get(event_id)
2209
+ record = (desired if existing is None
2210
+ else _preserve_non_derivable_state(existing, desired))
2211
+ if existing is None or (_semantic_event(record)
2212
+ != _semantic_event(existing)):
2213
+ out[event_id] = dict(record)
2214
+ return out
2215
+ desired: dict[str, Mapping] = {}
2216
+ for event_id, existing in current.items():
2217
+ if event_id in retired:
2218
+ continue
2219
+ candidate = with_gap.get(event_id)
2220
+ if treatment == "replaced" and candidate is not None:
2221
+ desired[event_id] = candidate
2222
+ elif (treatment == "frozen" and candidate is not None
2223
+ and event_id in acted):
2224
+ record = _milestone_overlay(candidate, existing)
2225
+ existing_payload = existing.get("payload") or {}
2226
+ if (record.get("at") != existing.get("at")
2227
+ or _lineage_identity(record["payload"])
2228
+ != _lineage_identity(existing_payload)):
2229
+ decision = lineage.restoration(event_id, existing)
2230
+ if decision.action == "restore":
2231
+ record["payload"]["block_cost_usd"] = decision.value
2232
+ desired[event_id] = record
2233
+ else:
2234
+ desired[event_id] = existing
2235
+ required = set()
2236
+ for event_id, candidate in with_gap.items():
2237
+ if event_id in current or event_id not in acted:
2238
+ continue
2239
+ if event_id not in selection.by_id:
2240
+ required.add(event_id)
2241
+ elif treatment in ("marginal", "none"):
2242
+ continue # an optional revival: only F, K and R rewrite
2243
+ desired[event_id] = candidate
2244
+ desired.update(fixed)
2245
+ if not desired:
2246
+ return {}
2247
+ block = _five_hour_block(
2248
+ (next(iter(desired.values())).get("payload") or {}))
2249
+ replaced = treatment == "replaced"
2250
+ resolved = _resolve_closed_block_milestones(
2251
+ # A fixed record is today's: it stands in the chain at its own money.
2252
+ {event_id: record for event_id, record in current.items()
2253
+ if event_id not in fixed},
2254
+ desired, {},
2255
+ frozenset() if replaced else frozenset({block}),
2256
+ frozenset({block}) if replaced else frozenset(),
2257
+ selection=selection, lineage=None,
2258
+ )
2259
+ out = {}
2260
+ for event_id in sorted(resolved):
2261
+ if event_id in fixed:
2262
+ continue
2263
+ if treatment == "none" and event_id not in required:
2264
+ continue
2265
+ existing = current.get(event_id)
2266
+ record = resolved[event_id]
2267
+ if existing is None or (_semantic_event(record)
2268
+ != _semantic_event(existing)):
2269
+ out[event_id] = record
2270
+ return out
2271
+
2272
+ @dataclasses.dataclass(frozen=True)
2273
+ class BlockDecision:
2274
+ """#1040 (spec §5.4): one touched block's lineage-checked decision.
2275
+
2276
+ ``landed`` names the candidate whose records land (``"class"``, ``"F"``,
2277
+ ``"P"``, ``"K"``, ``"M"`` or ``"A"``); ``violations`` maps every
2278
+ candidate the decision simulated to its violation keys.
2279
+ """
2280
+ block: tuple
2281
+ block_class: str
2282
+ component: tuple
2283
+ landed: str
2284
+ violations: Mapping[str, frozenset]
2285
+
2286
+
2287
+ @dataclasses.dataclass(frozen=True)
2288
+ class CompletionFiveHourPlan:
2289
+ """#1040: what derivation completion appends for touched blocks.
2290
+
2291
+ ``targets`` maps an id to the record to append (each differs from its
2292
+ retained record or has none); ``handled`` are the five-hour ids whose
2293
+ every record lies in a decided block, retirements and the acted closes
2294
+ left to today's path excluded, which today's per-id path must not append
2295
+ again (``targets`` is a subset); ``decisions`` are sorted by block.
2296
+ """
2297
+ targets: Mapping[str, Mapping]
2298
+ handled: frozenset
2299
+ decisions: tuple
2300
+
2301
+
2302
+ #: §5.4: the candidates of each block class, in decision order.
2303
+ _COMPLETION_CANDIDATE_ORDER = {
2304
+ COMPLETION_FROZEN: ("class", "K", "M", "A"),
2305
+ COMPLETION_REPLACED: ("class", "F", "K", "M", "A"),
2306
+ COMPLETION_ADDED: ("class", "P", "M", "A"),
2307
+ }
2308
+ #: §5.4: each later candidate's milestone treatment.
2309
+ _COMPLETION_CANDIDATE_TREATMENT = {
2310
+ "F": "frozen", "P": "main", "K": "frozen", "M": "marginal", "A": "none",
2311
+ }
2312
+ _RESTORATION_RANK = {"keep": 0, "restore": 1, "withhold": 2}
2313
+
2314
+
2315
+ def _selected_milestone_view(lineage: FiveHourLineage, event_id: str) -> dict:
2316
+ """The selected active occurrence's block cost, as the record
2317
+ :meth:`FiveHourLineage.restoration` reads (``{}`` when there is none)."""
2318
+ pair = lineage.selected.get(event_id)
2319
+ for occurrence in lineage.milestones.get(event_id, ()):
2320
+ if (pair is not None and occurrence.status == "active"
2321
+ and (occurrence.rev, occurrence.content_hash) == pair):
2322
+ return {"payload": {"block_cost_usd": occurrence.block_cost_usd}}
2323
+ return {"payload": {}}
2324
+
2325
+
2326
+ def _violation_keys(before: FiveHourLineage, after: FiveHourLineage,
2327
+ milestone_ids, records: Mapping[str, Mapping]) -> frozenset:
2328
+ """#1040 (spec §5.4): how a candidate makes #884 decisions worse.
2329
+
2330
+ For each milestone, its decision in ``after`` against its decision in
2331
+ ``before``; a milestone ``before`` does not know counts as ``keep`` with
2332
+ no reason. ``records`` are the candidate's records by id; a milestone it
2333
+ does not name, and every ``before`` decision, reads the lineage's
2334
+ selected occurrence. Keys: ``(id, "action", a)`` when the action rises
2335
+ to ``a`` (``keep`` < ``restore`` < ``withhold``), ``(id, "reason",
2336
+ code)`` per gained reason code, and ``(id, "moved")`` when both are
2337
+ ``restore`` to different values.
2338
+ """
2339
+ keys = set()
2340
+ for event_id in sorted(milestone_ids):
2341
+ old = before.restoration(
2342
+ event_id, _selected_milestone_view(before, event_id))
2343
+ record = records.get(event_id)
2344
+ new = after.restoration(
2345
+ event_id, record if record is not None
2346
+ else _selected_milestone_view(after, event_id))
2347
+ if _RESTORATION_RANK[new.action] > _RESTORATION_RANK[old.action]:
2348
+ keys.add((event_id, "action", new.action))
2349
+ for code in set(new.reason_codes) - set(old.reason_codes):
2350
+ keys.add((event_id, "reason", code))
2351
+ if (old.action == new.action == "restore"
2352
+ and old.value != new.value):
2353
+ keys.add((event_id, "moved"))
2354
+ return frozenset(keys)
2355
+
2356
+
2357
+ def _hashable_block(payload: Mapping):
2358
+ block = _five_hour_block(payload)
2359
+ try:
2360
+ hash(block)
2361
+ except TypeError:
2362
+ return None
2363
+ return block
2364
+
2365
+
2366
+ def plan_completion_five_hour(
2367
+ *, current: Mapping[str, Mapping], with_gap: Mapping[str, Mapping],
2368
+ without_gap: Mapping[str, Mapping], acted: frozenset,
2369
+ physical: Mapping[str, int], selection, lineage: FiveHourLineage,
2370
+ retirements: Sequence[Mapping],
2371
+ ) -> CompletionFiveHourPlan:
2372
+ """#1040 (spec §5.1–§5.5): completion's five-hour targets, pure.
2373
+
2374
+ ``current`` holds the selected active five-hour records of the whole
2375
+ journal; ``with_gap`` / ``without_gap`` are the round's candidates by
2376
+ id (any kind) and ``acted`` the ids whose two candidates differ;
2377
+ ``physical`` maps an id to its highest physical revision (selected,
2378
+ admission-excluded or inert); ``retirements`` are the tombstone actions
2379
+ today's path appends, which land whatever is chosen and so ride along in
2380
+ every simulation.
2381
+
2382
+ A block is touched when an acted close or milestone names it. Each
2383
+ touched block's closes are classified (:func:`classify_completion_close`).
2384
+ A block is OPEN, and keeps today's path, when no close it decides is
2385
+ closed in the final state; in a block that stays decided an acted close
2386
+ of rule 2 still keeps today's path. The planner owns an id only when
2387
+ every block its retained, with-gap and without-gap records name is
2388
+ decided (``handled``); it never targets another id, and every
2389
+ simulation carries today's record of each acted id it leaves to that
2390
+ path, like the retirements. Every decided block builds its whole
2391
+ close-and-milestone candidates, each carrying the block's required
2392
+ additions (records whose id the selection does not know), and the
2393
+ candidates of a component of blocks that share a close or milestone id
2394
+ are simulated through :meth:`FiveHourLineage.extended` with the append
2395
+ policy's coordinates. A component with one touched block lands the
2396
+ first candidate whose violations are contained in its additions' alone
2397
+ and strictly exceed no other candidate's; a class proposal without
2398
+ violations lands at once. Several touched blocks choose likewise with
2399
+ each other's additions carried, and their union lands only when it adds
2400
+ nothing beyond all their additions together; otherwise each lands its
2401
+ additions alone.
2402
+ """
2403
+ def payload_of(record):
2404
+ return (record or {}).get("payload") or {}
2405
+
2406
+ def five_hour(record):
2407
+ return (record is not None
2408
+ and payload_of(record).get("kind") in _LINEAGE_KINDS)
2409
+
2410
+ def by_block(records):
2411
+ out: dict = {}
2412
+ for event_id, record in records.items():
2413
+ if five_hour(record):
2414
+ block = _hashable_block(payload_of(record))
2415
+ if block is not None:
2416
+ out.setdefault(block, {})[event_id] = record
2417
+ return out
2418
+
2419
+ current_by_block = by_block(current)
2420
+ with_by_block = by_block(with_gap)
2421
+ without_by_block = by_block(without_gap)
2422
+ touched = set()
2423
+ for event_id in acted:
2424
+ for record in (with_gap.get(event_id), current.get(event_id)):
2425
+ if five_hour(record):
2426
+ block = _hashable_block(payload_of(record))
2427
+ if block is not None:
2428
+ touched.add(block)
2429
+
2430
+ # Every block an id's retained, with-gap or without-gap record names
2431
+ # (an unusable block key stands for a block nothing decides).
2432
+ unusable_block = object()
2433
+ blocks_of_id: dict = {}
2434
+ for table in (current, with_gap, without_gap):
2435
+ for event_id, record in table.items():
2436
+ if five_hour(record):
2437
+ block = _hashable_block(payload_of(record))
2438
+ blocks_of_id.setdefault(event_id, set()).add(
2439
+ unusable_block if block is None else block)
2440
+
2441
+ def todays_record(event_id):
2442
+ """The record today's per-id path leaves active for ``event_id``."""
2443
+ existing = current.get(event_id)
2444
+ if event_id not in acted:
2445
+ return existing
2446
+ desired = with_gap.get(event_id)
2447
+ if desired is None:
2448
+ return None # a retirement tombstone, or nothing
2449
+ if existing is None:
2450
+ return desired
2451
+ return _preserve_non_derivable_state(existing, desired)
2452
+
2453
+ def classify_blocks(not_owned):
2454
+ """§5.2 per touched block: its class and the closes the planner
2455
+ decides. A block is OPEN only when no close it decides is closed in
2456
+ the final state; an acted close of rule 2 (and an acted close the
2457
+ planner does not own) keeps today's path."""
2458
+ classes, owned_closes, todays_closes = {}, {}, set()
2459
+ for block in sorted(touched, key=repr):
2460
+ close_ids = sorted({
2461
+ event_id for records in (
2462
+ current_by_block.get(block, {}),
2463
+ with_by_block.get(block, {}),
2464
+ without_by_block.get(block, {}))
2465
+ for event_id, record in records.items()
2466
+ if payload_of(record).get("kind") == "five_hour_block_close"
2467
+ })
2468
+ several = len(close_ids) > 1
2469
+ closes = {}
2470
+ for close_id in close_ids:
2471
+ close = classify_completion_close(
2472
+ current.get(close_id), with_gap.get(close_id),
2473
+ without_gap.get(close_id), acted=close_id in acted,
2474
+ has_physical=close_id in physical,
2475
+ several_close_ids=several)
2476
+ if close_id in acted and (
2477
+ close.block_class == COMPLETION_OPEN
2478
+ or close_id in not_owned):
2479
+ todays_closes.add(close_id)
2480
+ elif close.block_class != COMPLETION_OPEN:
2481
+ closes[close_id] = close
2482
+ if not closes:
2483
+ classes[block] = COMPLETION_OPEN
2484
+ continue
2485
+ kinds = {close.block_class for close in closes.values()}
2486
+ if len(closes) == 1:
2487
+ classes[block] = next(iter(kinds))
2488
+ elif kinds == {COMPLETION_ADDED}:
2489
+ classes[block] = COMPLETION_ADDED
2490
+ else:
2491
+ classes[block] = COMPLETION_FROZEN
2492
+ owned_closes[block] = closes
2493
+ return classes, owned_closes, todays_closes
2494
+
2495
+ # An id whose records reach a block the planner does not decide stays
2496
+ # on today's path (the planner never writes it); deciding fewer blocks
2497
+ # only adds such ids, so this settles.
2498
+ not_owned: set = set()
2499
+ while True:
2500
+ classes, owned_closes, todays_closes = classify_blocks(not_owned)
2501
+ decided_blocks = set(owned_closes)
2502
+ reaching = {
2503
+ event_id for event_id, blocks in blocks_of_id.items()
2504
+ if blocks & decided_blocks and not blocks <= decided_blocks}
2505
+ if reaching <= not_owned:
2506
+ break
2507
+ not_owned |= reaching
2508
+ not_owned |= todays_closes
2509
+
2510
+ candidates: dict = {}
2511
+ retired_ids: set = set()
2512
+ for block in sorted(owned_closes, key=repr):
2513
+ closes = owned_closes[block]
2514
+ block_class = classes[block]
2515
+ cur = current_by_block.get(block, {})
2516
+ wg = with_by_block.get(block, {})
2517
+ milestones_current = {
2518
+ event_id: record for event_id, record in cur.items()
2519
+ if payload_of(record).get("kind") == "five_hour_milestone"}
2520
+ milestones_with = {
2521
+ event_id: record for event_id, record in wg.items()
2522
+ if payload_of(record).get("kind") == "five_hour_milestone"
2523
+ and event_id not in not_owned}
2524
+ retired = frozenset(
2525
+ event_id for event_id in milestones_current
2526
+ if event_id in acted and with_gap.get(event_id) is None)
2527
+ retired_ids.update(retired)
2528
+ # Milestones of this block the planner does not own: today's record
2529
+ # stands in the chain when it stays here, else it leaves the chain.
2530
+ fixed: dict = {}
2531
+ leaving = set()
2532
+ for event_id in sorted(not_owned):
2533
+ if block not in blocks_of_id.get(event_id, ()):
2534
+ continue
2535
+ record = todays_record(event_id)
2536
+ if (record is not None and payload_of(record).get("kind")
2537
+ == "five_hour_milestone"
2538
+ and _hashable_block(payload_of(record)) == block):
2539
+ fixed[event_id] = record
2540
+ elif event_id in milestones_current:
2541
+ leaving.add(event_id)
2542
+ retired_chain = retired | frozenset(leaving)
2543
+
2544
+ def candidate(name, *, closes=closes, block_class=block_class,
2545
+ milestones_current=milestones_current,
2546
+ milestones_with=milestones_with,
2547
+ retired_chain=retired_chain, fixed=fixed):
2548
+ records: dict = {}
2549
+ for close_id, close in closes.items():
2550
+ existing = current.get(close_id)
2551
+ desired = with_gap.get(close_id)
2552
+ if name == "class":
2553
+ target = close.target
2554
+ elif name == "F":
2555
+ target = (close.frozen_form if existing is not None
2556
+ else desired if close_id in acted else None)
2557
+ elif name == "P":
2558
+ target = desired if close_id in acted else None
2559
+ else:
2560
+ target = None
2561
+ if (close_id in acted and desired is not None
2562
+ and close_id not in selection.by_id):
2563
+ target = desired # a required addition
2564
+ if target is not None and (
2565
+ existing is None
2566
+ or _semantic_event(target) != _semantic_event(existing)):
2567
+ records[close_id] = dict(target)
2568
+ treatment = (
2569
+ ("replaced" if block_class == COMPLETION_REPLACED
2570
+ else "frozen")
2571
+ if name == "class" else _COMPLETION_CANDIDATE_TREATMENT[name])
2572
+ records.update(completion_block_milestones(
2573
+ treatment=treatment, block_class=block_class,
2574
+ current=milestones_current, with_gap=milestones_with,
2575
+ acted=acted, retired=retired_chain, lineage=lineage,
2576
+ selection=selection, fixed=fixed))
2577
+ return records
2578
+
2579
+ candidates[block] = {
2580
+ name: candidate(name)
2581
+ for name in _COMPLETION_CANDIDATE_ORDER[block_class]}
2582
+
2583
+ decided = sorted(candidates, key=repr)
2584
+ # The planner owns an id only when every block its records name is
2585
+ # decided; today's path keeps every other id (spec §5.1).
2586
+ handled = {
2587
+ event_id for event_id, blocks in blocks_of_id.items()
2588
+ if blocks <= decided_blocks and event_id not in not_owned
2589
+ and event_id not in retired_ids}
2590
+ if not decided:
2591
+ return CompletionFiveHourPlan(
2592
+ targets={}, handled=frozenset(handled), decisions=())
2593
+ # Today's records of the acted ids the planner leaves to that path in a
2594
+ # decided block's reach: they land whatever is chosen, so every
2595
+ # candidate's simulation carries them (like the retirements).
2596
+ todays_records: dict = {}
2597
+ for event_id in sorted(not_owned & acted):
2598
+ if not blocks_of_id.get(event_id, set()) & decided_blocks:
2599
+ continue
2600
+ record = todays_record(event_id)
2601
+ existing = current.get(event_id)
2602
+ if record is not None and (
2603
+ existing is None
2604
+ or _semantic_event(record) != _semantic_event(existing)):
2605
+ todays_records[event_id] = dict(record)
2606
+
2607
+ # §5.4 components: blocks sharing a close or milestone id, over the
2608
+ # lineage's occurrences and every candidate's records.
2609
+ id_blocks: dict = {}
2610
+ block_ids: dict = {}
2611
+
2612
+ def link(event_id, block):
2613
+ if block is None:
2614
+ return
2615
+ id_blocks.setdefault(event_id, set()).add(block)
2616
+ block_ids.setdefault(block, set()).add(event_id)
2617
+
2618
+ for table in (lineage.milestones, lineage.closes):
2619
+ for event_id, items in table.items():
2620
+ for occurrence in items:
2621
+ if occurrence.status == "active" and occurrence.block is not None:
2622
+ try:
2623
+ hash(occurrence.block)
2624
+ except TypeError:
2625
+ continue
2626
+ link(event_id, occurrence.block)
2627
+ for block in decided:
2628
+ for records in candidates[block].values():
2629
+ for event_id, record in records.items():
2630
+ link(event_id, _hashable_block(payload_of(record)))
2631
+ for event_id, record in todays_records.items():
2632
+ link(event_id, _hashable_block(payload_of(record)))
2633
+ components: dict = {}
2634
+ for start in decided:
2635
+ if start in components:
2636
+ continue
2637
+ component: set = set()
2638
+ frontier = [start]
2639
+ while frontier:
2640
+ block = frontier.pop()
2641
+ if block in component:
2642
+ continue
2643
+ component.add(block)
2644
+ for event_id in block_ids.get(block, ()):
2645
+ frontier.extend(id_blocks.get(event_id, ()) - component)
2646
+ frozen_component = tuple(sorted(component, key=repr))
2647
+ for block in component:
2648
+ components[block] = frozen_component
2649
+
2650
+ order_index = {event_id: index for index, event_id in enumerate(with_gap)}
2651
+ retirement_blocks = []
2652
+ for action in retirements:
2653
+ record = current.get(action.get("id"))
2654
+ retirement_blocks.append(
2655
+ (action, _hashable_block(payload_of(record))
2656
+ if five_hour(record) else None))
2657
+
2658
+ def landed_records(choices):
2659
+ out: dict = {}
2660
+ for block, name in choices.items():
2661
+ out.update(candidates[block][name])
2662
+ return out
2663
+
2664
+ targets: dict = {}
2665
+ decisions: list = []
2666
+ done: set = set()
2667
+ for start in decided:
2668
+ component = components[start]
2669
+ if start in done:
2670
+ continue
2671
+ members = set(component)
2672
+ touched_here = [block for block in component if block in candidates]
2673
+ done.update(touched_here)
2674
+ milestone_ids = {
2675
+ event_id for block in component
2676
+ for event_id in block_ids.get(block, ())
2677
+ if event_id in lineage.milestones
2678
+ or any(payload_of(records.get(event_id)).get("kind")
2679
+ == "five_hour_milestone"
2680
+ for records in candidates.get(block, {}).values())
2681
+ }
2682
+ retired_actions = [dict(action) for action, block in retirement_blocks
2683
+ if block in members]
2684
+ todays_here = {
2685
+ event_id: record for event_id, record in todays_records.items()
2686
+ if blocks_of_id.get(event_id, set()) & members
2687
+ or _hashable_block(payload_of(record)) in members}
2688
+ milestone_ids |= {
2689
+ event_id for event_id, record in todays_here.items()
2690
+ if payload_of(record).get("kind") == "five_hour_milestone"}
2691
+ cache: dict = {}
2692
+
2693
+ def violations(records, *, retired_actions=retired_actions,
2694
+ milestone_ids=milestone_ids, cache=cache,
2695
+ todays_here=todays_here):
2696
+ records = {**todays_here, **records}
2697
+ key = _canonical_bytes([
2698
+ [event_id, _semantic_event(records[event_id])]
2699
+ for event_id in sorted(records)])
2700
+ if key in cache:
2701
+ return cache[key]
2702
+ actions, plain = [], []
2703
+ for event_id in sorted(records):
2704
+ record = records[event_id]
2705
+ existing = current.get(event_id)
2706
+ if existing is not None and (_semantic_event(existing)
2707
+ == _semantic_event(record)):
2708
+ continue
2709
+ if event_id in physical:
2710
+ actions.append({
2711
+ "action": "replace", "id": event_id,
2712
+ "rev": physical[event_id] + 1,
2713
+ "at": str(record.get("at")),
2714
+ "payload": dict(record.get("payload") or {}),
2715
+ })
2716
+ else:
2717
+ plain.append(record)
2718
+ actions.extend(retired_actions)
2719
+ if not actions and not plain:
2720
+ # Nothing would be appended: every decision stays as it is.
2721
+ cache[key] = frozenset()
2722
+ return cache[key]
2723
+ actions.sort(key=lambda action: action["id"])
2724
+ plain.sort(key=lambda record: order_index.get(
2725
+ record.get("id"), len(order_index)))
2726
+ after = lineage.extended(batch_actions=actions, plain=plain)
2727
+ result = _violation_keys(lineage, after, milestone_ids, records)
2728
+ cache[key] = result
2729
+ return result
2730
+
2731
+ def choose(block, carried):
2732
+ order = _COMPLETION_CANDIDATE_ORDER[classes[block]]
2733
+ own = candidates[block]
2734
+ found = {"class": violations({**carried, **own["class"]})}
2735
+ if not found["class"]:
2736
+ return "class", found
2737
+ for name in order[1:]:
2738
+ found[name] = violations({**carried, **own[name]})
2739
+ ceiling = found["A"]
2740
+ for name in order:
2741
+ if found[name] <= ceiling and not any(
2742
+ found[other] < found[name] for other in order):
2743
+ return name, found
2744
+ return "A", found # unreachable: A always qualifies
2745
+
2746
+ if len(touched_here) == 1:
2747
+ [block] = touched_here
2748
+ name, found = choose(block, {})
2749
+ choices = {block: name}
2750
+ found_by_block = {block: found}
2751
+ else:
2752
+ choices, found_by_block = {}, {}
2753
+ for block in touched_here:
2754
+ carried: dict = {}
2755
+ for other in touched_here:
2756
+ if other != block:
2757
+ carried.update(candidates[other]["A"])
2758
+ choices[block], found_by_block[block] = choose(block, carried)
2759
+ alone = landed_records({block: "A" for block in touched_here})
2760
+ if not violations(landed_records(choices)) <= violations(alone):
2761
+ choices = {block: "A" for block in touched_here}
2762
+ targets.update(landed_records(choices))
2763
+ for block in touched_here:
2764
+ decisions.append(BlockDecision(
2765
+ block=block, block_class=classes[block], component=component,
2766
+ landed=choices[block],
2767
+ violations=dict(found_by_block[block])))
2768
+ decisions.sort(key=lambda decision: repr(decision.block))
2769
+ if not set(targets) <= handled:
2770
+ # Both the planner and today's path would write these ids.
2771
+ raise RuntimeError(
2772
+ "#1040 planner targets outside the ids it owns: "
2773
+ f"{sorted(set(targets) - handled)}")
2774
+ return CompletionFiveHourPlan(
2775
+ targets={event_id: targets[event_id] for event_id in sorted(targets)},
2776
+ handled=frozenset(handled),
2777
+ decisions=tuple(decisions),
2778
+ )
2779
+
910
2780
  _SNAPSHOT_IDENTITY_FIELDS = (
911
2781
  "account_key", "week_start_date", "week_start_at", "week_end_at",
912
2782
  "weekly_percent", "five_hour_percent", "five_hour_window_key",
@@ -1876,6 +3746,19 @@ class RederivePlan:
1876
3746
  #: (`_retraction_scope`). Not part of the plan body: the retractions are
1877
3747
  #: already in the config fingerprint through the decision op's payload.
1878
3748
  retraction_event_ids: frozenset = frozenset()
3749
+ #: #956 spec 4.6.3: the windows whose opening fact the selection this plan
3750
+ #: was built over tombstones. Not part of the plan body: it is a property
3751
+ #: of the journal prefix the plan already names. A filtered plan
3752
+ #: (:func:`causal_delta_plan`) evaluates parent presence with it.
3753
+ retired_openings: frozenset = frozenset()
3754
+ #: #943/#956 spec 4.4: the current five-hour milestones whose window
3755
+ #: resolves no block before the plan (pre-existing damage). Reported,
3756
+ #: never a reason to refuse, and not part of the plan body.
3757
+ orphaned_milestone_ids: frozenset = frozenset()
3758
+ #: #884 spec §5.7: the five-hour blocks whose milestone repair the
3759
+ #: lineage withheld (:class:`WithheldBlock`). A diagnostic, outside the
3760
+ #: plan body; its count reaches ``uncertain_cost_fact_count``.
3761
+ withheld_milestone_repairs: tuple = ()
1879
3762
 
1880
3763
  def _body(self) -> dict:
1881
3764
  body = {
@@ -1940,6 +3823,7 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
1940
3823
  uncertain_cost_fact_ids=(),
1941
3824
  historical_close_markers=None,
1942
3825
  physical_revisions=None,
3826
+ milestone_lineage=None,
1943
3827
  enforce_guard=True) -> RederivePlan:
1944
3828
  """Diff current effective events against one scratch-derived desired set.
1945
3829
 
@@ -1972,7 +3856,15 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
1972
3856
  preserved history retained or revived, or corrected without moving its
1973
3857
  money) or replaced (added, or corrected in a way that writes its money)
1974
3858
  are decided by :func:`_resolve_closed_block_milestones` before the diff,
1975
- the same way closes are decided before milestones."""
3859
+ the same way closes are decided before milestones.
3860
+
3861
+ #884: ``milestone_lineage`` (:func:`build_five_hour_lineage`, built from
3862
+ the same selection's fold) lets an unchanged crossing of a frozen block
3863
+ regain the amount its history proves an earlier rederive repriced, and
3864
+ fails closed otherwise: each reported withheld block counts once in
3865
+ ``uncertain_cost_fact_count`` and is listed in
3866
+ ``withheld_milestone_repairs``. ``None`` reproduces #875's plan exactly;
3867
+ a missing lineage is a compatibility default, never evidence."""
1976
3868
  conflicted_event_ids = frozenset(conflicted_event_ids or ())
1977
3869
  historical_close_markers = dict(historical_close_markers or {})
1978
3870
  # #825/#832 fix round A (Codex fix-round-1 Q2): the highest revision of
@@ -2009,6 +3901,8 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
2009
3901
  current, desired, reviewed_weekly_hold_ids, historical_close_markers,
2010
3902
  unsupported_resets=unsupported_resets,
2011
3903
  )
3904
+ # #943 spec 4.4: an `fhcb` follows its `fhc` by natural key.
3905
+ _companions_follow_kept_credits(current, desired)
2012
3906
  retained_event_count = sum(
2013
3907
  1 for selected in selection.by_id.values()
2014
3908
  if selected.status == "active"
@@ -2091,9 +3985,11 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
2091
3985
  # and a replaced one is frozen. The causal block closure applies the same
2092
3986
  # rule (:func:`_replaced_block_closure`).
2093
3987
  replaced_close_blocks -= frozen_close_blocks
3988
+ withheld: dict = {}
2094
3989
  milestone_decisions = _resolve_closed_block_milestones(
2095
3990
  current, desired, preserved, frozen_close_blocks,
2096
3991
  replaced_close_blocks, selection=selection,
3992
+ lineage=milestone_lineage, withheld=withheld,
2097
3993
  )
2098
3994
 
2099
3995
  for event_id in sorted(set(current) | set(desired) | set(preserved)):
@@ -2186,6 +4082,19 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
2186
4082
  kind=kind,
2187
4083
  ))
2188
4084
 
4085
+ # #943/#956 spec 4.4: parent validation over the resulting effective
4086
+ # graph. Every five-hour milestone that resolves a block before the plan
4087
+ # must still resolve one after it. The diff above already tombstones a
4088
+ # current milestone the desired replay lacks, so a survivor without its
4089
+ # parent is one the replay itself keeps (or preserved history): refuse
4090
+ # rather than leave it parentless.
4091
+ retired_openings = retired_openings_of(selection)
4092
+ orphaned_before, lost = five_hour_parent_report(
4093
+ current, actions, retired_openings)
4094
+ if lost:
4095
+ raise RederiveConflict(
4096
+ "a five-hour block removal would orphan a retained milestone: "
4097
+ + ", ".join(lost[:10]))
2189
4098
  # #869 F7/Q9: a close counts only while its cost stays unprovable in the
2190
4099
  # state this plan leaves behind — not a provably priced close, not one this
2191
4100
  # plan corrects, and not one an earlier correction already proved.
@@ -2207,6 +4116,18 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
2207
4116
  set(uncertain_cost_fact_ids or ()) | uncertain_closes
2208
4117
  | drifted_close_ids
2209
4118
  )
4119
+ # #884 spec §5.7: a withheld block counts once, by its smallest close id,
4120
+ # and never beside a close of its own that already counts.
4121
+ withheld_blocks = _withheld_blocks(withheld, current, actions, selection)
4122
+ for block in withheld_blocks:
4123
+ if any(close_id in uncertain_ids for close_id in block.close_event_ids):
4124
+ continue
4125
+ # The synthetic key is defensive only: a frozen block always has a
4126
+ # close in the plan's final state, so it is unreachable.
4127
+ uncertain_ids.add(
4128
+ min(block.close_event_ids) if block.close_event_ids
4129
+ else ("withheld-block", block.account_key,
4130
+ block.five_hour_window_key))
2210
4131
  plan = RederivePlan(
2211
4132
  family=FAMILY,
2212
4133
  journal_high_water=journal_high_water,
@@ -2219,6 +4140,9 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
2219
4140
  uncertain_cost_fact_count=len(uncertain_ids),
2220
4141
  action_counts_by_event_kind=counts_by_kind,
2221
4142
  retraction_event_ids=retraction_event_ids,
4143
+ retired_openings=retired_openings,
4144
+ orphaned_milestone_ids=orphaned_before,
4145
+ withheld_milestone_repairs=withheld_blocks,
2222
4146
  )
2223
4147
  if enforce_guard:
2224
4148
  enforce_week_reset_add_burst_guard(plan)
@@ -2502,6 +4426,89 @@ def _retained_reference_closure(actions, reviewed_by_id, current_events):
2502
4426
  return actions
2503
4427
 
2504
4428
 
4429
+ def _five_hour_parent_closure(included: list, reviewed_actions,
4430
+ current_events: Mapping[str, Mapping],
4431
+ retired_openings=frozenset()) -> list:
4432
+ """#943/#956 spec 4.4: the reviewed actions an included action needs.
4433
+
4434
+ * Derived-FK parent closure: an included five-hour milestone add or
4435
+ supersede resolves its `block_id` from its window at fold, so when the
4436
+ graph the included actions leave behind materializes no block for
4437
+ that window, it keeps the reviewed plan's parent-materializing action
4438
+ for the window (an opening or close add or supersede), even when that
4439
+ action is common to the baseline. A window whose block the resulting
4440
+ graph already materializes (a current close or opening the plan keeps)
4441
+ needs no action: pulling a baseline-common close correction in there
4442
+ would rewrite a close the #875 frozen-block rule keeps as it stands.
4443
+ * A credit and its boundary companion travel together: an included
4444
+ action on either brings the reviewed action on the other.
4445
+ * A seed never outlives a removed block: an included close tombstone
4446
+ brings the reviewed opening tombstone of its window (the reviewed plan
4447
+ tombstones the opening only when its replay has no block there).
4448
+
4449
+ Only reviewed actions are ever added, each once."""
4450
+ parents: dict = {}
4451
+ opening_tombstones: dict = {}
4452
+ pairs: dict = {}
4453
+ for action in reviewed_actions:
4454
+ payload = _action_payload(action, current_events)
4455
+ if action.kind in _BLOCK_PARENT_KINDS:
4456
+ window = _window_of(payload)
4457
+ if action.disposition in {"add", "supersede"}:
4458
+ parents.setdefault(window, []).append(action)
4459
+ elif action.kind == "five_hour_block_open":
4460
+ opening_tombstones.setdefault(window, []).append(action)
4461
+ elif action.kind in _CREDIT_PAIR_KINDS:
4462
+ pairs.setdefault(_credit_pair_key(payload), []).append(action)
4463
+ expanded = list(included)
4464
+ included_ids = {action.event_id for action in included}
4465
+ present = None
4466
+ for action in included:
4467
+ payload = _action_payload(action, current_events)
4468
+ needed = []
4469
+ if (action.kind == "five_hour_milestone"
4470
+ and action.disposition != "tombstone"):
4471
+ if present is None:
4472
+ after_events, after_retired = _post_plan_state(
4473
+ current_events, included, retired_openings)
4474
+ present = five_hour_parent_presence(
4475
+ after_events, after_retired)
4476
+ if _window_of(payload) not in present:
4477
+ needed = parents.get(_window_of(payload), [])
4478
+ elif action.kind in _CREDIT_PAIR_KINDS:
4479
+ needed = pairs.get(_credit_pair_key(payload), [])
4480
+ elif (action.kind == "five_hour_block_close"
4481
+ and action.disposition == "tombstone"):
4482
+ needed = opening_tombstones.get(_window_of(payload), [])
4483
+ for dependency in needed:
4484
+ if dependency.event_id not in included_ids:
4485
+ expanded.append(dependency)
4486
+ included_ids.add(dependency.event_id)
4487
+ return expanded
4488
+
4489
+
4490
+ def _parent_validation_completion(included: list, reviewed_by_id,
4491
+ current_events: Mapping[str, Mapping],
4492
+ retired_openings) -> list:
4493
+ """#943/#956 spec 4.4: a filtered plan whose block removal would orphan
4494
+ an unchanged, retained milestone takes that milestone's reviewed
4495
+ tombstone (the desired replay lacks it). A survivor with no reviewed
4496
+ tombstone is left for the final validation, which refuses."""
4497
+ _orphaned_before, lost = five_hour_parent_report(
4498
+ current_events, included, retired_openings)
4499
+ if not lost:
4500
+ return included
4501
+ expanded = list(included)
4502
+ included_ids = {action.event_id for action in included}
4503
+ for event_id in lost:
4504
+ reviewed = reviewed_by_id.get(event_id)
4505
+ if (reviewed is not None and reviewed.disposition == "tombstone"
4506
+ and event_id not in included_ids):
4507
+ expanded.append(reviewed)
4508
+ included_ids.add(event_id)
4509
+ return expanded
4510
+
4511
+
2505
4512
  def causal_delta_plan(
2506
4513
  baseline: RederivePlan, reviewed: RederivePlan, *,
2507
4514
  current_events: Mapping[str, Mapping],
@@ -2568,8 +4575,23 @@ def causal_delta_plan(
2568
4575
  if retraction_scope:
2569
4576
  actions = _retained_reference_closure(
2570
4577
  actions, reviewed_by_id, current_events)
4578
+ # #943/#956 spec 4.4: over the final filtered set, so a parent or a
4579
+ # companion the B/R filter dropped as baseline-common returns.
4580
+ actions = _five_hour_parent_closure(
4581
+ actions, reviewed.actions, current_events,
4582
+ reviewed.retired_openings)
4583
+ actions = _parent_validation_completion(
4584
+ actions, reviewed_by_id, current_events,
4585
+ reviewed.retired_openings)
2571
4586
  if {action.event_id for action in actions} == previous_ids:
2572
4587
  break
4588
+ # Parent validation over the resulting effective graph (spec 4.4).
4589
+ orphaned_before, lost = five_hour_parent_report(
4590
+ current_events, actions, reviewed.retired_openings)
4591
+ if lost:
4592
+ raise RederiveConflict(
4593
+ "a five-hour block removal would orphan a retained milestone: "
4594
+ + ", ".join(lost[:10]))
2573
4595
  included_ids = {action.event_id for action in actions}
2574
4596
  actions = tuple(
2575
4597
  action for action in reviewed.actions
@@ -2599,6 +4621,9 @@ def causal_delta_plan(
2599
4621
  uncertain_cost_fact_count=reviewed.uncertain_cost_fact_count,
2600
4622
  action_counts_by_event_kind=counts_by_kind,
2601
4623
  retraction_event_ids=reviewed.retraction_event_ids,
4624
+ retired_openings=reviewed.retired_openings,
4625
+ orphaned_milestone_ids=orphaned_before,
4626
+ withheld_milestone_repairs=reviewed.withheld_milestone_repairs,
2602
4627
  )
2603
4628
 
2604
4629