@optimystic/db-p2p 1.0.0-beta.3 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/dist/src/cluster/block-transfer-service.d.ts +73 -1
  2. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  3. package/dist/src/cluster/block-transfer-service.js +70 -3
  4. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  5. package/dist/src/cluster/block-transfer.d.ts +16 -1
  6. package/dist/src/cluster/block-transfer.d.ts.map +1 -1
  7. package/dist/src/cluster/block-transfer.js +73 -72
  8. package/dist/src/cluster/block-transfer.js.map +1 -1
  9. package/dist/src/cluster/client.d.ts +4 -6
  10. package/dist/src/cluster/client.d.ts.map +1 -1
  11. package/dist/src/cluster/client.js +8 -13
  12. package/dist/src/cluster/client.js.map +1 -1
  13. package/dist/src/cluster/cluster-policy.d.ts +2 -1
  14. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  15. package/dist/src/cluster/cluster-policy.js.map +1 -1
  16. package/dist/src/cluster/cluster-repo.d.ts +155 -9
  17. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  18. package/dist/src/cluster/cluster-repo.js +446 -58
  19. package/dist/src/cluster/cluster-repo.js.map +1 -1
  20. package/dist/src/cluster/commit-proof.js +2 -2
  21. package/dist/src/cluster/race-resolution.d.ts +51 -7
  22. package/dist/src/cluster/race-resolution.d.ts.map +1 -1
  23. package/dist/src/cluster/race-resolution.js +71 -7
  24. package/dist/src/cluster/race-resolution.js.map +1 -1
  25. package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
  26. package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
  27. package/dist/src/cluster/rebalance-monitor.js +120 -26
  28. package/dist/src/cluster/rebalance-monitor.js.map +1 -1
  29. package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
  30. package/dist/src/cluster/spread-on-churn.js +34 -47
  31. package/dist/src/cluster/spread-on-churn.js.map +1 -1
  32. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  33. package/dist/src/cohort-topic/host.js +8 -6
  34. package/dist/src/cohort-topic/host.js.map +1 -1
  35. package/dist/src/cohort-topic/membership-source.js +1 -1
  36. package/dist/src/cohort-topic/membership-source.js.map +1 -1
  37. package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
  38. package/dist/src/cohort-topic/peer-codec.js +5 -2
  39. package/dist/src/cohort-topic/peer-codec.js.map +1 -1
  40. package/dist/src/cohort-topic/stream-util.d.ts +24 -3
  41. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  42. package/dist/src/cohort-topic/stream-util.js +36 -3
  43. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  44. package/dist/src/cohort-topic/topic-router.d.ts +9 -0
  45. package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
  46. package/dist/src/cohort-topic/topic-router.js +12 -2
  47. package/dist/src/cohort-topic/topic-router.js.map +1 -1
  48. package/dist/src/dispute/client.d.ts.map +1 -1
  49. package/dist/src/dispute/client.js +15 -3
  50. package/dist/src/dispute/client.js.map +1 -1
  51. package/dist/src/index.d.ts +5 -0
  52. package/dist/src/index.d.ts.map +1 -1
  53. package/dist/src/index.js +5 -0
  54. package/dist/src/index.js.map +1 -1
  55. package/dist/src/libp2p-key-network.d.ts +78 -17
  56. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  57. package/dist/src/libp2p-key-network.js +234 -147
  58. package/dist/src/libp2p-key-network.js.map +1 -1
  59. package/dist/src/libp2p-node-base.d.ts +35 -1
  60. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  61. package/dist/src/libp2p-node-base.js +151 -26
  62. package/dist/src/libp2p-node-base.js.map +1 -1
  63. package/dist/src/libp2p-node-rn.d.ts +2 -2
  64. package/dist/src/libp2p-node-rn.d.ts.map +1 -1
  65. package/dist/src/libp2p-node-rn.js.map +1 -1
  66. package/dist/src/libp2p-node.d.ts +2 -2
  67. package/dist/src/libp2p-node.d.ts.map +1 -1
  68. package/dist/src/libp2p-node.js.map +1 -1
  69. package/dist/src/logger.d.ts.map +1 -1
  70. package/dist/src/logger.js +6 -1
  71. package/dist/src/logger.js.map +1 -1
  72. package/dist/src/matchmaking/query-transport.d.ts +3 -4
  73. package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
  74. package/dist/src/matchmaking/query-transport.js +9 -9
  75. package/dist/src/matchmaking/query-transport.js.map +1 -1
  76. package/dist/src/network/network-manager-service.d.ts +10 -5
  77. package/dist/src/network/network-manager-service.d.ts.map +1 -1
  78. package/dist/src/network/network-manager-service.js +11 -2
  79. package/dist/src/network/network-manager-service.js.map +1 -1
  80. package/dist/src/network/relay-reservation.d.ts +170 -0
  81. package/dist/src/network/relay-reservation.d.ts.map +1 -0
  82. package/dist/src/network/relay-reservation.js +527 -0
  83. package/dist/src/network/relay-reservation.js.map +1 -0
  84. package/dist/src/pend-validation.d.ts +3 -2
  85. package/dist/src/pend-validation.d.ts.map +1 -1
  86. package/dist/src/pend-validation.js +3 -2
  87. package/dist/src/pend-validation.js.map +1 -1
  88. package/dist/src/protocol-client.d.ts +8 -1
  89. package/dist/src/protocol-client.d.ts.map +1 -1
  90. package/dist/src/protocol-client.js +11 -0
  91. package/dist/src/protocol-client.js.map +1 -1
  92. package/dist/src/reactivity/forwarder-host.d.ts +2 -1
  93. package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
  94. package/dist/src/reactivity/forwarder-host.js +2 -1
  95. package/dist/src/reactivity/forwarder-host.js.map +1 -1
  96. package/dist/src/reactivity/origination-manager.d.ts +3 -3
  97. package/dist/src/reactivity/origination-manager.js +3 -3
  98. package/dist/src/reactivity/recover-transport.d.ts +22 -14
  99. package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
  100. package/dist/src/reactivity/recover-transport.js +32 -19
  101. package/dist/src/reactivity/recover-transport.js.map +1 -1
  102. package/dist/src/reactivity/subscription-manager.d.ts +2 -2
  103. package/dist/src/reactivity/topic-bytes.d.ts +6 -7
  104. package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
  105. package/dist/src/reactivity/topic-bytes.js +8 -9
  106. package/dist/src/reactivity/topic-bytes.js.map +1 -1
  107. package/dist/src/repo/client.d.ts +5 -2
  108. package/dist/src/repo/client.d.ts.map +1 -1
  109. package/dist/src/repo/client.js +49 -40
  110. package/dist/src/repo/client.js.map +1 -1
  111. package/dist/src/repo/cluster-coordinator.d.ts +96 -11
  112. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  113. package/dist/src/repo/cluster-coordinator.js +164 -25
  114. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  115. package/dist/src/repo/coordinator-repo.d.ts +240 -122
  116. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  117. package/dist/src/repo/coordinator-repo.js +627 -473
  118. package/dist/src/repo/coordinator-repo.js.map +1 -1
  119. package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
  120. package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
  121. package/dist/src/repo/i-under-replication-ledger.js +2 -0
  122. package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
  123. package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
  124. package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
  125. package/dist/src/repo/kv-under-replication-ledger.js +287 -0
  126. package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
  127. package/dist/src/repo/responsibility.d.ts +37 -0
  128. package/dist/src/repo/responsibility.d.ts.map +1 -0
  129. package/dist/src/repo/responsibility.js +37 -0
  130. package/dist/src/repo/responsibility.js.map +1 -0
  131. package/dist/src/repo/service.d.ts +44 -15
  132. package/dist/src/repo/service.d.ts.map +1 -1
  133. package/dist/src/repo/service.js +64 -27
  134. package/dist/src/repo/service.js.map +1 -1
  135. package/dist/src/repo/stuck-reservation.d.ts +127 -0
  136. package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
  137. package/dist/src/repo/stuck-reservation.js +179 -0
  138. package/dist/src/repo/stuck-reservation.js.map +1 -0
  139. package/dist/src/repo/under-replication-drain.d.ts +208 -0
  140. package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
  141. package/dist/src/repo/under-replication-drain.js +461 -0
  142. package/dist/src/repo/under-replication-drain.js.map +1 -0
  143. package/dist/src/rn.d.ts +5 -0
  144. package/dist/src/rn.d.ts.map +1 -1
  145. package/dist/src/rn.js +5 -0
  146. package/dist/src/rn.js.map +1 -1
  147. package/dist/src/storage/block-latch.d.ts +3 -2
  148. package/dist/src/storage/block-latch.d.ts.map +1 -1
  149. package/dist/src/storage/block-latch.js +7 -4
  150. package/dist/src/storage/block-latch.js.map +1 -1
  151. package/dist/src/storage/block-storage.d.ts +45 -3
  152. package/dist/src/storage/block-storage.d.ts.map +1 -1
  153. package/dist/src/storage/block-storage.js +156 -9
  154. package/dist/src/storage/block-storage.js.map +1 -1
  155. package/dist/src/storage/i-block-storage.d.ts +57 -5
  156. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  157. package/dist/src/storage/i-block-storage.js.map +1 -1
  158. package/dist/src/storage/pending-claim.d.ts +167 -0
  159. package/dist/src/storage/pending-claim.d.ts.map +1 -0
  160. package/dist/src/storage/pending-claim.js +156 -0
  161. package/dist/src/storage/pending-claim.js.map +1 -0
  162. package/dist/src/storage/raw-store-codec.d.ts +13 -0
  163. package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
  164. package/dist/src/storage/raw-store-codec.js +24 -3
  165. package/dist/src/storage/raw-store-codec.js.map +1 -1
  166. package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
  167. package/dist/src/storage/restoration-coordinator.js +2 -1
  168. package/dist/src/storage/restoration-coordinator.js.map +1 -1
  169. package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
  170. package/dist/src/storage/ring-shift-coordinator.js +3 -3
  171. package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
  172. package/dist/src/storage/storage-repo.d.ts +122 -5
  173. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  174. package/dist/src/storage/storage-repo.js +245 -68
  175. package/dist/src/storage/storage-repo.js.map +1 -1
  176. package/dist/src/storage/struct.d.ts +58 -0
  177. package/dist/src/storage/struct.d.ts.map +1 -1
  178. package/dist/src/testing/mesh-harness.d.ts +77 -11
  179. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  180. package/dist/src/testing/mesh-harness.js +226 -100
  181. package/dist/src/testing/mesh-harness.js.map +1 -1
  182. package/package.json +2 -2
  183. package/readme.md +27 -3
  184. package/src/cluster/block-transfer-service.ts +132 -4
  185. package/src/cluster/block-transfer.ts +79 -84
  186. package/src/cluster/client.ts +115 -119
  187. package/src/cluster/cluster-policy.ts +2 -1
  188. package/src/cluster/cluster-repo.ts +493 -65
  189. package/src/cluster/commit-proof.ts +2 -2
  190. package/src/cluster/race-resolution.ts +76 -7
  191. package/src/cluster/rebalance-monitor.ts +677 -539
  192. package/src/cluster/spread-on-churn.ts +34 -54
  193. package/src/cohort-topic/host.ts +8 -6
  194. package/src/cohort-topic/membership-source.ts +1 -1
  195. package/src/cohort-topic/peer-codec.ts +34 -31
  196. package/src/cohort-topic/stream-util.ts +39 -4
  197. package/src/cohort-topic/topic-router.ts +12 -2
  198. package/src/dispute/client.ts +17 -7
  199. package/src/index.ts +64 -59
  200. package/src/libp2p-key-network.ts +1377 -1260
  201. package/src/libp2p-node-base.ts +186 -27
  202. package/src/libp2p-node-rn.ts +31 -30
  203. package/src/libp2p-node.ts +37 -36
  204. package/src/logger.ts +6 -1
  205. package/src/matchmaking/query-transport.ts +9 -9
  206. package/src/network/network-manager-service.ts +16 -5
  207. package/src/network/relay-reservation.ts +636 -0
  208. package/src/pend-validation.ts +3 -2
  209. package/src/protocol-client.ts +13 -1
  210. package/src/reactivity/forwarder-host.ts +2 -1
  211. package/src/reactivity/origination-manager.ts +3 -3
  212. package/src/reactivity/recover-transport.ts +39 -22
  213. package/src/reactivity/subscription-manager.ts +2 -2
  214. package/src/reactivity/topic-bytes.ts +35 -37
  215. package/src/repo/client.ts +172 -173
  216. package/src/repo/cluster-coordinator.ts +183 -25
  217. package/src/repo/coordinator-repo.ts +656 -506
  218. package/src/repo/i-under-replication-ledger.ts +89 -0
  219. package/src/repo/kv-under-replication-ledger.ts +302 -0
  220. package/src/repo/responsibility.ts +48 -0
  221. package/src/repo/service.ts +76 -39
  222. package/src/repo/stuck-reservation.ts +224 -0
  223. package/src/repo/under-replication-drain.ts +582 -0
  224. package/src/rn.ts +74 -69
  225. package/src/storage/block-latch.ts +7 -4
  226. package/src/storage/block-storage.ts +161 -10
  227. package/src/storage/i-block-storage.ts +60 -5
  228. package/src/storage/pending-claim.ts +200 -0
  229. package/src/storage/raw-store-codec.ts +26 -3
  230. package/src/storage/restoration-coordinator.ts +212 -211
  231. package/src/storage/ring-shift-coordinator.ts +271 -271
  232. package/src/storage/storage-repo.ts +280 -72
  233. package/src/storage/struct.ts +58 -0
  234. package/src/testing/mesh-harness.ts +293 -112
@@ -1,7 +1,7 @@
1
- import type { IRepo, ClusterRecord, ClusterPeers, Signature, RepoMessage, ITransactionValidator, ClusterConsensusConfig, UnvalidatablePendPolicy, CommitResult, PendResult, BlockId, ActionId, ActionRev, CommitRequest, CommitCert, InvalidateRequest, MemberApplyOutcome } from "@optimystic/db-core";
1
+ import type { IRepo, ClusterRecord, ClusterPeers, Signature, RepoMessage, ITransactionValidator, ClusterConsensusConfig, UnvalidatablePendPolicy, CommitResult, PendResult, PendRequest, BlockId, ActionId, ActionRev, CommitRequest, CommitCert, InvalidateRequest, MemberApplyOutcome } from "@optimystic/db-core";
2
2
  import type { ICluster } from "@optimystic/db-core";
3
3
  import type { IPeerNetwork } from "@optimystic/db-core";
4
- import { blockIdsForTransforms, isOwnRevision, isConflictFailure, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
4
+ import { blockIdsForTransforms, transformForBlockId, isOwnRevision, isConflictFailure, DEFAULT_SUPER_MAJORITY_THRESHOLD, localDurability } from "@optimystic/db-core";
5
5
  import { computeClusterCommitHash, computeClusterMessageHash, computeClusterPromiseHash, membershipDigest, recordMembershipDigest, clusterVoteSigningPayload, clusterVoteVerificationPayload } from "@optimystic/db-core";
6
6
  import { verifyInvalidationCertificate, type ArbitratorSetRecompute } from "../dispute/invalidation.js";
7
7
  import { buildCommitCert, invalidationActionId } from "./commit-cert.js";
@@ -17,12 +17,15 @@ import type { FretService } from "p2p-fret";
17
17
  import type { IPeerReputation } from "../reputation/types.js";
18
18
  import { PenaltyReason } from "../reputation/types.js";
19
19
  import type { ITransactionStateStore } from "./i-transaction-state-store.js";
20
- import { isMissingBaseRevisionFailure, type CommitDigestPreview, type ICommitDigestPreviewer, type ICommitProofPersister, type IRevisionActionReader } from "../storage/storage-repo.js";
20
+ import { isMissingBaseRevisionFailure, type CommitDigestPreview, type ICommitDigestPreviewer, type ICommitProofPersister, type IRevisionActionReader, type IPendingClaimReader } from "../storage/storage-repo.js";
21
+ import { isReservationAgainst, reservationRequestFor, cohortCanMissAPend, type PendingClaim, type ReservationRequest } from "../storage/pending-claim.js";
22
+ import { StuckReservationTracker } from "../repo/stuck-reservation.js";
21
23
  import { checkPendValidation } from "../pend-validation.js";
22
24
  import { getAffectedBlockIds } from "./record-operations.js";
23
25
  import { operationsConflict, resolveRace } from "./race-resolution.js";
24
26
  import { buildBlockCommitProof, mintSoloCommitProof, type BlockCommitProof } from "./commit-proof.js";
25
27
  import { RECONCILE_TIMEOUT_MS } from "./reconcile-block.js";
28
+ import type { CommittedHolders } from "./rebalance-monitor.js";
26
29
 
27
30
  const log = createLogger('cluster-member')
28
31
 
@@ -68,6 +71,45 @@ type VerifyOutcome =
68
71
  | { valid: true }
69
72
  | { valid: false; penalize: boolean };
70
73
 
74
+ /**
75
+ * This member's promise-round decision on a record: approve, or one of two *different kinds* of no.
76
+ *
77
+ * The distinction is the whole reason the type exists. `invalid` is a validity judgement — the record
78
+ * is wrong and will be wrong on every retry — and becomes a `reject` vote the coordinator counts
79
+ * toward the permanent-rejection threshold. `held` is "not right now": the pend's blocks are reserved
80
+ * by a different unresolved action in this member's storage, a condition that disappears the moment
81
+ * that rival commits or cancels, and it becomes a `held` vote counted toward neither approvals nor
82
+ * rejections.
83
+ *
84
+ * A boolean plus prose cannot carry that difference, and collapsing the two was a real defect rather
85
+ * than a tidiness question: on a cohort of three or fewer members `maxAllowedRejections` is zero at
86
+ * the default threshold, so one member saying "someone else is holding this right now" became a
87
+ * permanent `ValidatorRejectionError` for the whole transaction.
88
+ *
89
+ * `reason` stays optional on the `invalid` arm because it is signed: it is folded into the vote
90
+ * payload verbatim, and an absent reason and an empty one must keep producing the bytes they do today.
91
+ */
92
+ type PromiseVerdict =
93
+ | { valid: true }
94
+ | { valid: false; kind: 'invalid'; reason?: string }
95
+ /** `heldBy` is the rival's ACTION id (storage's pending list), never a messageHash — see {@link Signature}. */
96
+ | { valid: false; kind: 'held'; reason: string; heldBy: ActionId };
97
+
98
+ /** A refusal that judges the record invalid — the permanent kind, and every kind but the pending-conflict one. */
99
+ function invalidVerdict(reason?: string): PromiseVerdict {
100
+ return { valid: false, kind: 'invalid', reason };
101
+ }
102
+
103
+ /**
104
+ * Widen a boolean-plus-prose check into a {@link PromiseVerdict}. Every check written this way judges
105
+ * validity, so its refusal is the permanent kind; only the pending-conflict branch of
106
+ * {@link ClusterMember.validatePendOperations} produces the transient one, and it builds its verdict
107
+ * directly.
108
+ */
109
+ function verdictOf(result: { valid: boolean; reason?: string }): PromiseVerdict {
110
+ return result.valid ? { valid: true } : invalidVerdict(result.reason);
111
+ }
112
+
71
113
  /**
72
114
  * Actively reconciles a block this member committed without having seen the matching
73
115
  * pend (cohort drift between the independent pend and commit cluster-transactions).
@@ -91,6 +133,15 @@ export type ReconcileBlockCallback = (blockId: BlockId, committed: ActionRev, co
91
133
  */
92
134
  export type CommitCertificateSink = (actionId: ActionId, cert: CommitCert) => void;
93
135
 
136
+ /**
137
+ * Sink for who is known to hold a commit this node's own storage durably holds — fed to the rebalance
138
+ * monitor so it does not push freshly committed blocks back to the members that stored them (see
139
+ * `RebalanceMonitor.recordCommittedHolders`). Fired by the member after applying a consensus commit,
140
+ * and by `CoordinatorRepo.commit` when it acknowledges one. Optional; a throwing sink is isolated +
141
+ * logged (it must never break consensus or the writer's answer).
142
+ */
143
+ export type CommittedHoldersSink = (committed: CommittedHolders) => void;
144
+
94
145
  /**
95
146
  * Applies a consensus-ordered {@link InvalidateRequest} to local storage — the deterministic
96
147
  * reversal every member runs once consensus on the invalidation is reached. The implementation
@@ -179,6 +230,14 @@ export const MEMBERSHIP_NOT_ADMITTED = 'membership-not-admitted';
179
230
  */
180
231
  export const CONTENT_DIGEST_MISMATCH = 'content-digest-mismatch';
181
232
 
233
+ /**
234
+ * Stable reject reason a member emits when a commit declares, for a block, a base revision
235
+ * (`CommitRequest.blockDigests[id].baseRev`) other than the one the same action's pend carried to
236
+ * this member (`PendRequest.baseRevs`, kept with the pending record) — see
237
+ * `ClusterMember.validateCommitBaseDeclarations`. Signed like its sibling above.
238
+ */
239
+ export const BASE_DECLARATION_DISAGREES = 'base-declaration-disagrees';
240
+
182
241
  /**
183
242
  * The two stable reject reasons a validator-configured member emits from the shared
184
243
  * {@link checkPendValidation}: `PEND_NOT_VALIDATABLE` for a pend carrying no `validation` payload
@@ -205,6 +264,8 @@ interface ClusterMemberComponents {
205
264
  reconcileBlock?: ReconcileBlockCallback;
206
265
  /** Receives the consensus commit cert per committed action; see {@link CommitCertificateSink}. */
207
266
  onCommitCertificate?: CommitCertificateSink;
267
+ /** Receives who holds each consensus commit this member durably applied; see {@link CommittedHoldersSink}. */
268
+ onCommittedHolders?: CommittedHoldersSink;
208
269
  /** Applies a consensus-ordered invalidation to local storage; see {@link InvalidationApplySink}. */
209
270
  onInvalidate?: InvalidationApplySink;
210
271
  /** Layer-2 arbitrator-set recompute for invalidation verification; see {@link RecomputeArbitratorSetCapability}. */
@@ -244,7 +305,8 @@ export function clusterMember(components: ClusterMemberComponents): ClusterMembe
244
305
  components.onInvalidate,
245
306
  components.recomputeArbitratorSet,
246
307
  components.deriveExpectedCluster,
247
- components.now
308
+ components.now,
309
+ components.onCommittedHolders
248
310
  );
249
311
  }
250
312
 
@@ -301,6 +363,12 @@ export class ClusterMember implements ICluster {
301
363
  // applyConsensusOperation swallowed a refusal whose real cause was a RIVAL action holding the
302
364
  // requested revision. Pruned alongside executedTransactions (same TTL).
303
365
  private executedCommitResults: Map<string, CommitResult> = new Map();
366
+ // The messageHashes whose retained commit refusal above has the BEHIND shape (missing pend or
367
+ // missing base — this member held no usable revision), as opposed to the ahead shape, which must
368
+ // never be reconciled downward. Recorded from the apply branch that produced the refusal rather
369
+ // than read back from its prose. Only these are eligible for reconcileRefusedCommit. Same TTL,
370
+ // pruning and rollback as executedCommitResults.
371
+ private behindCommitRefusals: Set<string> = new Set();
304
372
  // Conflict-shaped pend refusals this member's storage produced at consensus-apply, keyed by the
305
373
  // refused action's id rather than by messageHash. The messageHash-keyed map above cannot serve the
306
374
  // commit-promise guard: a commit is a DIFFERENT message with a different hash, so a member holding
@@ -314,6 +382,14 @@ export class ClusterMember implements ICluster {
314
382
  // inside the sink); this map only spares redundant work when the same invalidation reaches
315
383
  // consensus twice (rebroadcast / sync) under different message hashes. (-> appliedAt timestamp)
316
384
  private appliedInvalidations: Map<string, number> = new Map();
385
+ /**
386
+ * Per block, what this member's own `held` votes have added up to — the member-side instance of
387
+ * the counter `CoordinatorRepo.noteStuckReservation` keeps from the coordinator's vantage. Fed by
388
+ * every `held` verdict {@link validatePendOperations} casts, so a reservation only THIS member
389
+ * holds (one the coordinator's storage cannot corroborate) is still named, once, where the record
390
+ * lives. Forgotten for a block the moment a vote finds it no longer reserved.
391
+ */
392
+ private readonly stuckReservations = new StuckReservationTracker(1000);
317
393
  // Queue of transactions to clean up
318
394
  private cleanupQueue: string[] = [];
319
395
  // Serialize concurrent updates for the same transaction
@@ -368,7 +444,8 @@ export class ClusterMember implements ICluster {
368
444
  private readonly onInvalidate?: InvalidationApplySink,
369
445
  private readonly recomputeArbitratorSet?: RecomputeArbitratorSetCapability,
370
446
  private readonly deriveExpectedCluster?: DeriveExpectedClusterCallback,
371
- now?: () => number
447
+ now?: () => number,
448
+ private readonly onCommittedHolders?: CommittedHoldersSink
372
449
  ) {
373
450
  this.now = now ?? ((): number => Date.now());
374
451
  this.superMajorityThreshold = consensusConfig?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD;
@@ -420,6 +497,7 @@ export class ClusterMember implements ICluster {
420
497
  this.cleanupQueue.length = 0;
421
498
  this.executedPendResults.clear();
422
499
  this.executedCommitResults.clear();
500
+ this.behindCommitRefusals.clear();
423
501
  this.refusedPendActions.clear();
424
502
  }
425
503
 
@@ -588,6 +666,15 @@ export class ClusterMember implements ICluster {
588
666
  });
589
667
  // Our own vote can be terminal (a reject where maxAllowedRejections is 0) or complete
590
668
  // the super-majority — recompute rather than guess which.
669
+ //
670
+ // Deliberately NOT the conflict vote's `shouldPersist = false`, for a `held` vote or
671
+ // any other. A conflict vote clears the record because this member holds the WINNER
672
+ // in `activeTransactions` and persisting the loser would reserve the same blocks a
673
+ // second time. A `held` vote has no such twin: the rival lives in durable storage,
674
+ // not in that table, so this record's entry is the only one and dropping it would
675
+ // only make this member forget a transaction the rest of the cohort may still
676
+ // carry to super-majority. Where one `held` vote IS terminal (a small cohort), the
677
+ // recomputed phase is `ConflictSuperseded`, which clears the record anyway.
591
678
  continue;
592
679
  case TransactionPhase.OurConflictVoteNeeded:
593
680
  currentRecord = await this.handleConflictVoteNeeded(currentRecord, phaseResult.conflictsWith);
@@ -640,7 +727,7 @@ export class ClusterMember implements ICluster {
640
727
  shouldPersist = false;
641
728
  break phaseLoop;
642
729
  case TransactionPhase.Promising:
643
- // We have already voted (approve, reject, or conflict); the record is still
730
+ // We have already voted (approve, reject, conflict, or held); the record is still
644
731
  // collecting promises from the rest of the cohort. Nothing to add — retain the
645
732
  // record only if our vote wasn't a conflict (`shouldPersist` already reflects that).
646
733
  log('cluster-member:phase-promising-waiting', {
@@ -788,8 +875,9 @@ export class ClusterMember implements ICluster {
788
875
 
789
876
  /**
790
877
  * Compares existing vs incoming signatures for the same peers.
791
- * If a peer's vote type changed (approve↔reject), that's equivocation:
792
- * report a penalty and keep the first-seen signature.
878
+ * ANY change of vote type is equivocation — the comparison is over `Signature['type']`, so it
879
+ * covers every kind without enumerating them (a new kind joins the check by existing): report a
880
+ * penalty and keep the first-seen signature.
793
881
  * New peers are accepted normally.
794
882
  */
795
883
  private detectEquivocation(
@@ -991,19 +1079,21 @@ export class ClusterMember implements ICluster {
991
1079
  const maxAllowedRejections = peerCount - superMajority;
992
1080
 
993
1081
  // Check for rejections — rejected if too many rejections to ever reach super-majority.
994
- // ONLY `reject` votes count here: a `conflict` vote is "not now", never a validity
995
- // judgement, so it must not push a record into the permanent `Rejected` phase.
1082
+ // ONLY `reject` votes count here: `conflict` and `held` both mean "not now", never a validity
1083
+ // judgement, so neither may push a record into the permanent `Rejected` phase.
996
1084
  const rejectedPromises = Object.values(record.promises).filter(s => s.type === 'reject');
997
- const conflictPromises = Object.values(record.promises).filter(s => s.type === 'conflict');
1085
+ const retryableRefusals = Object.values(record.promises).filter(s => s.type === 'conflict' || s.type === 'held');
998
1086
  const rejectedCommits = Object.values(record.commits).filter(s => s.type === 'reject');
999
1087
  if (rejectedPromises.length > maxAllowedRejections || this.hasMajority(rejectedCommits.length, peerCount)) {
1000
1088
  return { phase: TransactionPhase.Rejected };
1001
1089
  }
1002
1090
 
1003
- // Conflict votes don't judge validity, but enough of them still make super-majority
1091
+ // Retryable refusals don't judge validity, but enough of them still make super-majority
1004
1092
  // unreachable — a distinct terminal outcome (retryable as a fresh transaction) so logs and
1005
- // reputation-adjacent paths keep meaning what they say.
1006
- if (conflictPromises.length > 0 && rejectedPromises.length + conflictPromises.length > maxAllowedRejections) {
1093
+ // reputation-adjacent paths keep meaning what they say. `held` joins `conflict` here for the
1094
+ // same reason `conflict` clears its record: a member that keeps reserving the blocks of a
1095
+ // transaction that provably cannot win holds them against the very retry meant to win.
1096
+ if (retryableRefusals.length > 0 && rejectedPromises.length + retryableRefusals.length > maxAllowedRejections) {
1007
1097
  return { phase: TransactionPhase.ConflictSuperseded };
1008
1098
  }
1009
1099
 
@@ -1073,24 +1163,20 @@ export class ClusterMember implements ICluster {
1073
1163
  // peer set is a legitimate cluster it belongs to, and refuses (reject vote) rather than rubber-stamping
1074
1164
  // a set the coordinator chose (e.g. a self-shrunk minority-partition set). On admission failure we skip
1075
1165
  // pend validation entirely and emit the membership rejection.
1076
- const validationResult = await this.evaluatePromise(record);
1077
-
1078
- const promiseHash = await this.computePromiseHash(record);
1079
- const type = validationResult.valid ? 'approve' as const : 'reject' as const;
1080
- const rejectReason = validationResult.valid ? undefined : validationResult.reason;
1081
- const sig = await this.signVote(promiseHash, type, rejectReason);
1166
+ const verdict = await this.evaluatePromise(record);
1082
1167
 
1083
- const signature: Signature = validationResult.valid
1084
- ? { type: 'approve', signature: sig }
1085
- : { type: 'reject', signature: sig, rejectReason };
1086
-
1087
- if (!validationResult.valid) {
1088
- log('cluster-member:validation-rejected', {
1168
+ if (!verdict.valid) {
1169
+ // Two tags, because the two refusals are two different events for an operator: one says the
1170
+ // cohort judged a write invalid, the other says a write queued behind a live reservation.
1171
+ log(verdict.kind === 'held' ? 'cluster-member:validation-held' : 'cluster-member:validation-rejected', {
1089
1172
  messageHash: record.messageHash,
1090
- reason: validationResult.reason
1173
+ reason: verdict.reason
1091
1174
  });
1092
1175
  }
1093
1176
 
1177
+ const promiseHash = await this.computePromiseHash(record);
1178
+ const signature = await this.signPromiseVerdict(promiseHash, verdict);
1179
+
1094
1180
  return {
1095
1181
  ...record,
1096
1182
  promises: {
@@ -1100,6 +1186,23 @@ export class ClusterMember implements ICluster {
1100
1186
  };
1101
1187
  }
1102
1188
 
1189
+ /**
1190
+ * Turn a {@link PromiseVerdict} into this member's signed promise vote — one vote kind per refusal
1191
+ * kind. The single place the mapping lives, so a new refusal kind cannot reach the wire as the
1192
+ * wrong vote: the coordinator's thresholds read only `Signature.type`, and until `held` existed the
1193
+ * transient refusal had nowhere to go but `reject`.
1194
+ */
1195
+ private async signPromiseVerdict(promiseHash: string, verdict: PromiseVerdict): Promise<Signature> {
1196
+ if (verdict.valid) {
1197
+ return { type: 'approve', signature: await this.signVote(promiseHash, 'approve') };
1198
+ }
1199
+ if (verdict.kind === 'held') {
1200
+ return { type: 'held', signature: await this.signVote(promiseHash, 'held', verdict.heldBy), heldBy: verdict.heldBy };
1201
+ }
1202
+ const rejectReason = verdict.reason;
1203
+ return { type: 'reject', signature: await this.signVote(promiseHash, 'reject', rejectReason), rejectReason };
1204
+ }
1205
+
1103
1206
  /**
1104
1207
  * Answer a record that lost the deterministic race to a transaction this member already holds
1105
1208
  * (`docs/correctness.md` Theorems 1 & 9: the loser is TOLD it lost, not ignored — an unanswered
@@ -1129,17 +1232,19 @@ export class ClusterMember implements ICluster {
1129
1232
 
1130
1233
  /**
1131
1234
  * The full promise-phase decision for a record: admit the declared membership FIRST, then (only if
1132
- * admitted) validate its pend operations, then its commit operations. Failing any yields a
1133
- * `{ valid:false, reason }` the caller turns into a `reject` vote. Keeping the three separate keeps
1235
+ * admitted) validate its pend operations, then its commit operations. Failing any yields a refusal
1236
+ * the caller turns into a vote — a `reject` for every validity judgement here, and a `held` for the
1237
+ * one transient refusal {@link validatePendOperations} can make (see {@link PromiseVerdict}).
1238
+ * Keeping the checks separate keeps
1134
1239
  * the reason strings distinct — a `membership-not-admitted` reject is a different signal (feeds the
1135
1240
  * dispute path) than a stale-revision / custom-validator reject, which is different again from a
1136
1241
  * `content-digest-mismatch` (see {@link validateCommitOperations}). A record carries pend OR commit
1137
1242
  * operations, so in practice exactly one of the latter two has anything to inspect.
1138
1243
  */
1139
- private async evaluatePromise(record: ClusterRecord): Promise<{ valid: boolean; reason?: string }> {
1244
+ private async evaluatePromise(record: ClusterRecord): Promise<PromiseVerdict> {
1140
1245
  const admission = await this.admitMembership(record);
1141
1246
  if (!admission.admit) {
1142
- return { valid: false, reason: admission.reason ?? MEMBERSHIP_NOT_ADMITTED };
1247
+ return invalidVerdict(admission.reason ?? MEMBERSHIP_NOT_ADMITTED);
1143
1248
  }
1144
1249
  const pendValidation = await this.validatePendOperations(record);
1145
1250
  if (!pendValidation.valid) {
@@ -1151,7 +1256,7 @@ export class ClusterMember implements ICluster {
1151
1256
  // declared one).
1152
1257
  const commitRevValidation = await this.validateCommitRevisions(record);
1153
1258
  if (!commitRevValidation.valid) {
1154
- return commitRevValidation;
1259
+ return verdictOf(commitRevValidation);
1155
1260
  }
1156
1261
  // Then our own refusal history: a commit whose pend THIS member refused, where local state
1157
1262
  // still corroborates the refusal. Runs after the revision check because that one is sharper
@@ -1159,9 +1264,16 @@ export class ClusterMember implements ICluster {
1159
1264
  // applied here; this arm covers the window where it has not.
1160
1265
  const refusedPendValidation = await this.validateCommitAgainstRefusedPend(record);
1161
1266
  if (!refusedPendValidation.valid) {
1162
- return refusedPendValidation;
1267
+ return verdictOf(refusedPendValidation);
1163
1268
  }
1164
- return await this.validateCommitOperations(record);
1269
+ // The declared base before the declared content: it needs no materialization, and when the two
1270
+ // disagree the digest check would be previewing operations computed against a different base
1271
+ // than the one declared — its verdict names the symptom, this one names the cause.
1272
+ const baseDeclarationValidation = await this.validateCommitBaseDeclarations(record);
1273
+ if (!baseDeclarationValidation.valid) {
1274
+ return verdictOf(baseDeclarationValidation);
1275
+ }
1276
+ return verdictOf(await this.validateCommitOperations(record));
1165
1277
  }
1166
1278
 
1167
1279
  /**
@@ -1459,11 +1571,21 @@ export class ClusterMember implements ICluster {
1459
1571
 
1460
1572
  /**
1461
1573
  * Validates pend operations in a cluster record using the transaction validator.
1462
- * Also checks for stale revisions, and for blocks held by a different unresolved pending
1574
+ * Also checks for stale revisions, and for blocks RESERVED by a different unresolved pending
1463
1575
  * action, to prevent consensus on operations that storage would refuse at apply.
1464
1576
  * Returns success if no validator is configured (backwards compatibility).
1577
+ *
1578
+ * Every refusal here is a validity judgement EXCEPT the pending-conflict one, which is transient by
1579
+ * construction and returns the `held` kind — see {@link PromiseVerdict}. Transient means the
1580
+ * reservation it answers is one its holder is still going to commit or cancel: a record claiming
1581
+ * the slot this pend wants, or a later one. A record claiming a slot the collection has already
1582
+ * moved past is NOT a reservation against this pend and is not refused on — see
1583
+ * {@link reservingRivals} — because its holder is never going to remove it: the holder either
1584
+ * committed at that slot on the rest of the cohort (this member missed the commit) or lost the
1585
+ * slot, and in both cases the record would otherwise refuse every later write to the block, from
1586
+ * every writer, for as long as this member lives.
1465
1587
  */
1466
- private async validatePendOperations(record: ClusterRecord): Promise<{ valid: boolean; reason?: string }> {
1588
+ private async validatePendOperations(record: ClusterRecord): Promise<PromiseVerdict> {
1467
1589
  // Find pend operations in the message
1468
1590
  for (const operation of record.message.operations) {
1469
1591
  if ('pend' in operation) {
@@ -1489,7 +1611,7 @@ export class ClusterMember implements ICluster {
1489
1611
  blockId,
1490
1612
  reason: blockResult.unavailable
1491
1613
  });
1492
- return { valid: false, reason: `block ${blockId} unavailable (${blockResult.unavailable}): cannot verify revision` };
1614
+ return invalidVerdict(`block ${blockId} unavailable (${blockResult.unavailable}): cannot verify revision`);
1493
1615
  }
1494
1616
  const latest = blockResult?.state?.latest;
1495
1617
  if (latest !== undefined && latest.rev >= pendRequest.rev) {
@@ -1512,19 +1634,19 @@ export class ClusterMember implements ICluster {
1512
1634
  // is NOT a StaleFailure producer, so StaleFailure.staleAt does not apply; the
1513
1635
  // coordinator's own local re-read (CoordinatorRepo.classifyStaleRejection)
1514
1636
  // supplies that number when it can confirm the revision itself.
1515
- return { valid: false, reason: `stale revision: block ${blockId} at rev ${latest.rev}, requested rev ${pendRequest.rev}` };
1637
+ return invalidVerdict(`stale revision: block ${blockId} at rev ${latest.rev}, requested rev ${pendRequest.rev}`);
1516
1638
  }
1517
1639
  }
1518
1640
  }
1519
1641
 
1520
- // Reject a pend whose blocks are held by a DIFFERENT unresolved pending action. This is
1521
- // the durable reservation the in-memory table (`findConflict` / `activeTransactions`)
1642
+ // Refuse a pend whose blocks are RESERVED by a DIFFERENT unresolved pending action. This
1643
+ // is the durable reservation the in-memory table (`findConflict` / `activeTransactions`)
1522
1644
  // cannot provide: that table clears the moment the rival's PEND record reaches
1523
1645
  // consensus, but the rival's storage pending record — written at pend-apply, removed at
1524
1646
  // commit or cancel — spans exactly the pend→commit window in which `latest.rev` has not
1525
1647
  // yet advanced. Storage's own pend would refuse this request at consensus-apply for the
1526
- // same reason (`StorageRepo.pend`'s listPendingTransactions scan); voting reject here
1527
- // moves that verdict into the phase where the cohort aggregates it, so the loser is
1648
+ // same reason (`StorageRepo.pend`'s pending-claim scan, under the same rule); voting
1649
+ // here moves that verdict into the phase where the cohort aggregates it, so the loser is
1528
1650
  // refused with a real answer instead of burning a consensus round it cannot win. A
1529
1651
  // member that has not yet applied the rival's pend has no record and simply abstains
1530
1652
  // from this reason; the apply-time verdict catches that residual — retained locally
@@ -1533,19 +1655,53 @@ export class ClusterMember implements ICluster {
1533
1655
  // member. Self is excluded so a redelivered pend
1534
1656
  // for this same action stays approvable. An unavailable block carries no `pendings` and
1535
1657
  // abstains (the rev branch above already fail-closes when a revision claim is at stake).
1536
- // Reason stays plain prose: it is fed to computeSigningPayload and carried as
1537
- // Signature.rejectReason, exactly like the stale-revision reason above.
1658
+ //
1659
+ // This is the ONE refusal in this method that is not a validity judgement: a RESERVING
1660
+ // rival's record is removed the moment it commits or cancels, so the very same pend
1661
+ // succeeds on retry. It therefore returns the `held` kind, which becomes a `held` vote
1662
+ // the coordinator counts toward neither approvals nor rejections. `heldBy` carries the
1663
+ // first rival as signed structured data; the prose reason names the same one, and stays
1664
+ // prose because it is fed to computeSigningPayload exactly like the reasons above.
1665
+ //
1666
+ // Which rivals reserve is decided by the slot each record claims against what the
1667
+ // incoming writer built on, not by the record's presence (`reservingRivals`): a record
1668
+ // whose slot is at or below the base this pend declares for the block — or, for a pend
1669
+ // naming no base, below the revision it requests — belongs to a commit this member missed
1670
+ // or a race its holder lost, and would never clear on its own; refusing on it wedged the
1671
+ // block for every writer (ticket `a-member-that-missed-a-commit-refuses-every-later-write`).
1672
+ // Approving over it is safe because the incoming writer's operations were computed
1673
+ // against a version of the block that already holds that record's change. This member
1674
+ // comes current when the approved pend's own commit applies here — through
1675
+ // `internalCommit`, or through the behind-reconcile its fork guard triggers. In a cohort
1676
+ // that can reach its promise bar without one member (`cohortCanMissAPend`: four members up
1677
+ // at the default threshold), a record claiming a slot PAST the declared base still
1678
+ // reserves even when the requested revision has moved beyond it: that is a writer that
1679
+ // read the block without the record's change — served by a member that never held the
1680
+ // rival's pend — and admitting it would lose the change. See `isReservationAgainst`.
1538
1681
  for (const blockId of blockIds) {
1539
- const rivals = (blockResults[blockId]?.state?.pendings ?? []).filter(actionId => actionId !== pendRequest.actionId);
1540
- if (rivals.length > 0) {
1541
- log('cluster-member:validation-pending-conflict', {
1542
- messageHash: record.messageHash,
1543
- blockId,
1544
- actionId: pendRequest.actionId,
1545
- rivals
1546
- });
1547
- return { valid: false, reason: `pending conflict: block ${blockId} held by unresolved action(s) ${rivals.join(', ')}` };
1682
+ const rivalIds = (blockResults[blockId]?.state?.pendings ?? []).filter(actionId => actionId !== pendRequest.actionId);
1683
+ const rivals = rivalIds.length === 0 ? [] : await this.reservingRivals(record, blockId, rivalIds, pendRequest);
1684
+ if (rivals.length === 0) {
1685
+ // Not reserved (any more): whatever episode this member was counting on the block
1686
+ // has ended, so a later wedge gets its own count.
1687
+ this.stuckReservations.forget([blockId]);
1688
+ continue;
1548
1689
  }
1690
+ const heldBy = rivals[0]!;
1691
+ log('cluster-member:validation-pending-conflict', {
1692
+ messageHash: record.messageHash,
1693
+ blockId,
1694
+ actionId: pendRequest.actionId,
1695
+ requestedRev: pendRequest.rev,
1696
+ rivals
1697
+ });
1698
+ this.nameStuckReservation(blockId, rivals, pendRequest.actionId);
1699
+ return {
1700
+ valid: false,
1701
+ kind: 'held',
1702
+ heldBy,
1703
+ reason: `pending conflict: block ${blockId} held by unresolved action(s) ${rivals.join(', ')}`
1704
+ };
1549
1705
  }
1550
1706
 
1551
1707
  // Re-check the transaction when a validator is configured. The unvalidatable-pend
@@ -1572,7 +1728,7 @@ export class ClusterMember implements ICluster {
1572
1728
  })
1573
1729
  );
1574
1730
  if (!validation.valid) {
1575
- return { valid: false, reason: validation.reason };
1731
+ return invalidVerdict(validation.reason);
1576
1732
  }
1577
1733
  }
1578
1734
  }
@@ -1580,6 +1736,88 @@ export class ClusterMember implements ICluster {
1580
1736
  return { valid: true };
1581
1737
  }
1582
1738
 
1739
+ /**
1740
+ * Of the rival pending records `get` listed on `blockId`, the ones that RESERVE the block against
1741
+ * `pendRequest` — see `isReservationAgainst` for the rule, fed the pend's revision and the base it
1742
+ * declares for this block (`reservationRequestFor`, the same reading storage applies at apply).
1743
+ * Asks storage's {@link IPendingClaimReader} for the slot each record claims; a repo without that
1744
+ * capability (a plain `IRepo` mock) or a read that fails degrades to "every rival reserves", which
1745
+ * is the refusal this member cast before claims were recorded — never to silently admitting one. A
1746
+ * rival that `get` listed but that is gone by the time the claims are read has resolved in
1747
+ * between, and is not a rival any more.
1748
+ */
1749
+ private async reservingRivals(record: ClusterRecord, blockId: BlockId, rivalIds: ActionId[], pendRequest: PendRequest): Promise<ActionId[]> {
1750
+ const reader = this.storageRepo as IRepo & Partial<IPendingClaimReader>;
1751
+ if (typeof reader.listPendingClaims !== 'function') {
1752
+ return rivalIds;
1753
+ }
1754
+ let claims: PendingClaim[];
1755
+ try {
1756
+ claims = await reader.listPendingClaims(blockId);
1757
+ } catch (err) {
1758
+ log('cluster-member:pending-claims-read-error', { messageHash: record.messageHash, blockId, error: (err as Error).message });
1759
+ return rivalIds;
1760
+ }
1761
+ const reservation = this.reservationRequestOf(record, pendRequest, blockId);
1762
+ const claimOf = new Map(claims.map(claim => [claim.actionId, claim]));
1763
+ const reserving: ActionId[] = [];
1764
+ for (const actionId of rivalIds) {
1765
+ const claim = claimOf.get(actionId);
1766
+ if (claim === undefined) continue;
1767
+ if (isReservationAgainst(claim, reservation)) {
1768
+ reserving.push(actionId);
1769
+ } else {
1770
+ log('cluster-member:validation-pending-superseded', {
1771
+ messageHash: record.messageHash,
1772
+ blockId,
1773
+ actionId: pendRequest.actionId,
1774
+ requestedRev: pendRequest.rev,
1775
+ baseRev: reservation.baseRev,
1776
+ rival: actionId,
1777
+ claimedRev: claim.rev
1778
+ });
1779
+ }
1780
+ }
1781
+ return reserving;
1782
+ }
1783
+
1784
+ /** `pendRequest`'s {@link ReservationRequest} for `blockId`, logging a base the rule cannot read —
1785
+ * never a refusal: `baseRevs` is untrusted wire data, and a malformed entry only drops that block
1786
+ * back to the revision rule. The base is read only when `record`'s cohort can reach its promise
1787
+ * super-majority without one of its members (`cohortCanMissAPend`); in a cohort that needs every
1788
+ * member, a stray record on one member (a cancel that never reached it) would otherwise refuse
1789
+ * every later writer for good, and no member can have missed the pend the base arm guards against. */
1790
+ private reservationRequestOf(record: ClusterRecord, pendRequest: PendRequest, blockId: BlockId): ReservationRequest {
1791
+ if (!cohortCanMissAPend(Object.keys(record.peers).length, this.superMajorityThreshold)) {
1792
+ return { rev: pendRequest.rev };
1793
+ }
1794
+ const { request, ignoredBase } = reservationRequestFor(pendRequest, blockId, transformForBlockId(pendRequest.transforms, blockId));
1795
+ if (ignoredBase !== undefined) {
1796
+ log('cluster-member:pend-base-ignored', {
1797
+ messageHash: record.messageHash,
1798
+ blockId,
1799
+ actionId: pendRequest.actionId,
1800
+ requestedRev: pendRequest.rev,
1801
+ base: ignoredBase
1802
+ });
1803
+ }
1804
+ return request;
1805
+ }
1806
+
1807
+ /**
1808
+ * Feed this member's own `held` vote to its {@link stuckReservations} counter and say, once per
1809
+ * episode and in words, when a block is wedged behind a reservation that is not going to clear —
1810
+ * the member-side twin of `CoordinatorRepo.noteStuckReservation`, for the reservations that
1811
+ * node cannot corroborate from its own storage. `peerId` is carried because this logger is not
1812
+ * suffixed with one and several members can share a process.
1813
+ */
1814
+ private nameStuckReservation(blockId: BlockId, holders: readonly ActionId[], refusedActionId: ActionId): void {
1815
+ const { named } = this.stuckReservations.note(new Map([[blockId, holders]]), refusedActionId);
1816
+ for (const episode of named) {
1817
+ log('cluster-member:stuck-reservation', { peerId: this.peerId.toString(), ...episode });
1818
+ }
1819
+ }
1820
+
1583
1821
  /**
1584
1822
  * Promise-round check that a commit record's requested revision is not already committed HERE
1585
1823
  * under a different action. This is the member-side arm that keeps a DEAD rival's re-broadcast
@@ -1793,6 +2031,113 @@ export class ClusterMember implements ICluster {
1793
2031
  return undefined;
1794
2032
  }
1795
2033
 
2034
+ /**
2035
+ * Promise-round check that a commit declares, for each block, the base revision the same action's
2036
+ * PEND carried to this member. The author says the base twice — `PendRequest.baseRevs[id]`, kept
2037
+ * with this member's pending record as `PendingClaim.baseRev`, and
2038
+ * `CommitRequest.blockDigests[id].baseRev` — and an honest author says the same thing both times
2039
+ * (`Tracker` pins the base at staging; the pend and the digest both read that pin). When they
2040
+ * differ, this member's record holds operations computed against a base other than the one the
2041
+ * commit is about to be applied as, so vote reject with {@link BASE_DECLARATION_DISAGREES}.
2042
+ * `StorageRepo.internalCommit` refuses the same shape at apply (`guardCommitBase`); refusing here
2043
+ * too puts a signed verdict on the record one round earlier, and when the cohort's records agree
2044
+ * with each other but not with the commit (a writer whose commit contradicts its own pend) the
2045
+ * whole cohort refuses at the vote, where the apply-time refusal alone would have every member
2046
+ * refuse after consensus and reconcile against a revision nobody holds.
2047
+ *
2048
+ * Needs no preview, so it runs on a repo that can read one record's claim
2049
+ * ({@link IPendingClaimReader}`.pendingClaimOf`) whether or not it can materialize. Abstains — votes
2050
+ * as it would without the check — whenever it cannot compare: the commit declares nothing, an entry
2051
+ * is surplus to `blockIds` or carries no numeric `baseRev` (untrusted wire data, same posture as the
2052
+ * digest check), this member holds no record for the action on the block, the record carries no
2053
+ * base (an inserted or deleted block, a base-less sender, or a record written before bases were
2054
+ * kept), the read fails — or this member does not hold the declared base as its latest.
2055
+ *
2056
+ * That last abstain is what keeps an HONEST retry off this reject. The one disagreement an honest
2057
+ * writer can meet here is a stale record from an earlier attempt of a retried action (same action
2058
+ * id) meeting the retry's commit, when the retry's pend — which would have overwritten the record
2059
+ * — never reached this member. Such a member is typically behind: it missed a pend, and the retry
2060
+ * declares a base the rest of the cohort moved to. At three members one reject sinks the commit
2061
+ * record (the default super-majority allows none), where abstaining lets the others commit and
2062
+ * leaves this member to `guardCommitBase`, which refuses to apply the stale record and reconciles.
2063
+ * When this member's latest IS the declared base, the stale operations are exactly what it would
2064
+ * apply over it, and the digest check below rejects the same shape whenever they materialize
2065
+ * differently; rejecting here names the cause instead.
2066
+ */
2067
+ private async validateCommitBaseDeclarations(record: ClusterRecord): Promise<{ valid: boolean; reason?: string }> {
2068
+ const reader = this.storageRepo as IRepo & Partial<IPendingClaimReader>;
2069
+ if (typeof reader.pendingClaimOf !== 'function') {
2070
+ return { valid: true };
2071
+ }
2072
+ for (const operation of record.message.operations) {
2073
+ if (!('commit' in operation)) {
2074
+ continue;
2075
+ }
2076
+ const commit = operation.commit;
2077
+ if (!commit.blockDigests) {
2078
+ continue;
2079
+ }
2080
+ const committedIds = new Set<string>(commit.blockIds);
2081
+ for (const [blockId, declared] of Object.entries(commit.blockDigests)) {
2082
+ const declaredBaseRev: unknown = declared?.baseRev;
2083
+ if (!committedIds.has(blockId) || typeof declaredBaseRev !== 'number') {
2084
+ continue;
2085
+ }
2086
+ let storedBaseRev: number | undefined;
2087
+ try {
2088
+ storedBaseRev = (await reader.pendingClaimOf(blockId as BlockId, commit.actionId))?.baseRev;
2089
+ } catch (err) {
2090
+ log('cluster-member:base-declaration-read-error', {
2091
+ messageHash: record.messageHash,
2092
+ blockId,
2093
+ error: err instanceof Error ? err.message : String(err)
2094
+ });
2095
+ continue; // a local read fault is an abstain, never a verdict
2096
+ }
2097
+ if (storedBaseRev === undefined || storedBaseRev === declaredBaseRev) {
2098
+ continue;
2099
+ }
2100
+ const latestRev = await this.latestRevOf(record, blockId as BlockId);
2101
+ if (latestRev !== declaredBaseRev) {
2102
+ log('cluster-member:base-declaration-disagrees-abstained', {
2103
+ messageHash: record.messageHash,
2104
+ blockId,
2105
+ actionId: commit.actionId,
2106
+ storedBaseRev,
2107
+ declaredBaseRev,
2108
+ latestRev
2109
+ });
2110
+ continue;
2111
+ }
2112
+ log('cluster-member:base-declaration-disagrees', {
2113
+ messageHash: record.messageHash,
2114
+ blockId,
2115
+ actionId: commit.actionId,
2116
+ rev: commit.rev,
2117
+ storedBaseRev,
2118
+ declaredBaseRev
2119
+ });
2120
+ return { valid: false, reason: BASE_DECLARATION_DISAGREES };
2121
+ }
2122
+ }
2123
+ return { valid: true };
2124
+ }
2125
+
2126
+ /** This member's own committed revision of `blockId` (raw storage, no cluster recursion), or
2127
+ * `undefined` when it holds none or the read fails — a fault reads as "cannot judge". */
2128
+ private async latestRevOf(record: ClusterRecord, blockId: BlockId): Promise<number | undefined> {
2129
+ try {
2130
+ return (await this.storageRepo.get({ blockIds: [blockId] }))[blockId]?.state?.latest?.rev;
2131
+ } catch (err) {
2132
+ log('cluster-member:base-declaration-read-error', {
2133
+ messageHash: record.messageHash,
2134
+ blockId,
2135
+ error: err instanceof Error ? err.message : String(err)
2136
+ });
2137
+ return undefined;
2138
+ }
2139
+ }
2140
+
1796
2141
  /**
1797
2142
  * Promise-round check of a commit record's declared content digests
1798
2143
  * (`CommitRequest.blockDigests`) against what this member's OWN pended copy of each transform
@@ -1991,6 +2336,7 @@ export class ClusterMember implements ICluster {
1991
2336
  this.executedTransactions.delete(record.messageHash);
1992
2337
  this.executedPendResults.delete(record.messageHash);
1993
2338
  this.executedCommitResults.delete(record.messageHash);
2339
+ this.behindCommitRefusals.delete(record.messageHash);
1994
2340
  for (const operation of record.message.operations) {
1995
2341
  if ('pend' in operation) this.refusedPendActions.delete(operation.pend.actionId);
1996
2342
  }
@@ -2138,7 +2484,7 @@ export class ClusterMember implements ICluster {
2138
2484
  membershipVersion: record.membershipVersion
2139
2485
  });
2140
2486
  }
2141
- const applied = await this.applyCommitToStorage(record, commit, proof);
2487
+ const { applied, behind } = await this.applyCommitToStorage(record, commit, proof);
2142
2488
  // Retain the POST-RECONCILE durable verdict (see getExecutedCommitResult and
2143
2489
  // withOwnApplyOutcome): what this member's storage holds NOW, after any behind-reconcile the
2144
2490
  // apply triggered — not what the first apply attempt said. A member that pulled the
@@ -2147,7 +2493,7 @@ export class ClusterMember implements ICluster {
2147
2493
  // verdicts — its own member's through getExecutedCommitResult, every other member's off the
2148
2494
  // response record — and the two readers must see the same answer, hence one verdict,
2149
2495
  // computed once, retained here for both.
2150
- this.executedCommitResults.set(messageHash, await this.durableCommitVerdict(commit, applied));
2496
+ this.retainCommitVerdict(record, commit, await this.durableCommitVerdict(commit, applied), behind);
2151
2497
  return;
2152
2498
  }
2153
2499
  if ('invalidate' in operation) {
@@ -2159,12 +2505,14 @@ export class ClusterMember implements ICluster {
2159
2505
  /**
2160
2506
  * Apply one consensus commit to local storage, tolerating every divergence shape the way the
2161
2507
  * doc comment on {@link applyConsensusOperation} describes and reconciling the behind ones.
2162
- * Returns storage's own result — or, for the thrown missing-pend shape, which produces no
2508
+ * `applied` is storage's own result — or, for the thrown missing-pend shape, which produces no
2163
2509
  * `CommitResult` at all, a refusal built from the throw — so {@link durableCommitVerdict} works
2164
- * from one uniform shape. Genuine faults (a bare-reason returned failure, an unrecognized throw)
2165
- * propagate so {@link handleConsensus} rolls back the executed marker and rethrows.
2510
+ * from one uniform shape. `behind` is true exactly when one of the two behind branches ran, which
2511
+ * is what makes a refusal eligible for {@link reconcileRefusedCommit}. Genuine faults (a
2512
+ * bare-reason returned failure, an unrecognized throw) propagate so {@link handleConsensus} rolls
2513
+ * back the executed marker and rethrows.
2166
2514
  */
2167
- private async applyCommitToStorage(record: ClusterRecord, commit: CommitRequest, proof: BlockCommitProof | undefined): Promise<CommitResult> {
2515
+ private async applyCommitToStorage(record: ClusterRecord, commit: CommitRequest, proof: BlockCommitProof | undefined): Promise<{ applied: CommitResult; behind: boolean }> {
2168
2516
  const messageHash = record.messageHash;
2169
2517
  let result: CommitResult;
2170
2518
  try {
@@ -2184,12 +2532,12 @@ export class ClusterMember implements ICluster {
2184
2532
  // cohort peer so the block is no longer under-replicated. Best-effort:
2185
2533
  // failures are logged inside, never thrown (a throw would reset the stream).
2186
2534
  await this.reconcileDivergentCommit(record, commit);
2187
- return { success: false, reason: (err as Error).message };
2535
+ return { applied: { success: false, reason: (err as Error).message }, behind: true };
2188
2536
  }
2189
2537
  throw err;
2190
2538
  }
2191
2539
  if (result.success) {
2192
- return result;
2540
+ return { applied: result, behind: false };
2193
2541
  }
2194
2542
  // success:false is a StaleFailure. `missing` ⇒ ahead/stale divergence
2195
2543
  // (we already hold ≥ this rev): tolerate, do NOT reconcile downward. A
@@ -2219,7 +2567,7 @@ export class ClusterMember implements ICluster {
2219
2567
  reason: result.reason,
2220
2568
  hasMissing: true
2221
2569
  });
2222
- return result;
2570
+ return { applied: result, behind: false };
2223
2571
  }
2224
2572
  // This member holds no materializable base for one of the blocks, so
2225
2573
  // `StorageRepo.commit` REFUSED rather than record a revision it could never serve.
@@ -2235,11 +2583,64 @@ export class ClusterMember implements ICluster {
2235
2583
  reason: result.reason
2236
2584
  });
2237
2585
  await this.reconcileDivergentCommit(record, commit);
2238
- return result;
2586
+ return { applied: result, behind: true };
2239
2587
  }
2240
2588
  throw new Error(`Consensus commit for action ${commit.actionId} failed: ${result.reason ?? 'unknown reason'}`);
2241
2589
  }
2242
2590
 
2591
+ /**
2592
+ * Retain `verdict` as this member's durable verdict for `record`'s commit (see
2593
+ * {@link executedCommitResults}), remember whether a refusal was the behind shape, and — once the
2594
+ * verdict is a success — report the committed holders. Shared by the consensus apply and
2595
+ * {@link reconcileRefusedCommit} so both leave the same state behind.
2596
+ */
2597
+ private retainCommitVerdict(record: ClusterRecord, commit: CommitRequest, verdict: CommitResult, behind: boolean): void {
2598
+ this.executedCommitResults.set(record.messageHash, verdict);
2599
+ if (verdict.success) {
2600
+ this.behindCommitRefusals.delete(record.messageHash);
2601
+ this.reportCommittedHolders(record, commit);
2602
+ } else if (behind) {
2603
+ this.behindCommitRefusals.add(record.messageHash);
2604
+ }
2605
+ }
2606
+
2607
+ /**
2608
+ * One more reconcile for a commit this member already applied at consensus but could not hold:
2609
+ * its retained verdict is a BEHIND-shaped refusal (missing pend or missing base), and the
2610
+ * reconcile that ran during the apply found no cohort peer holding the revision yet. The
2611
+ * coordinating node calls this for its own member once the remote members have applied and at
2612
+ * least one reported holding the revision. Its own delivery goes first (see
2613
+ * `ClusterCoordinator.broadcastMergedRecord`), so that first reconcile ran before anyone else
2614
+ * held anything. The remote holder's copy carries the cohort's commit proof, so the existing
2615
+ * certified single-holder path adopts it; no new trust is involved.
2616
+ *
2617
+ * Does nothing unless the retained verdict for `record.messageHash` is a behind refusal; in
2618
+ * particular an ahead-shaped refusal (`missing`) is never reconciled downward. Recomputes and
2619
+ * re-retains the durable verdict afterwards, so `getExecutedCommitResult` then answers a success
2620
+ * when the revision landed. Bounded by the same per-block reconcile timeout and never throws.
2621
+ */
2622
+ async reconcileRefusedCommit(record: ClusterRecord): Promise<void> {
2623
+ const messageHash = record.messageHash;
2624
+ const refused = this.executedCommitResults.get(messageHash);
2625
+ if (refused === undefined || refused.success || !this.behindCommitRefusals.has(messageHash)) {
2626
+ return;
2627
+ }
2628
+ const operation = record.message.operations.find((op): op is { commit: CommitRequest } => 'commit' in op);
2629
+ if (operation === undefined) {
2630
+ return;
2631
+ }
2632
+ const commit = operation.commit;
2633
+ log('cluster-member:consensus-commit-reconcile-again', { messageHash, actionId: commit.actionId, rev: commit.rev });
2634
+ await this.reconcileDivergentCommit(record, commit);
2635
+ const verdict = await this.durableCommitVerdict(commit, refused);
2636
+ // The apply's retention may have been pruned or rolled back while the reconcile ran; never
2637
+ // resurrect it.
2638
+ if (this.executedCommitResults.get(messageHash) !== refused) {
2639
+ return;
2640
+ }
2641
+ this.retainCommitVerdict(record, commit, verdict, true);
2642
+ }
2643
+
2243
2644
  /**
2244
2645
  * The durable verdict for a consensus commit: `applied` itself when storage landed it (a
2245
2646
  * success from `StorageRepo.commit` means every block is committed, or already was, under this
@@ -2268,7 +2669,10 @@ export class ClusterMember implements ICluster {
2268
2669
  return applied;
2269
2670
  }
2270
2671
  log('cluster-member:consensus-commit-durable-after-reconcile', { actionId: commit.actionId, rev: commit.rev });
2271
- return { success: true };
2672
+ // A member's own verdict about its own storage: `local`. Read by the coordinator for its
2673
+ // `success` flag only (`cohortCommitOutcomes`, `localCommitResult`) — it never becomes the
2674
+ // answer handed to the writer; `CoordinatorRepo.commit` computes the cohort's own on every exit.
2675
+ return { success: true, durability: localDurability() };
2272
2676
  }
2273
2677
 
2274
2678
  /**
@@ -2392,6 +2796,29 @@ export class ClusterMember implements ICluster {
2392
2796
  }
2393
2797
  }
2394
2798
 
2799
+ /**
2800
+ * Tell the {@link CommittedHoldersSink} which cohort members hold a commit this member just
2801
+ * durably applied. A member learns nothing about the others' storage at apply time, so the
2802
+ * evidence is the record's approving commit signers: each is a cohort member that signed to apply
2803
+ * this commit. A signer that then failed to apply is wrongly recorded, which only spares it a
2804
+ * rebalance push; its own reconcile, read-repair, and — on the coordinating node — the
2805
+ * coordinator's durability-checked report (which lands after this one and overrides it) still
2806
+ * reach it.
2807
+ */
2808
+ private reportCommittedHolders(record: ClusterRecord, commit: CommitRequest): void {
2809
+ if (!this.onCommittedHolders) {
2810
+ return;
2811
+ }
2812
+ const holders = Object.entries(record.commits)
2813
+ .filter(([peerId, vote]) => vote.type === 'approve' && peerId in record.peers)
2814
+ .map(([peerId]) => peerId);
2815
+ try {
2816
+ this.onCommittedHolders({ blockIds: commit.blockIds, holders });
2817
+ } catch (err) {
2818
+ log('cluster-member:committed-holders-sink-error', { actionId: commit.actionId, error: (err as Error).message });
2819
+ }
2820
+ }
2821
+
2395
2822
  /**
2396
2823
  * After tolerating a "behind" commit divergence, pull the committed revision of
2397
2824
  * each block from a cohort peer that holds it and restore it locally. Best-effort:
@@ -2616,6 +3043,7 @@ export class ClusterMember implements ICluster {
2616
3043
  this.executedTransactions.delete(messageHash);
2617
3044
  this.executedPendResults.delete(messageHash);
2618
3045
  this.executedCommitResults.delete(messageHash);
3046
+ this.behindCommitRefusals.delete(messageHash);
2619
3047
  }
2620
3048
  }
2621
3049
  // Prune actionId-keyed pend refusals on the same TTL as the verdicts they were derived from.