@optimystic/db-p2p 1.0.0-beta.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/dist/src/cluster/block-transfer-service.d.ts +73 -1
  2. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  3. package/dist/src/cluster/block-transfer-service.js +70 -3
  4. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  5. package/dist/src/cluster/block-transfer.d.ts +16 -1
  6. package/dist/src/cluster/block-transfer.d.ts.map +1 -1
  7. package/dist/src/cluster/block-transfer.js +73 -72
  8. package/dist/src/cluster/block-transfer.js.map +1 -1
  9. package/dist/src/cluster/client.d.ts +4 -6
  10. package/dist/src/cluster/client.d.ts.map +1 -1
  11. package/dist/src/cluster/client.js +8 -13
  12. package/dist/src/cluster/client.js.map +1 -1
  13. package/dist/src/cluster/cluster-policy.d.ts +2 -1
  14. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  15. package/dist/src/cluster/cluster-policy.js.map +1 -1
  16. package/dist/src/cluster/cluster-repo.d.ts +172 -17
  17. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  18. package/dist/src/cluster/cluster-repo.js +553 -149
  19. package/dist/src/cluster/cluster-repo.js.map +1 -1
  20. package/dist/src/cluster/commit-proof.js +2 -2
  21. package/dist/src/cluster/race-resolution.d.ts +51 -7
  22. package/dist/src/cluster/race-resolution.d.ts.map +1 -1
  23. package/dist/src/cluster/race-resolution.js +71 -7
  24. package/dist/src/cluster/race-resolution.js.map +1 -1
  25. package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
  26. package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
  27. package/dist/src/cluster/rebalance-monitor.js +120 -26
  28. package/dist/src/cluster/rebalance-monitor.js.map +1 -1
  29. package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
  30. package/dist/src/cluster/spread-on-churn.js +34 -47
  31. package/dist/src/cluster/spread-on-churn.js.map +1 -1
  32. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  33. package/dist/src/cohort-topic/host.js +8 -6
  34. package/dist/src/cohort-topic/host.js.map +1 -1
  35. package/dist/src/cohort-topic/membership-source.js +1 -1
  36. package/dist/src/cohort-topic/membership-source.js.map +1 -1
  37. package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
  38. package/dist/src/cohort-topic/peer-codec.js +5 -2
  39. package/dist/src/cohort-topic/peer-codec.js.map +1 -1
  40. package/dist/src/cohort-topic/stream-util.d.ts +24 -3
  41. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  42. package/dist/src/cohort-topic/stream-util.js +36 -3
  43. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  44. package/dist/src/cohort-topic/topic-router.d.ts +9 -0
  45. package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
  46. package/dist/src/cohort-topic/topic-router.js +12 -2
  47. package/dist/src/cohort-topic/topic-router.js.map +1 -1
  48. package/dist/src/dispute/client.d.ts.map +1 -1
  49. package/dist/src/dispute/client.js +15 -3
  50. package/dist/src/dispute/client.js.map +1 -1
  51. package/dist/src/index.d.ts +5 -0
  52. package/dist/src/index.d.ts.map +1 -1
  53. package/dist/src/index.js +5 -0
  54. package/dist/src/index.js.map +1 -1
  55. package/dist/src/libp2p-key-network.d.ts +78 -17
  56. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  57. package/dist/src/libp2p-key-network.js +241 -147
  58. package/dist/src/libp2p-key-network.js.map +1 -1
  59. package/dist/src/libp2p-node-base.d.ts +35 -1
  60. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  61. package/dist/src/libp2p-node-base.js +151 -26
  62. package/dist/src/libp2p-node-base.js.map +1 -1
  63. package/dist/src/libp2p-node-rn.d.ts +2 -2
  64. package/dist/src/libp2p-node-rn.d.ts.map +1 -1
  65. package/dist/src/libp2p-node-rn.js.map +1 -1
  66. package/dist/src/libp2p-node.d.ts +2 -2
  67. package/dist/src/libp2p-node.d.ts.map +1 -1
  68. package/dist/src/libp2p-node.js.map +1 -1
  69. package/dist/src/logger.d.ts.map +1 -1
  70. package/dist/src/logger.js +19 -7
  71. package/dist/src/logger.js.map +1 -1
  72. package/dist/src/matchmaking/query-transport.d.ts +3 -4
  73. package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
  74. package/dist/src/matchmaking/query-transport.js +9 -9
  75. package/dist/src/matchmaking/query-transport.js.map +1 -1
  76. package/dist/src/network/network-manager-service.d.ts +10 -5
  77. package/dist/src/network/network-manager-service.d.ts.map +1 -1
  78. package/dist/src/network/network-manager-service.js +11 -2
  79. package/dist/src/network/network-manager-service.js.map +1 -1
  80. package/dist/src/network/relay-reservation.d.ts +170 -0
  81. package/dist/src/network/relay-reservation.d.ts.map +1 -0
  82. package/dist/src/network/relay-reservation.js +527 -0
  83. package/dist/src/network/relay-reservation.js.map +1 -0
  84. package/dist/src/pend-validation.d.ts +3 -2
  85. package/dist/src/pend-validation.d.ts.map +1 -1
  86. package/dist/src/pend-validation.js +3 -2
  87. package/dist/src/pend-validation.js.map +1 -1
  88. package/dist/src/protocol-client.d.ts +8 -1
  89. package/dist/src/protocol-client.d.ts.map +1 -1
  90. package/dist/src/protocol-client.js +11 -0
  91. package/dist/src/protocol-client.js.map +1 -1
  92. package/dist/src/reactivity/forwarder-host.d.ts +2 -1
  93. package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
  94. package/dist/src/reactivity/forwarder-host.js +2 -1
  95. package/dist/src/reactivity/forwarder-host.js.map +1 -1
  96. package/dist/src/reactivity/origination-manager.d.ts +3 -3
  97. package/dist/src/reactivity/origination-manager.js +3 -3
  98. package/dist/src/reactivity/recover-transport.d.ts +22 -14
  99. package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
  100. package/dist/src/reactivity/recover-transport.js +32 -19
  101. package/dist/src/reactivity/recover-transport.js.map +1 -1
  102. package/dist/src/reactivity/subscription-manager.d.ts +2 -2
  103. package/dist/src/reactivity/topic-bytes.d.ts +6 -7
  104. package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
  105. package/dist/src/reactivity/topic-bytes.js +8 -9
  106. package/dist/src/reactivity/topic-bytes.js.map +1 -1
  107. package/dist/src/repo/client.d.ts +5 -2
  108. package/dist/src/repo/client.d.ts.map +1 -1
  109. package/dist/src/repo/client.js +49 -40
  110. package/dist/src/repo/client.js.map +1 -1
  111. package/dist/src/repo/cluster-coordinator.d.ts +119 -5
  112. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  113. package/dist/src/repo/cluster-coordinator.js +181 -29
  114. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  115. package/dist/src/repo/coordinator-repo.d.ts +259 -76
  116. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  117. package/dist/src/repo/coordinator-repo.js +716 -328
  118. package/dist/src/repo/coordinator-repo.js.map +1 -1
  119. package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
  120. package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
  121. package/dist/src/repo/i-under-replication-ledger.js +2 -0
  122. package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
  123. package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
  124. package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
  125. package/dist/src/repo/kv-under-replication-ledger.js +287 -0
  126. package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
  127. package/dist/src/repo/responsibility.d.ts +37 -0
  128. package/dist/src/repo/responsibility.d.ts.map +1 -0
  129. package/dist/src/repo/responsibility.js +37 -0
  130. package/dist/src/repo/responsibility.js.map +1 -0
  131. package/dist/src/repo/service.d.ts +44 -15
  132. package/dist/src/repo/service.d.ts.map +1 -1
  133. package/dist/src/repo/service.js +64 -27
  134. package/dist/src/repo/service.js.map +1 -1
  135. package/dist/src/repo/stuck-reservation.d.ts +127 -0
  136. package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
  137. package/dist/src/repo/stuck-reservation.js +179 -0
  138. package/dist/src/repo/stuck-reservation.js.map +1 -0
  139. package/dist/src/repo/under-replication-drain.d.ts +208 -0
  140. package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
  141. package/dist/src/repo/under-replication-drain.js +461 -0
  142. package/dist/src/repo/under-replication-drain.js.map +1 -0
  143. package/dist/src/rn.d.ts +5 -0
  144. package/dist/src/rn.d.ts.map +1 -1
  145. package/dist/src/rn.js +5 -0
  146. package/dist/src/rn.js.map +1 -1
  147. package/dist/src/storage/block-latch.d.ts +3 -2
  148. package/dist/src/storage/block-latch.d.ts.map +1 -1
  149. package/dist/src/storage/block-latch.js +7 -4
  150. package/dist/src/storage/block-latch.js.map +1 -1
  151. package/dist/src/storage/block-storage.d.ts +45 -3
  152. package/dist/src/storage/block-storage.d.ts.map +1 -1
  153. package/dist/src/storage/block-storage.js +156 -9
  154. package/dist/src/storage/block-storage.js.map +1 -1
  155. package/dist/src/storage/i-block-storage.d.ts +57 -5
  156. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  157. package/dist/src/storage/i-block-storage.js.map +1 -1
  158. package/dist/src/storage/pending-claim.d.ts +167 -0
  159. package/dist/src/storage/pending-claim.d.ts.map +1 -0
  160. package/dist/src/storage/pending-claim.js +156 -0
  161. package/dist/src/storage/pending-claim.js.map +1 -0
  162. package/dist/src/storage/raw-store-codec.d.ts +13 -0
  163. package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
  164. package/dist/src/storage/raw-store-codec.js +24 -3
  165. package/dist/src/storage/raw-store-codec.js.map +1 -1
  166. package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
  167. package/dist/src/storage/restoration-coordinator.js +2 -1
  168. package/dist/src/storage/restoration-coordinator.js.map +1 -1
  169. package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
  170. package/dist/src/storage/ring-shift-coordinator.js +3 -3
  171. package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
  172. package/dist/src/storage/storage-repo.d.ts +137 -5
  173. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  174. package/dist/src/storage/storage-repo.js +273 -68
  175. package/dist/src/storage/storage-repo.js.map +1 -1
  176. package/dist/src/storage/struct.d.ts +58 -0
  177. package/dist/src/storage/struct.d.ts.map +1 -1
  178. package/dist/src/testing/mesh-harness.d.ts +92 -11
  179. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  180. package/dist/src/testing/mesh-harness.js +246 -103
  181. package/dist/src/testing/mesh-harness.js.map +1 -1
  182. package/package.json +2 -2
  183. package/readme.md +27 -3
  184. package/src/cluster/block-transfer-service.ts +132 -4
  185. package/src/cluster/block-transfer.ts +79 -84
  186. package/src/cluster/client.ts +115 -119
  187. package/src/cluster/cluster-policy.ts +2 -1
  188. package/src/cluster/cluster-repo.ts +3115 -2671
  189. package/src/cluster/commit-proof.ts +2 -2
  190. package/src/cluster/race-resolution.ts +76 -7
  191. package/src/cluster/rebalance-monitor.ts +677 -539
  192. package/src/cluster/spread-on-churn.ts +34 -54
  193. package/src/cohort-topic/host.ts +8 -6
  194. package/src/cohort-topic/membership-source.ts +1 -1
  195. package/src/cohort-topic/peer-codec.ts +34 -31
  196. package/src/cohort-topic/stream-util.ts +39 -4
  197. package/src/cohort-topic/topic-router.ts +12 -2
  198. package/src/dispute/client.ts +17 -7
  199. package/src/index.ts +64 -59
  200. package/src/libp2p-key-network.ts +1377 -1253
  201. package/src/libp2p-node-base.ts +186 -27
  202. package/src/libp2p-node-rn.ts +31 -30
  203. package/src/libp2p-node.ts +37 -36
  204. package/src/logger.ts +20 -7
  205. package/src/matchmaking/query-transport.ts +9 -9
  206. package/src/network/network-manager-service.ts +16 -5
  207. package/src/network/relay-reservation.ts +636 -0
  208. package/src/pend-validation.ts +3 -2
  209. package/src/protocol-client.ts +13 -1
  210. package/src/reactivity/forwarder-host.ts +2 -1
  211. package/src/reactivity/origination-manager.ts +3 -3
  212. package/src/reactivity/recover-transport.ts +39 -22
  213. package/src/reactivity/subscription-manager.ts +2 -2
  214. package/src/reactivity/topic-bytes.ts +35 -37
  215. package/src/repo/client.ts +172 -173
  216. package/src/repo/cluster-coordinator.ts +1296 -1113
  217. package/src/repo/coordinator-repo.ts +3087 -2687
  218. package/src/repo/i-under-replication-ledger.ts +89 -0
  219. package/src/repo/kv-under-replication-ledger.ts +302 -0
  220. package/src/repo/responsibility.ts +48 -0
  221. package/src/repo/service.ts +76 -39
  222. package/src/repo/stuck-reservation.ts +224 -0
  223. package/src/repo/under-replication-drain.ts +582 -0
  224. package/src/rn.ts +74 -69
  225. package/src/storage/block-latch.ts +7 -4
  226. package/src/storage/block-storage.ts +161 -10
  227. package/src/storage/i-block-storage.ts +60 -5
  228. package/src/storage/pending-claim.ts +200 -0
  229. package/src/storage/raw-store-codec.ts +26 -3
  230. package/src/storage/restoration-coordinator.ts +212 -211
  231. package/src/storage/ring-shift-coordinator.ts +271 -271
  232. package/src/storage/storage-repo.ts +311 -73
  233. package/src/storage/struct.ts +58 -0
  234. package/src/testing/mesh-harness.ts +329 -115
@@ -1,4 +1,4 @@
1
- import { blockIdsForTransforms, isOwnRevision, isConflictFailure, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
1
+ import { blockIdsForTransforms, transformForBlockId, isOwnRevision, isConflictFailure, DEFAULT_SUPER_MAJORITY_THRESHOLD, localDurability } from "@optimystic/db-core";
2
2
  import { computeClusterCommitHash, computeClusterMessageHash, computeClusterPromiseHash, membershipDigest, recordMembershipDigest, clusterVoteSigningPayload, clusterVoteVerificationPayload } from "@optimystic/db-core";
3
3
  import { verifyInvalidationCertificate } from "../dispute/invalidation.js";
4
4
  import { buildCommitCert, invalidationActionId } from "./commit-cert.js";
@@ -10,6 +10,8 @@ import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } fr
10
10
  import { createLogger } from '../logger.js';
11
11
  import { PenaltyReason } from "../reputation/types.js";
12
12
  import { isMissingBaseRevisionFailure } from "../storage/storage-repo.js";
13
+ import { isReservationAgainst, reservationRequestFor, cohortCanMissAPend } from "../storage/pending-claim.js";
14
+ import { StuckReservationTracker } from "../repo/stuck-reservation.js";
13
15
  import { checkPendValidation } from "../pend-validation.js";
14
16
  import { getAffectedBlockIds } from "./record-operations.js";
15
17
  import { operationsConflict, resolveRace } from "./race-resolution.js";
@@ -28,6 +30,19 @@ var TransactionPhase;
28
30
  TransactionPhase[TransactionPhase["ConflictSuperseded"] = 6] = "ConflictSuperseded";
29
31
  TransactionPhase[TransactionPhase["Propagating"] = 7] = "Propagating"; // Transaction is being propagated
30
32
  })(TransactionPhase || (TransactionPhase = {}));
33
+ /** A refusal that judges the record invalid — the permanent kind, and every kind but the pending-conflict one. */
34
+ function invalidVerdict(reason) {
35
+ return { valid: false, kind: 'invalid', reason };
36
+ }
37
+ /**
38
+ * Widen a boolean-plus-prose check into a {@link PromiseVerdict}. Every check written this way judges
39
+ * validity, so its refusal is the permanent kind; only the pending-conflict branch of
40
+ * {@link ClusterMember.validatePendOperations} produces the transient one, and it builds its verdict
41
+ * directly.
42
+ */
43
+ function verdictOf(result) {
44
+ return result.valid ? { valid: true } : invalidVerdict(result.reason);
45
+ }
31
46
  /** Stable reject reason a member emits when a declared peer set fails the membership admission gate. */
32
47
  export const MEMBERSHIP_NOT_ADMITTED = 'membership-not-admitted';
33
48
  /**
@@ -37,6 +52,13 @@ export const MEMBERSHIP_NOT_ADMITTED = 'membership-not-admitted';
37
52
  * so the rejection itself is integrity-protected.
38
53
  */
39
54
  export const CONTENT_DIGEST_MISMATCH = 'content-digest-mismatch';
55
+ /**
56
+ * Stable reject reason a member emits when a commit declares, for a block, a base revision
57
+ * (`CommitRequest.blockDigests[id].baseRev`) other than the one the same action's pend carried to
58
+ * this member (`PendRequest.baseRevs`, kept with the pending record) — see
59
+ * `ClusterMember.validateCommitBaseDeclarations`. Signed like its sibling above.
60
+ */
61
+ export const BASE_DECLARATION_DISAGREES = 'base-declaration-disagrees';
40
62
  /**
41
63
  * The two stable reject reasons a validator-configured member emits from the shared
42
64
  * {@link checkPendValidation}: `PEND_NOT_VALIDATABLE` for a pend carrying no `validation` payload
@@ -46,7 +68,7 @@ export const CONTENT_DIGEST_MISMATCH = 'content-digest-mismatch';
46
68
  */
47
69
  export { PEND_NOT_VALIDATABLE, VALIDATOR_FAULT } from "../pend-validation.js";
48
70
  export function clusterMember(components) {
49
- return new ClusterMember(components.storageRepo, components.peerNetwork, components.peerId, components.privateKey, components.protocolPrefix, components.partitionDetector, components.fretService, components.validator, components.reputation, components.consensusConfig, components.stateStore, components.reconcileBlock, components.onCommitCertificate, components.onInvalidate, components.recomputeArbitratorSet, components.deriveExpectedCluster, components.now);
71
+ return new ClusterMember(components.storageRepo, components.peerNetwork, components.peerId, components.privateKey, components.protocolPrefix, components.partitionDetector, components.fretService, components.validator, components.reputation, components.consensusConfig, components.stateStore, components.reconcileBlock, components.onCommitCertificate, components.onInvalidate, components.recomputeArbitratorSet, components.deriveExpectedCluster, components.now, components.onCommittedHolders);
50
72
  }
51
73
  // How long to keep executed transaction records (10 minutes)
52
74
  const ExecutedTransactionTtlMs = 10 * 60 * 1000;
@@ -90,6 +112,7 @@ export class ClusterMember {
90
112
  onInvalidate;
91
113
  recomputeArbitratorSet;
92
114
  deriveExpectedCluster;
115
+ onCommittedHolders;
93
116
  // Track active transactions by their message hash
94
117
  activeTransactions = new Map();
95
118
  // Track executed consensus transactions to prevent duplicate execution (messageHash -> executedAt timestamp)
@@ -99,13 +122,17 @@ export class ClusterMember {
99
122
  // a pend that every member refused (rival pending action, or the revision already taken) must
100
123
  // reach the writer as a conflict, not a win. Pruned alongside executedTransactions (same TTL).
101
124
  executedPendResults = new Map();
102
- // Local storage's verdict for a COMMIT operation applied during consensus (messageHash ->
103
- // CommitResult). Retained so the coordinator can detect when the ahead-divergence tolerance in
125
+ // This member's POST-RECONCILE durable verdict for a COMMIT operation applied during consensus
126
+ // (messageHash -> CommitResult): whether local storage holds the committed revision under the
127
+ // record's action once the apply — and any behind-reconcile it triggered — has run. Two readers,
128
+ // which must see the same answer: the coordinating node reads its own member's directly
129
+ // (getExecutedCommitResult), every other member's reaches the coordinator on the response record
130
+ // (withOwnApplyOutcome → ClusterRecord.applyOutcomes[peer].commit). Together they feed the
131
+ // durability gate in CoordinatorRepo.commit, which acknowledges a commit only when a majority of
132
+ // the cohort reports holding it — consensus votes were never evidence of storage. A retained
133
+ // refusal also lets the coordinator detect when the ahead-divergence tolerance in
104
134
  // applyConsensusOperation swallowed a refusal whose real cause was a RIVAL action holding the
105
- // requested revision the commit-tier acknowledgement hole: a commit that assembled consensus
106
- // inside every member's signed-but-not-yet-applied window is refused by every member's storage
107
- // at apply, and without this verdict the coordinator fabricates a success no member durably
108
- // stored. Pruned alongside executedTransactions (same TTL).
135
+ // requested revision. Pruned alongside executedTransactions (same TTL).
109
136
  executedCommitResults = new Map();
110
137
  // Conflict-shaped pend refusals this member's storage produced at consensus-apply, keyed by the
111
138
  // refused action's id rather than by messageHash. The messageHash-keyed map above cannot serve the
@@ -120,6 +147,14 @@ export class ClusterMember {
120
147
  // inside the sink); this map only spares redundant work when the same invalidation reaches
121
148
  // consensus twice (rebroadcast / sync) under different message hashes. (-> appliedAt timestamp)
122
149
  appliedInvalidations = new Map();
150
+ /**
151
+ * Per block, what this member's own `held` votes have added up to — the member-side instance of
152
+ * the counter `CoordinatorRepo.noteStuckReservation` keeps from the coordinator's vantage. Fed by
153
+ * every `held` verdict {@link validatePendOperations} casts, so a reservation only THIS member
154
+ * holds (one the coordinator's storage cannot corroborate) is still named, once, where the record
155
+ * lives. Forgotten for a block the moment a vote finds it no longer reserved.
156
+ */
157
+ stuckReservations = new StuckReservationTracker(1000);
123
158
  // Queue of transactions to clean up
124
159
  cleanupQueue = [];
125
160
  // Serialize concurrent updates for the same transaction
@@ -155,7 +190,7 @@ export class ClusterMember {
155
190
  now;
156
191
  constructor(storageRepo, peerNetwork, peerId, privateKey, protocolPrefix,
157
192
  // Reserved for partition-healing (backlog ticket 6.5-partition-healing); held but not yet consumed.
158
- _partitionDetector, fretService, validator, reputation, consensusConfig, stateStore, reconcileBlock, onCommitCertificate, onInvalidate, recomputeArbitratorSet, deriveExpectedCluster, now) {
193
+ _partitionDetector, fretService, validator, reputation, consensusConfig, stateStore, reconcileBlock, onCommitCertificate, onInvalidate, recomputeArbitratorSet, deriveExpectedCluster, now, onCommittedHolders) {
159
194
  this.storageRepo = storageRepo;
160
195
  this.peerNetwork = peerNetwork;
161
196
  this.peerId = peerId;
@@ -170,6 +205,7 @@ export class ClusterMember {
170
205
  this.onInvalidate = onInvalidate;
171
206
  this.recomputeArbitratorSet = recomputeArbitratorSet;
172
207
  this.deriveExpectedCluster = deriveExpectedCluster;
208
+ this.onCommittedHolders = onCommittedHolders;
173
209
  this.now = now ?? (() => Date.now());
174
210
  this.superMajorityThreshold = consensusConfig?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD;
175
211
  this.minAbsoluteClusterSize = consensusConfig?.minAbsoluteClusterSize ?? 3;
@@ -243,13 +279,15 @@ export class ClusterMember {
243
279
  return this.executedPendResults.get(messageHash);
244
280
  }
245
281
  /**
246
- * Commit-shaped sibling of {@link getExecutedPendResult}: local storage's verdict for a commit
247
- * operation applied during consensus, when this member retained one. `CoordinatorRepo.commit`
248
- * consults it after a locally-executed commit-consensus a retained refusal whose cause a local
249
- * re-read confirms as a rival holding the requested revision is returned to the writer as a
250
- * retryable conflict instead of the fabricated success the ahead-divergence tolerance would
251
- * otherwise imply. Same availability caveats as the pend accessor: in-memory only, absent for
252
- * pre-restart applies, pruned on the executed-transaction TTL.
282
+ * Commit-shaped sibling of {@link getExecutedPendResult}: this member's post-reconcile durable
283
+ * verdict for a commit operation applied during consensus, when it retained one `success: true`
284
+ * iff local storage holds the committed revision under the record's action for every block the
285
+ * commit named, after the apply and any behind-reconcile it triggered. `CoordinatorRepo.commit`
286
+ * reads it after a locally-executed commit-consensus, both as this node's own contribution to the
287
+ * durable-holder count its durability gate needs, and for a retained refusal whose cause a
288
+ * local re-read confirms as a rival holding the requested revision — as the trigger for a
289
+ * retryable conflict answer. Same availability caveats as the pend accessor: in-memory only,
290
+ * absent for pre-restart applies, pruned on the executed-transaction TTL.
253
291
  */
254
292
  getExecutedCommitResult(messageHash) {
255
293
  return this.executedCommitResults.get(messageHash);
@@ -371,6 +409,15 @@ export class ClusterMember {
371
409
  });
372
410
  // Our own vote can be terminal (a reject where maxAllowedRejections is 0) or complete
373
411
  // the super-majority — recompute rather than guess which.
412
+ //
413
+ // Deliberately NOT the conflict vote's `shouldPersist = false`, for a `held` vote or
414
+ // any other. A conflict vote clears the record because this member holds the WINNER
415
+ // in `activeTransactions` and persisting the loser would reserve the same blocks a
416
+ // second time. A `held` vote has no such twin: the rival lives in durable storage,
417
+ // not in that table, so this record's entry is the only one and dropping it would
418
+ // only make this member forget a transaction the rest of the cohort may still
419
+ // carry to super-majority. Where one `held` vote IS terminal (a small cohort), the
420
+ // recomputed phase is `ConflictSuperseded`, which clears the record anyway.
374
421
  continue;
375
422
  case TransactionPhase.OurConflictVoteNeeded:
376
423
  currentRecord = await this.handleConflictVoteNeeded(currentRecord, phaseResult.conflictsWith);
@@ -423,7 +470,7 @@ export class ClusterMember {
423
470
  shouldPersist = false;
424
471
  break phaseLoop;
425
472
  case TransactionPhase.Promising:
426
- // We have already voted (approve, reject, or conflict); the record is still
473
+ // We have already voted (approve, reject, conflict, or held); the record is still
427
474
  // collecting promises from the rest of the cohort. Nothing to add — retain the
428
475
  // record only if our vote wasn't a conflict (`shouldPersist` already reflects that).
429
476
  log('cluster-member:phase-promising-waiting', {
@@ -472,21 +519,28 @@ export class ClusterMember {
472
519
  return currentRecord;
473
520
  }
474
521
  /**
475
- * Add this member's own conflict-shaped pend verdict to a record's {@link ClusterRecord.applyOutcomes},
476
- * when storage produced one for this transaction. A success, a bare-reason fault, or no retained
477
- * verdict at all leaves the record untouched the coordinator's rule is "an entry means retry",
478
- * so an entry that does not mean retry must not exist.
522
+ * Add this member's own apply verdicts to a record's {@link ClusterRecord.applyOutcomes}, when
523
+ * storage produced any for this transaction. The two arms follow the rules on
524
+ * {@link MemberApplyOutcome}: the pend arm carries ONLY a conflict-shaped refusal (a success, a
525
+ * bare-reason fault, or no retained verdict leaves it off the coordinator's rule is "an entry
526
+ * means retry", so an entry that does not mean retry must not exist); the commit arm carries the
527
+ * retained durable verdict whatever it says, because the coordinator counts the successes.
479
528
  *
480
529
  * The member never signs this and never writes another peer's entry.
481
530
  */
482
531
  withOwnApplyOutcome(record) {
483
- const verdict = this.executedPendResults.get(record.messageHash);
484
- if (verdict === undefined || verdict.success || !isConflictFailure(verdict)) {
532
+ const pend = this.executedPendResults.get(record.messageHash);
533
+ const commit = this.executedCommitResults.get(record.messageHash);
534
+ const own = {
535
+ ...(pend !== undefined && !pend.success && isConflictFailure(pend) ? { pend } : {}),
536
+ ...(commit !== undefined ? { commit } : {})
537
+ };
538
+ if (Object.keys(own).length === 0) {
485
539
  return record;
486
540
  }
487
541
  return {
488
542
  ...record,
489
- applyOutcomes: { ...record.applyOutcomes, [this.peerId.toString()]: { pend: verdict } }
543
+ applyOutcomes: { ...record.applyOutcomes, [this.peerId.toString()]: own }
490
544
  };
491
545
  }
492
546
  /**
@@ -552,8 +606,9 @@ export class ClusterMember {
552
606
  }
553
607
  /**
554
608
  * Compares existing vs incoming signatures for the same peers.
555
- * If a peer's vote type changed (approve↔reject), that's equivocation:
556
- * report a penalty and keep the first-seen signature.
609
+ * ANY change of vote type is equivocation — the comparison is over `Signature['type']`, so it
610
+ * covers every kind without enumerating them (a new kind joins the check by existing): report a
611
+ * penalty and keep the first-seen signature.
557
612
  * New peers are accepted normally.
558
613
  */
559
614
  detectEquivocation(existing, incoming, phase, messageHash) {
@@ -729,18 +784,20 @@ export class ClusterMember {
729
784
  const superMajority = Math.ceil(peerCount * this.superMajorityThreshold);
730
785
  const maxAllowedRejections = peerCount - superMajority;
731
786
  // Check for rejections — rejected if too many rejections to ever reach super-majority.
732
- // ONLY `reject` votes count here: a `conflict` vote is "not now", never a validity
733
- // judgement, so it must not push a record into the permanent `Rejected` phase.
787
+ // ONLY `reject` votes count here: `conflict` and `held` both mean "not now", never a validity
788
+ // judgement, so neither may push a record into the permanent `Rejected` phase.
734
789
  const rejectedPromises = Object.values(record.promises).filter(s => s.type === 'reject');
735
- const conflictPromises = Object.values(record.promises).filter(s => s.type === 'conflict');
790
+ const retryableRefusals = Object.values(record.promises).filter(s => s.type === 'conflict' || s.type === 'held');
736
791
  const rejectedCommits = Object.values(record.commits).filter(s => s.type === 'reject');
737
792
  if (rejectedPromises.length > maxAllowedRejections || this.hasMajority(rejectedCommits.length, peerCount)) {
738
793
  return { phase: TransactionPhase.Rejected };
739
794
  }
740
- // Conflict votes don't judge validity, but enough of them still make super-majority
795
+ // Retryable refusals don't judge validity, but enough of them still make super-majority
741
796
  // unreachable — a distinct terminal outcome (retryable as a fresh transaction) so logs and
742
- // reputation-adjacent paths keep meaning what they say.
743
- if (conflictPromises.length > 0 && rejectedPromises.length + conflictPromises.length > maxAllowedRejections) {
797
+ // reputation-adjacent paths keep meaning what they say. `held` joins `conflict` here for the
798
+ // same reason `conflict` clears its record: a member that keeps reserving the blocks of a
799
+ // transaction that provably cannot win holds them against the very retry meant to win.
800
+ if (retryableRefusals.length > 0 && rejectedPromises.length + retryableRefusals.length > maxAllowedRejections) {
744
801
  return { phase: TransactionPhase.ConflictSuperseded };
745
802
  }
746
803
  // Check if we need to vote. A lost race is answered with a conflict vote, not silence:
@@ -803,20 +860,17 @@ export class ClusterMember {
803
860
  // peer set is a legitimate cluster it belongs to, and refuses (reject vote) rather than rubber-stamping
804
861
  // a set the coordinator chose (e.g. a self-shrunk minority-partition set). On admission failure we skip
805
862
  // pend validation entirely and emit the membership rejection.
806
- const validationResult = await this.evaluatePromise(record);
807
- const promiseHash = await this.computePromiseHash(record);
808
- const type = validationResult.valid ? 'approve' : 'reject';
809
- const rejectReason = validationResult.valid ? undefined : validationResult.reason;
810
- const sig = await this.signVote(promiseHash, type, rejectReason);
811
- const signature = validationResult.valid
812
- ? { type: 'approve', signature: sig }
813
- : { type: 'reject', signature: sig, rejectReason };
814
- if (!validationResult.valid) {
815
- log('cluster-member:validation-rejected', {
863
+ const verdict = await this.evaluatePromise(record);
864
+ if (!verdict.valid) {
865
+ // Two tags, because the two refusals are two different events for an operator: one says the
866
+ // cohort judged a write invalid, the other says a write queued behind a live reservation.
867
+ log(verdict.kind === 'held' ? 'cluster-member:validation-held' : 'cluster-member:validation-rejected', {
816
868
  messageHash: record.messageHash,
817
- reason: validationResult.reason
869
+ reason: verdict.reason
818
870
  });
819
871
  }
872
+ const promiseHash = await this.computePromiseHash(record);
873
+ const signature = await this.signPromiseVerdict(promiseHash, verdict);
820
874
  return {
821
875
  ...record,
822
876
  promises: {
@@ -825,6 +879,22 @@ export class ClusterMember {
825
879
  }
826
880
  };
827
881
  }
882
+ /**
883
+ * Turn a {@link PromiseVerdict} into this member's signed promise vote — one vote kind per refusal
884
+ * kind. The single place the mapping lives, so a new refusal kind cannot reach the wire as the
885
+ * wrong vote: the coordinator's thresholds read only `Signature.type`, and until `held` existed the
886
+ * transient refusal had nowhere to go but `reject`.
887
+ */
888
+ async signPromiseVerdict(promiseHash, verdict) {
889
+ if (verdict.valid) {
890
+ return { type: 'approve', signature: await this.signVote(promiseHash, 'approve') };
891
+ }
892
+ if (verdict.kind === 'held') {
893
+ return { type: 'held', signature: await this.signVote(promiseHash, 'held', verdict.heldBy), heldBy: verdict.heldBy };
894
+ }
895
+ const rejectReason = verdict.reason;
896
+ return { type: 'reject', signature: await this.signVote(promiseHash, 'reject', rejectReason), rejectReason };
897
+ }
828
898
  /**
829
899
  * Answer a record that lost the deterministic race to a transaction this member already holds
830
900
  * (`docs/correctness.md` Theorems 1 & 9: the loser is TOLD it lost, not ignored — an unanswered
@@ -852,8 +922,10 @@ export class ClusterMember {
852
922
  }
853
923
  /**
854
924
  * The full promise-phase decision for a record: admit the declared membership FIRST, then (only if
855
- * admitted) validate its pend operations, then its commit operations. Failing any yields a
856
- * `{ valid:false, reason }` the caller turns into a `reject` vote. Keeping the three separate keeps
925
+ * admitted) validate its pend operations, then its commit operations. Failing any yields a refusal
926
+ * the caller turns into a vote — a `reject` for every validity judgement here, and a `held` for the
927
+ * one transient refusal {@link validatePendOperations} can make (see {@link PromiseVerdict}).
928
+ * Keeping the checks separate keeps
857
929
  * the reason strings distinct — a `membership-not-admitted` reject is a different signal (feeds the
858
930
  * dispute path) than a stale-revision / custom-validator reject, which is different again from a
859
931
  * `content-digest-mismatch` (see {@link validateCommitOperations}). A record carries pend OR commit
@@ -862,7 +934,7 @@ export class ClusterMember {
862
934
  async evaluatePromise(record) {
863
935
  const admission = await this.admitMembership(record);
864
936
  if (!admission.admit) {
865
- return { valid: false, reason: admission.reason ?? MEMBERSHIP_NOT_ADMITTED };
937
+ return invalidVerdict(admission.reason ?? MEMBERSHIP_NOT_ADMITTED);
866
938
  }
867
939
  const pendValidation = await this.validatePendOperations(record);
868
940
  if (!pendValidation.valid) {
@@ -874,7 +946,7 @@ export class ClusterMember {
874
946
  // declared one).
875
947
  const commitRevValidation = await this.validateCommitRevisions(record);
876
948
  if (!commitRevValidation.valid) {
877
- return commitRevValidation;
949
+ return verdictOf(commitRevValidation);
878
950
  }
879
951
  // Then our own refusal history: a commit whose pend THIS member refused, where local state
880
952
  // still corroborates the refusal. Runs after the revision check because that one is sharper
@@ -882,9 +954,16 @@ export class ClusterMember {
882
954
  // applied here; this arm covers the window where it has not.
883
955
  const refusedPendValidation = await this.validateCommitAgainstRefusedPend(record);
884
956
  if (!refusedPendValidation.valid) {
885
- return refusedPendValidation;
957
+ return verdictOf(refusedPendValidation);
958
+ }
959
+ // The declared base before the declared content: it needs no materialization, and when the two
960
+ // disagree the digest check would be previewing operations computed against a different base
961
+ // than the one declared — its verdict names the symptom, this one names the cause.
962
+ const baseDeclarationValidation = await this.validateCommitBaseDeclarations(record);
963
+ if (!baseDeclarationValidation.valid) {
964
+ return verdictOf(baseDeclarationValidation);
886
965
  }
887
- return await this.validateCommitOperations(record);
966
+ return verdictOf(await this.validateCommitOperations(record));
888
967
  }
889
968
  /**
890
969
  * Membership admission gate. Decides whether the coordinator-declared peer set (`record.peers`, call it
@@ -1172,9 +1251,19 @@ export class ClusterMember {
1172
1251
  }
1173
1252
  /**
1174
1253
  * Validates pend operations in a cluster record using the transaction validator.
1175
- * Also checks for stale revisions, and for blocks held by a different unresolved pending
1254
+ * Also checks for stale revisions, and for blocks RESERVED by a different unresolved pending
1176
1255
  * action, to prevent consensus on operations that storage would refuse at apply.
1177
1256
  * Returns success if no validator is configured (backwards compatibility).
1257
+ *
1258
+ * Every refusal here is a validity judgement EXCEPT the pending-conflict one, which is transient by
1259
+ * construction and returns the `held` kind — see {@link PromiseVerdict}. Transient means the
1260
+ * reservation it answers is one its holder is still going to commit or cancel: a record claiming
1261
+ * the slot this pend wants, or a later one. A record claiming a slot the collection has already
1262
+ * moved past is NOT a reservation against this pend and is not refused on — see
1263
+ * {@link reservingRivals} — because its holder is never going to remove it: the holder either
1264
+ * committed at that slot on the rest of the cohort (this member missed the commit) or lost the
1265
+ * slot, and in both cases the record would otherwise refuse every later write to the block, from
1266
+ * every writer, for as long as this member lives.
1178
1267
  */
1179
1268
  async validatePendOperations(record) {
1180
1269
  // Find pend operations in the message
@@ -1201,7 +1290,7 @@ export class ClusterMember {
1201
1290
  blockId,
1202
1291
  reason: blockResult.unavailable
1203
1292
  });
1204
- return { valid: false, reason: `block ${blockId} unavailable (${blockResult.unavailable}): cannot verify revision` };
1293
+ return invalidVerdict(`block ${blockId} unavailable (${blockResult.unavailable}): cannot verify revision`);
1205
1294
  }
1206
1295
  const latest = blockResult?.state?.latest;
1207
1296
  if (latest !== undefined && latest.rev >= pendRequest.rev) {
@@ -1224,18 +1313,18 @@ export class ClusterMember {
1224
1313
  // is NOT a StaleFailure producer, so StaleFailure.staleAt does not apply; the
1225
1314
  // coordinator's own local re-read (CoordinatorRepo.classifyStaleRejection)
1226
1315
  // supplies that number when it can confirm the revision itself.
1227
- return { valid: false, reason: `stale revision: block ${blockId} at rev ${latest.rev}, requested rev ${pendRequest.rev}` };
1316
+ return invalidVerdict(`stale revision: block ${blockId} at rev ${latest.rev}, requested rev ${pendRequest.rev}`);
1228
1317
  }
1229
1318
  }
1230
1319
  }
1231
- // Reject a pend whose blocks are held by a DIFFERENT unresolved pending action. This is
1232
- // the durable reservation the in-memory table (`findConflict` / `activeTransactions`)
1320
+ // Refuse a pend whose blocks are RESERVED by a DIFFERENT unresolved pending action. This
1321
+ // is the durable reservation the in-memory table (`findConflict` / `activeTransactions`)
1233
1322
  // cannot provide: that table clears the moment the rival's PEND record reaches
1234
1323
  // consensus, but the rival's storage pending record — written at pend-apply, removed at
1235
1324
  // commit or cancel — spans exactly the pend→commit window in which `latest.rev` has not
1236
1325
  // yet advanced. Storage's own pend would refuse this request at consensus-apply for the
1237
- // same reason (`StorageRepo.pend`'s listPendingTransactions scan); voting reject here
1238
- // moves that verdict into the phase where the cohort aggregates it, so the loser is
1326
+ // same reason (`StorageRepo.pend`'s pending-claim scan, under the same rule); voting
1327
+ // here moves that verdict into the phase where the cohort aggregates it, so the loser is
1239
1328
  // refused with a real answer instead of burning a consensus round it cannot win. A
1240
1329
  // member that has not yet applied the rival's pend has no record and simply abstains
1241
1330
  // from this reason; the apply-time verdict catches that residual — retained locally
@@ -1244,19 +1333,53 @@ export class ClusterMember {
1244
1333
  // member. Self is excluded so a redelivered pend
1245
1334
  // for this same action stays approvable. An unavailable block carries no `pendings` and
1246
1335
  // abstains (the rev branch above already fail-closes when a revision claim is at stake).
1247
- // Reason stays plain prose: it is fed to computeSigningPayload and carried as
1248
- // Signature.rejectReason, exactly like the stale-revision reason above.
1336
+ //
1337
+ // This is the ONE refusal in this method that is not a validity judgement: a RESERVING
1338
+ // rival's record is removed the moment it commits or cancels, so the very same pend
1339
+ // succeeds on retry. It therefore returns the `held` kind, which becomes a `held` vote
1340
+ // the coordinator counts toward neither approvals nor rejections. `heldBy` carries the
1341
+ // first rival as signed structured data; the prose reason names the same one, and stays
1342
+ // prose because it is fed to computeSigningPayload exactly like the reasons above.
1343
+ //
1344
+ // Which rivals reserve is decided by the slot each record claims against what the
1345
+ // incoming writer built on, not by the record's presence (`reservingRivals`): a record
1346
+ // whose slot is at or below the base this pend declares for the block — or, for a pend
1347
+ // naming no base, below the revision it requests — belongs to a commit this member missed
1348
+ // or a race its holder lost, and would never clear on its own; refusing on it wedged the
1349
+ // block for every writer (ticket `a-member-that-missed-a-commit-refuses-every-later-write`).
1350
+ // Approving over it is safe because the incoming writer's operations were computed
1351
+ // against a version of the block that already holds that record's change. This member
1352
+ // comes current when the approved pend's own commit applies here — through
1353
+ // `internalCommit`, or through the behind-reconcile its fork guard triggers. In a cohort
1354
+ // that can reach its promise bar without one member (`cohortCanMissAPend`: four members up
1355
+ // at the default threshold), a record claiming a slot PAST the declared base still
1356
+ // reserves even when the requested revision has moved beyond it: that is a writer that
1357
+ // read the block without the record's change — served by a member that never held the
1358
+ // rival's pend — and admitting it would lose the change. See `isReservationAgainst`.
1249
1359
  for (const blockId of blockIds) {
1250
- const rivals = (blockResults[blockId]?.state?.pendings ?? []).filter(actionId => actionId !== pendRequest.actionId);
1251
- if (rivals.length > 0) {
1252
- log('cluster-member:validation-pending-conflict', {
1253
- messageHash: record.messageHash,
1254
- blockId,
1255
- actionId: pendRequest.actionId,
1256
- rivals
1257
- });
1258
- return { valid: false, reason: `pending conflict: block ${blockId} held by unresolved action(s) ${rivals.join(', ')}` };
1360
+ const rivalIds = (blockResults[blockId]?.state?.pendings ?? []).filter(actionId => actionId !== pendRequest.actionId);
1361
+ const rivals = rivalIds.length === 0 ? [] : await this.reservingRivals(record, blockId, rivalIds, pendRequest);
1362
+ if (rivals.length === 0) {
1363
+ // Not reserved (any more): whatever episode this member was counting on the block
1364
+ // has ended, so a later wedge gets its own count.
1365
+ this.stuckReservations.forget([blockId]);
1366
+ continue;
1259
1367
  }
1368
+ const heldBy = rivals[0];
1369
+ log('cluster-member:validation-pending-conflict', {
1370
+ messageHash: record.messageHash,
1371
+ blockId,
1372
+ actionId: pendRequest.actionId,
1373
+ requestedRev: pendRequest.rev,
1374
+ rivals
1375
+ });
1376
+ this.nameStuckReservation(blockId, rivals, pendRequest.actionId);
1377
+ return {
1378
+ valid: false,
1379
+ kind: 'held',
1380
+ heldBy,
1381
+ reason: `pending conflict: block ${blockId} held by unresolved action(s) ${rivals.join(', ')}`
1382
+ };
1260
1383
  }
1261
1384
  // Re-check the transaction when a validator is configured. The unvalidatable-pend
1262
1385
  // policy and the throwing-validator catch live in the shared `checkPendValidation`,
@@ -1277,12 +1400,94 @@ export class ClusterMember {
1277
1400
  error: event.error
1278
1401
  }));
1279
1402
  if (!validation.valid) {
1280
- return { valid: false, reason: validation.reason };
1403
+ return invalidVerdict(validation.reason);
1281
1404
  }
1282
1405
  }
1283
1406
  }
1284
1407
  return { valid: true };
1285
1408
  }
1409
+ /**
1410
+ * Of the rival pending records `get` listed on `blockId`, the ones that RESERVE the block against
1411
+ * `pendRequest` — see `isReservationAgainst` for the rule, fed the pend's revision and the base it
1412
+ * declares for this block (`reservationRequestFor`, the same reading storage applies at apply).
1413
+ * Asks storage's {@link IPendingClaimReader} for the slot each record claims; a repo without that
1414
+ * capability (a plain `IRepo` mock) or a read that fails degrades to "every rival reserves", which
1415
+ * is the refusal this member cast before claims were recorded — never to silently admitting one. A
1416
+ * rival that `get` listed but that is gone by the time the claims are read has resolved in
1417
+ * between, and is not a rival any more.
1418
+ */
1419
+ async reservingRivals(record, blockId, rivalIds, pendRequest) {
1420
+ const reader = this.storageRepo;
1421
+ if (typeof reader.listPendingClaims !== 'function') {
1422
+ return rivalIds;
1423
+ }
1424
+ let claims;
1425
+ try {
1426
+ claims = await reader.listPendingClaims(blockId);
1427
+ }
1428
+ catch (err) {
1429
+ log('cluster-member:pending-claims-read-error', { messageHash: record.messageHash, blockId, error: err.message });
1430
+ return rivalIds;
1431
+ }
1432
+ const reservation = this.reservationRequestOf(record, pendRequest, blockId);
1433
+ const claimOf = new Map(claims.map(claim => [claim.actionId, claim]));
1434
+ const reserving = [];
1435
+ for (const actionId of rivalIds) {
1436
+ const claim = claimOf.get(actionId);
1437
+ if (claim === undefined)
1438
+ continue;
1439
+ if (isReservationAgainst(claim, reservation)) {
1440
+ reserving.push(actionId);
1441
+ }
1442
+ else {
1443
+ log('cluster-member:validation-pending-superseded', {
1444
+ messageHash: record.messageHash,
1445
+ blockId,
1446
+ actionId: pendRequest.actionId,
1447
+ requestedRev: pendRequest.rev,
1448
+ baseRev: reservation.baseRev,
1449
+ rival: actionId,
1450
+ claimedRev: claim.rev
1451
+ });
1452
+ }
1453
+ }
1454
+ return reserving;
1455
+ }
1456
+ /** `pendRequest`'s {@link ReservationRequest} for `blockId`, logging a base the rule cannot read —
1457
+ * never a refusal: `baseRevs` is untrusted wire data, and a malformed entry only drops that block
1458
+ * back to the revision rule. The base is read only when `record`'s cohort can reach its promise
1459
+ * super-majority without one of its members (`cohortCanMissAPend`); in a cohort that needs every
1460
+ * member, a stray record on one member (a cancel that never reached it) would otherwise refuse
1461
+ * every later writer for good, and no member can have missed the pend the base arm guards against. */
1462
+ reservationRequestOf(record, pendRequest, blockId) {
1463
+ if (!cohortCanMissAPend(Object.keys(record.peers).length, this.superMajorityThreshold)) {
1464
+ return { rev: pendRequest.rev };
1465
+ }
1466
+ const { request, ignoredBase } = reservationRequestFor(pendRequest, blockId, transformForBlockId(pendRequest.transforms, blockId));
1467
+ if (ignoredBase !== undefined) {
1468
+ log('cluster-member:pend-base-ignored', {
1469
+ messageHash: record.messageHash,
1470
+ blockId,
1471
+ actionId: pendRequest.actionId,
1472
+ requestedRev: pendRequest.rev,
1473
+ base: ignoredBase
1474
+ });
1475
+ }
1476
+ return request;
1477
+ }
1478
+ /**
1479
+ * Feed this member's own `held` vote to its {@link stuckReservations} counter and say, once per
1480
+ * episode and in words, when a block is wedged behind a reservation that is not going to clear —
1481
+ * the member-side twin of `CoordinatorRepo.noteStuckReservation`, for the reservations that
1482
+ * node cannot corroborate from its own storage. `peerId` is carried because this logger is not
1483
+ * suffixed with one and several members can share a process.
1484
+ */
1485
+ nameStuckReservation(blockId, holders, refusedActionId) {
1486
+ const { named } = this.stuckReservations.note(new Map([[blockId, holders]]), refusedActionId);
1487
+ for (const episode of named) {
1488
+ log('cluster-member:stuck-reservation', { peerId: this.peerId.toString(), ...episode });
1489
+ }
1490
+ }
1286
1491
  /**
1287
1492
  * Promise-round check that a commit record's requested revision is not already committed HERE
1288
1493
  * under a different action. This is the member-side arm that keeps a DEAD rival's re-broadcast
@@ -1496,6 +1701,113 @@ export class ClusterMember {
1496
1701
  }
1497
1702
  return undefined;
1498
1703
  }
1704
+ /**
1705
+ * Promise-round check that a commit declares, for each block, the base revision the same action's
1706
+ * PEND carried to this member. The author says the base twice — `PendRequest.baseRevs[id]`, kept
1707
+ * with this member's pending record as `PendingClaim.baseRev`, and
1708
+ * `CommitRequest.blockDigests[id].baseRev` — and an honest author says the same thing both times
1709
+ * (`Tracker` pins the base at staging; the pend and the digest both read that pin). When they
1710
+ * differ, this member's record holds operations computed against a base other than the one the
1711
+ * commit is about to be applied as, so vote reject with {@link BASE_DECLARATION_DISAGREES}.
1712
+ * `StorageRepo.internalCommit` refuses the same shape at apply (`guardCommitBase`); refusing here
1713
+ * too puts a signed verdict on the record one round earlier, and when the cohort's records agree
1714
+ * with each other but not with the commit (a writer whose commit contradicts its own pend) the
1715
+ * whole cohort refuses at the vote, where the apply-time refusal alone would have every member
1716
+ * refuse after consensus and reconcile against a revision nobody holds.
1717
+ *
1718
+ * Needs no preview, so it runs on a repo that can read one record's claim
1719
+ * ({@link IPendingClaimReader}`.pendingClaimOf`) whether or not it can materialize. Abstains — votes
1720
+ * as it would without the check — whenever it cannot compare: the commit declares nothing, an entry
1721
+ * is surplus to `blockIds` or carries no numeric `baseRev` (untrusted wire data, same posture as the
1722
+ * digest check), this member holds no record for the action on the block, the record carries no
1723
+ * base (an inserted or deleted block, a base-less sender, or a record written before bases were
1724
+ * kept), the read fails — or this member does not hold the declared base as its latest.
1725
+ *
1726
+ * That last abstain is what keeps an HONEST retry off this reject. The one disagreement an honest
1727
+ * writer can meet here is a stale record from an earlier attempt of a retried action (same action
1728
+ * id) meeting the retry's commit, when the retry's pend — which would have overwritten the record
1729
+ * — never reached this member. Such a member is typically behind: it missed a pend, and the retry
1730
+ * declares a base the rest of the cohort moved to. At three members one reject sinks the commit
1731
+ * record (the default super-majority allows none), where abstaining lets the others commit and
1732
+ * leaves this member to `guardCommitBase`, which refuses to apply the stale record and reconciles.
1733
+ * When this member's latest IS the declared base, the stale operations are exactly what it would
1734
+ * apply over it, and the digest check below rejects the same shape whenever they materialize
1735
+ * differently; rejecting here names the cause instead.
1736
+ */
1737
+ async validateCommitBaseDeclarations(record) {
1738
+ const reader = this.storageRepo;
1739
+ if (typeof reader.pendingClaimOf !== 'function') {
1740
+ return { valid: true };
1741
+ }
1742
+ for (const operation of record.message.operations) {
1743
+ if (!('commit' in operation)) {
1744
+ continue;
1745
+ }
1746
+ const commit = operation.commit;
1747
+ if (!commit.blockDigests) {
1748
+ continue;
1749
+ }
1750
+ const committedIds = new Set(commit.blockIds);
1751
+ for (const [blockId, declared] of Object.entries(commit.blockDigests)) {
1752
+ const declaredBaseRev = declared?.baseRev;
1753
+ if (!committedIds.has(blockId) || typeof declaredBaseRev !== 'number') {
1754
+ continue;
1755
+ }
1756
+ let storedBaseRev;
1757
+ try {
1758
+ storedBaseRev = (await reader.pendingClaimOf(blockId, commit.actionId))?.baseRev;
1759
+ }
1760
+ catch (err) {
1761
+ log('cluster-member:base-declaration-read-error', {
1762
+ messageHash: record.messageHash,
1763
+ blockId,
1764
+ error: err instanceof Error ? err.message : String(err)
1765
+ });
1766
+ continue; // a local read fault is an abstain, never a verdict
1767
+ }
1768
+ if (storedBaseRev === undefined || storedBaseRev === declaredBaseRev) {
1769
+ continue;
1770
+ }
1771
+ const latestRev = await this.latestRevOf(record, blockId);
1772
+ if (latestRev !== declaredBaseRev) {
1773
+ log('cluster-member:base-declaration-disagrees-abstained', {
1774
+ messageHash: record.messageHash,
1775
+ blockId,
1776
+ actionId: commit.actionId,
1777
+ storedBaseRev,
1778
+ declaredBaseRev,
1779
+ latestRev
1780
+ });
1781
+ continue;
1782
+ }
1783
+ log('cluster-member:base-declaration-disagrees', {
1784
+ messageHash: record.messageHash,
1785
+ blockId,
1786
+ actionId: commit.actionId,
1787
+ rev: commit.rev,
1788
+ storedBaseRev,
1789
+ declaredBaseRev
1790
+ });
1791
+ return { valid: false, reason: BASE_DECLARATION_DISAGREES };
1792
+ }
1793
+ }
1794
+ return { valid: true };
1795
+ }
1796
+ /** This member's own committed revision of `blockId` (raw storage, no cluster recursion), or
1797
+ * `undefined` when it holds none or the read fails — a fault reads as "cannot judge". */
1798
+ async latestRevOf(record, blockId) {
1799
+ try {
1800
+ return (await this.storageRepo.get({ blockIds: [blockId] }))[blockId]?.state?.latest?.rev;
1801
+ }
1802
+ catch (err) {
1803
+ log('cluster-member:base-declaration-read-error', {
1804
+ messageHash: record.messageHash,
1805
+ blockId,
1806
+ error: err instanceof Error ? err.message : String(err)
1807
+ });
1808
+ return undefined;
1809
+ }
1810
+ }
1499
1811
  /**
1500
1812
  * Promise-round check of a commit record's declared content digests
1501
1813
  * (`CommitRequest.blockDigests`) against what this member's OWN pended copy of each transform
@@ -1750,6 +2062,11 @@ export class ClusterMember {
1750
2062
  * mid-commit `internalCommit` fault (`success:false` with a bare `reason`, no
1751
2063
  * `missing`) is propagated so {@link handleConsensus} rolls back the executed marker
1752
2064
  * and rethrows — exactly like an unexpected thrown fault.
2065
+ *
2066
+ * Tolerated is not the same as acknowledged. Whatever shape a commit's apply took, this
2067
+ * member retains one post-reconcile durable verdict for it ({@link durableCommitVerdict}) and
2068
+ * reports it to the coordinator, whose durability gate (`CoordinatorRepo.commit`) acknowledges
2069
+ * the writer only when a majority of the cohort reports holding the revision.
1753
2070
  */
1754
2071
  async applyConsensusOperation(record, operation) {
1755
2072
  const messageHash = record.messageHash;
@@ -1832,87 +2149,19 @@ export class ClusterMember {
1832
2149
  membershipVersion: record.membershipVersion
1833
2150
  });
1834
2151
  }
1835
- let result;
1836
- try {
1837
- result = await this.storageRepo.commit(commit, undefined, proof);
1838
- }
1839
- catch (err) {
1840
- // `StorageRepo.commit` throws (rather than returning success:false) when
1841
- // the pending action is missing the canonical "behind" signal: this
1842
- // member reached commit-consensus without the matching pend (cohort drift).
1843
- if (isMissingPendingActionError(err)) {
1844
- log('cluster-member:consensus-commit-diverged', {
1845
- messageHash,
1846
- actionId: commit.actionId,
1847
- divergence: 'behind',
1848
- reason: err.message
1849
- });
1850
- // We hold no revision of these blocks; pull the committed revision from a
1851
- // cohort peer so the block is no longer under-replicated. Best-effort:
1852
- // failures are logged inside, never thrown (a throw would reset the stream).
1853
- await this.reconcileDivergentCommit(record, commit);
1854
- return;
1855
- }
1856
- throw err;
1857
- }
1858
- // Retain the verdict either way (see getExecutedCommitResult): a success confirms local
1859
- // durability, and an ahead-shaped refusal is the only evidence the coordinator has that
1860
- // the tolerance below swallowed a rival's win at the requested revision. The
1861
- // missing-pending throw path above retains nothing — no CommitResult exists there, and
1862
- // the coordinator's fabricated-success fallback plus cohort reconcile is the right shape
1863
- // for a member that is genuinely behind.
1864
- this.executedCommitResults.set(messageHash, result);
1865
- if (!result.success) {
1866
- // success:false is a StaleFailure. `missing` ⇒ ahead/stale divergence
1867
- // (we already hold ≥ this rev): tolerate, do NOT reconcile downward. A
1868
- // missing-base reason ⇒ behind divergence, reconcile (below). Any other bare
1869
- // `reason` with no `missing` ⇒ a genuine internalCommit fault: propagate so
1870
- // handleConsensus rolls back the executed marker and rethrows.
1871
- //
1872
- // NOTE: this 'ahead' tolerance is what turns a rival's commit that somehow reaches
1873
- // consensus into a reported success no member durably stored (consensus without
1874
- // durability — the commit-tier acknowledgement hole). It must stay: a member
1875
- // genuinely ahead of a redelivered/lagging commit is the common, correct case. The
1876
- // guards live UPSTREAM: `validateCommitRevisions` rejects the rival at the promise
1877
- // round, `CoordinatorRepo.commit` returns lost races as retryable conflicts instead
1878
- // of re-driving them, and the verdict retained just above (getExecutedCommitResult)
1879
- // lets the coordinating node convert its OWN member's rival-confirmed refusal into a
1880
- // conflict answer — that last guard is what closes the signed-but-not-yet-applied
1881
- // window, where two commits for one revision both assemble consensus because signing
1882
- // drops each member's reservation before applying advances its storage —
1883
- // `validateCommitAgainstRefusedPend` now refuses to sign that commit in the first
1884
- // place on any member that refused its pend. If consensus-without-durability is ever
1885
- // observed again, look at those guards' remaining abstain residuals (capability-less
1886
- // or history-truncated storage; a member whose refusal has aged out of retention),
1887
- // not at this branch.
1888
- if (result.missing?.length) {
1889
- log('cluster-member:consensus-commit-diverged', {
1890
- messageHash,
1891
- actionId: commit.actionId,
1892
- divergence: 'ahead',
1893
- reason: result.reason,
1894
- hasMissing: true
1895
- });
1896
- return;
1897
- }
1898
- // This member holds no materializable base for one of the blocks, so
1899
- // `StorageRepo.commit` REFUSED rather than record a revision it could never serve.
1900
- // Same "behind" divergence as a missing pend — and the same cure: pull the committed
1901
- // revision from a cohort peer. Reconciling here (after commit released its per-block
1902
- // latches) is what makes refusing safe; fetching inside the commit path would deadlock
1903
- // against the latch `saveReplicatedBlock` needs to persist what it fetched.
1904
- if (isMissingBaseRevisionFailure(result)) {
1905
- log('cluster-member:consensus-commit-diverged', {
1906
- messageHash,
1907
- actionId: commit.actionId,
1908
- divergence: 'behind',
1909
- reason: result.reason
1910
- });
1911
- await this.reconcileDivergentCommit(record, commit);
1912
- return;
1913
- }
1914
- throw new Error(`Consensus commit for action ${commit.actionId} failed: ${result.reason ?? 'unknown reason'}`);
1915
- }
2152
+ const applied = await this.applyCommitToStorage(record, commit, proof);
2153
+ // Retain the POST-RECONCILE durable verdict (see getExecutedCommitResult and
2154
+ // withOwnApplyOutcome): what this member's storage holds NOW, after any behind-reconcile the
2155
+ // apply triggered — not what the first apply attempt said. A member that pulled the
2156
+ // revision from a cohort peer is a durable holder and reports one; a member that could not
2157
+ // is not, whichever shape its refusal took. The coordinator's durability gate counts these
2158
+ // verdicts its own member's through getExecutedCommitResult, every other member's off the
2159
+ // response record and the two readers must see the same answer, hence one verdict,
2160
+ // computed once, retained here for both.
2161
+ const verdict = await this.durableCommitVerdict(commit, applied);
2162
+ this.executedCommitResults.set(messageHash, verdict);
2163
+ if (verdict.success)
2164
+ this.reportCommittedHolders(record, commit);
1916
2165
  return;
1917
2166
  }
1918
2167
  if ('invalidate' in operation) {
@@ -1920,6 +2169,138 @@ export class ClusterMember {
1920
2169
  return;
1921
2170
  }
1922
2171
  }
2172
+ /**
2173
+ * Apply one consensus commit to local storage, tolerating every divergence shape the way the
2174
+ * doc comment on {@link applyConsensusOperation} describes and reconciling the behind ones.
2175
+ * Returns storage's own result — or, for the thrown missing-pend shape, which produces no
2176
+ * `CommitResult` at all, a refusal built from the throw — so {@link durableCommitVerdict} works
2177
+ * from one uniform shape. Genuine faults (a bare-reason returned failure, an unrecognized throw)
2178
+ * propagate so {@link handleConsensus} rolls back the executed marker and rethrows.
2179
+ */
2180
+ async applyCommitToStorage(record, commit, proof) {
2181
+ const messageHash = record.messageHash;
2182
+ let result;
2183
+ try {
2184
+ result = await this.storageRepo.commit(commit, undefined, proof);
2185
+ }
2186
+ catch (err) {
2187
+ // `StorageRepo.commit` throws (rather than returning success:false) when
2188
+ // the pending action is missing — the canonical "behind" signal: this
2189
+ // member reached commit-consensus without the matching pend (cohort drift).
2190
+ if (isMissingPendingActionError(err)) {
2191
+ log('cluster-member:consensus-commit-diverged', {
2192
+ messageHash,
2193
+ actionId: commit.actionId,
2194
+ divergence: 'behind',
2195
+ reason: err.message
2196
+ });
2197
+ // We hold no revision of these blocks; pull the committed revision from a
2198
+ // cohort peer so the block is no longer under-replicated. Best-effort:
2199
+ // failures are logged inside, never thrown (a throw would reset the stream).
2200
+ await this.reconcileDivergentCommit(record, commit);
2201
+ return { success: false, reason: err.message };
2202
+ }
2203
+ throw err;
2204
+ }
2205
+ if (result.success) {
2206
+ return result;
2207
+ }
2208
+ // success:false is a StaleFailure. `missing` ⇒ ahead/stale divergence
2209
+ // (we already hold ≥ this rev): tolerate, do NOT reconcile downward. A
2210
+ // missing-base reason ⇒ behind divergence, reconcile (below). Any other bare
2211
+ // `reason` with no `missing` ⇒ a genuine internalCommit fault: propagate so
2212
+ // handleConsensus rolls back the executed marker and rethrows.
2213
+ //
2214
+ // NOTE: this 'ahead' tolerance is what turns a rival's commit that somehow reaches
2215
+ // consensus into a member-side no-op rather than a stream reset. It must stay: a member
2216
+ // genuinely ahead of a redelivered/lagging commit is the common, correct case. What keeps
2217
+ // the tolerance from turning into an acknowledged write no member stored is downstream:
2218
+ // the durable verdict retained by the caller reports this member as NOT holding the
2219
+ // revision (unless the revision index names this very action — an ahead member that did
2220
+ // land it), and `CoordinatorRepo.commit`'s durability gate acknowledges only a majority of
2221
+ // such reports; a retained refusal whose cause a local re-read confirms as a rival at the
2222
+ // requested revision is answered as a retryable conflict there. Upstream,
2223
+ // `validateCommitRevisions` rejects the rival at the promise round and
2224
+ // `validateCommitAgainstRefusedPend` refuses to sign the commit of a pend this member
2225
+ // refused. If consensus-without-durability is ever observed again, look at the gate's
2226
+ // inputs (a member reporting a durable success it does not hold, or a majority that is
2227
+ // genuinely durable on a forked lineage), not at this branch.
2228
+ if (result.missing?.length) {
2229
+ log('cluster-member:consensus-commit-diverged', {
2230
+ messageHash,
2231
+ actionId: commit.actionId,
2232
+ divergence: 'ahead',
2233
+ reason: result.reason,
2234
+ hasMissing: true
2235
+ });
2236
+ return result;
2237
+ }
2238
+ // This member holds no materializable base for one of the blocks, so
2239
+ // `StorageRepo.commit` REFUSED rather than record a revision it could never serve.
2240
+ // Same "behind" divergence as a missing pend — and the same cure: pull the committed
2241
+ // revision from a cohort peer. Reconciling here (after commit released its per-block
2242
+ // latches) is what makes refusing safe; fetching inside the commit path would deadlock
2243
+ // against the latch `saveReplicatedBlock` needs to persist what it fetched.
2244
+ if (isMissingBaseRevisionFailure(result)) {
2245
+ log('cluster-member:consensus-commit-diverged', {
2246
+ messageHash,
2247
+ actionId: commit.actionId,
2248
+ divergence: 'behind',
2249
+ reason: result.reason
2250
+ });
2251
+ await this.reconcileDivergentCommit(record, commit);
2252
+ return result;
2253
+ }
2254
+ throw new Error(`Consensus commit for action ${commit.actionId} failed: ${result.reason ?? 'unknown reason'}`);
2255
+ }
2256
+ /**
2257
+ * The durable verdict for a consensus commit: `applied` itself when storage landed it (a
2258
+ * success from `StorageRepo.commit` means every block is committed, or already was, under this
2259
+ * action), otherwise a fresh look at what local storage holds NOW — after the reconcile the
2260
+ * refusal triggered. Durable means every block in `commit.blockIds` records `commit.rev` under
2261
+ * `commit.actionId`. A refusal that is not durable is returned as-is, so the coordinating node
2262
+ * can still classify its own member's ahead-shaped refusal against a rival. A read fault counts
2263
+ * as not durable: the gate this feeds fails toward retry.
2264
+ */
2265
+ async durableCommitVerdict(commit, applied) {
2266
+ if (applied.success) {
2267
+ return applied;
2268
+ }
2269
+ try {
2270
+ for (const blockId of commit.blockIds) {
2271
+ if (!(await this.holdsCommittedRevision(blockId, commit))) {
2272
+ return applied;
2273
+ }
2274
+ }
2275
+ }
2276
+ catch (err) {
2277
+ log('cluster-member:consensus-commit-durability-read-error', {
2278
+ actionId: commit.actionId,
2279
+ rev: commit.rev,
2280
+ error: err.message
2281
+ });
2282
+ return applied;
2283
+ }
2284
+ log('cluster-member:consensus-commit-durable-after-reconcile', { actionId: commit.actionId, rev: commit.rev });
2285
+ // A member's own verdict about its own storage: `local`. Read by the coordinator for its
2286
+ // `success` flag only (`cohortCommitOutcomes`, `localCommitResult`) — it never becomes the
2287
+ // answer handed to the writer; `CoordinatorRepo.commit` computes the cohort's own on every exit.
2288
+ return { success: true, durability: localDurability() };
2289
+ }
2290
+ /**
2291
+ * True when local storage records `commit.rev` of `blockId` under `commit.actionId`. Read from
2292
+ * the revision index (the {@link IRevisionActionReader} capability) so a member whose `latest` has
2293
+ * since moved past the revision still counts as holding it; a repo without the capability is
2294
+ * judged on `latest` alone.
2295
+ */
2296
+ async holdsCommittedRevision(blockId, commit) {
2297
+ const reader = this.storageRepo;
2298
+ if (typeof reader.getRevisionAction === 'function') {
2299
+ return await reader.getRevisionAction(blockId, commit.rev) === commit.actionId;
2300
+ }
2301
+ const latest = (await this.storageRepo.get({ blockIds: [blockId] }))[blockId]?.state?.latest;
2302
+ return latest?.rev === commit.rev && latest.actionId === commit.actionId;
2303
+ }
1923
2304
  /**
1924
2305
  * Applies a consensus-ordered invalidation on this member: dedup → certificate verification →
1925
2306
  * capture the invalidation's commit cert (for reactivity reuse) → delegate the compensating write +
@@ -2018,6 +2399,29 @@ export class ClusterMember {
2018
2399
  log('cluster-member:commit-cert-sink-error', { actionId, error: err.message });
2019
2400
  }
2020
2401
  }
2402
+ /**
2403
+ * Tell the {@link CommittedHoldersSink} which cohort members hold a commit this member just
2404
+ * durably applied. A member learns nothing about the others' storage at apply time, so the
2405
+ * evidence is the record's approving commit signers: each is a cohort member that signed to apply
2406
+ * this commit. A signer that then failed to apply is wrongly recorded, which only spares it a
2407
+ * rebalance push; its own reconcile, read-repair, and — on the coordinating node — the
2408
+ * coordinator's durability-checked report (which lands after this one and overrides it) still
2409
+ * reach it.
2410
+ */
2411
+ reportCommittedHolders(record, commit) {
2412
+ if (!this.onCommittedHolders) {
2413
+ return;
2414
+ }
2415
+ const holders = Object.entries(record.commits)
2416
+ .filter(([peerId, vote]) => vote.type === 'approve' && peerId in record.peers)
2417
+ .map(([peerId]) => peerId);
2418
+ try {
2419
+ this.onCommittedHolders({ blockIds: commit.blockIds, holders });
2420
+ }
2421
+ catch (err) {
2422
+ log('cluster-member:committed-holders-sink-error', { actionId: commit.actionId, error: err.message });
2423
+ }
2424
+ }
2021
2425
  /**
2022
2426
  * After tolerating a "behind" commit divergence, pull the committed revision of
2023
2427
  * each block from a cohort peer that holds it and restore it locally. Best-effort: