@optimystic/db-p2p 0.24.2 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. package/{readme.md → README.md} +688 -668
  2. package/dist/src/cluster/block-transfer-service.d.ts +133 -10
  3. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  4. package/dist/src/cluster/block-transfer-service.js +155 -9
  5. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  6. package/dist/src/cluster/block-transfer.d.ts +67 -7
  7. package/dist/src/cluster/block-transfer.d.ts.map +1 -1
  8. package/dist/src/cluster/block-transfer.js +127 -24
  9. package/dist/src/cluster/block-transfer.js.map +1 -1
  10. package/dist/src/cluster/certified-claims.d.ts +170 -0
  11. package/dist/src/cluster/certified-claims.d.ts.map +1 -0
  12. package/dist/src/cluster/certified-claims.js +200 -0
  13. package/dist/src/cluster/certified-claims.js.map +1 -0
  14. package/dist/src/cluster/client-signature-verifier.d.ts +23 -0
  15. package/dist/src/cluster/client-signature-verifier.d.ts.map +1 -0
  16. package/dist/src/cluster/client-signature-verifier.js +33 -0
  17. package/dist/src/cluster/client-signature-verifier.js.map +1 -0
  18. package/dist/src/cluster/cluster-policy.d.ts +9 -3
  19. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  20. package/dist/src/cluster/cluster-policy.js +111 -27
  21. package/dist/src/cluster/cluster-policy.js.map +1 -1
  22. package/dist/src/cluster/cluster-repo.d.ts +140 -11
  23. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  24. package/dist/src/cluster/cluster-repo.js +485 -29
  25. package/dist/src/cluster/cluster-repo.js.map +1 -1
  26. package/dist/src/cluster/commit-proof.d.ts +134 -0
  27. package/dist/src/cluster/commit-proof.d.ts.map +1 -0
  28. package/dist/src/cluster/commit-proof.js +263 -0
  29. package/dist/src/cluster/commit-proof.js.map +1 -0
  30. package/dist/src/cluster/quorum-restore.d.ts +95 -8
  31. package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
  32. package/dist/src/cluster/quorum-restore.js +113 -14
  33. package/dist/src/cluster/quorum-restore.js.map +1 -1
  34. package/dist/src/cluster/rebalance-monitor.d.ts +84 -0
  35. package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
  36. package/dist/src/cluster/rebalance-monitor.js +207 -5
  37. package/dist/src/cluster/rebalance-monitor.js.map +1 -1
  38. package/dist/src/cluster/reconcile-block.d.ts +27 -7
  39. package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
  40. package/dist/src/cluster/reconcile-block.js +179 -38
  41. package/dist/src/cluster/reconcile-block.js.map +1 -1
  42. package/dist/src/cluster/service.d.ts +13 -4
  43. package/dist/src/cluster/service.d.ts.map +1 -1
  44. package/dist/src/cluster/service.js +17 -18
  45. package/dist/src/cluster/service.js.map +1 -1
  46. package/dist/src/cluster/spread-on-churn.d.ts +8 -2
  47. package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
  48. package/dist/src/cluster/spread-on-churn.js +7 -7
  49. package/dist/src/cluster/spread-on-churn.js.map +1 -1
  50. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  51. package/dist/src/cohort-topic/host.js +8 -7
  52. package/dist/src/cohort-topic/host.js.map +1 -1
  53. package/dist/src/cohort-topic/stream-util.d.ts +7 -6
  54. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  55. package/dist/src/cohort-topic/stream-util.js +14 -41
  56. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  57. package/dist/src/dispute/cascade.d.ts +7 -8
  58. package/dist/src/dispute/cascade.d.ts.map +1 -1
  59. package/dist/src/dispute/cascade.js +24 -7
  60. package/dist/src/dispute/cascade.js.map +1 -1
  61. package/dist/src/dispute/dispute-service.d.ts +5 -0
  62. package/dist/src/dispute/dispute-service.d.ts.map +1 -1
  63. package/dist/src/dispute/dispute-service.js +13 -3
  64. package/dist/src/dispute/dispute-service.js.map +1 -1
  65. package/dist/src/dispute/invalidation.d.ts +33 -15
  66. package/dist/src/dispute/invalidation.d.ts.map +1 -1
  67. package/dist/src/dispute/invalidation.js +127 -34
  68. package/dist/src/dispute/invalidation.js.map +1 -1
  69. package/dist/src/dispute/service.d.ts.map +1 -1
  70. package/dist/src/dispute/service.js +2 -1
  71. package/dist/src/dispute/service.js.map +1 -1
  72. package/dist/src/index.d.ts +7 -0
  73. package/dist/src/index.d.ts.map +1 -1
  74. package/dist/src/index.js +7 -0
  75. package/dist/src/index.js.map +1 -1
  76. package/dist/src/libp2p-key-network.d.ts +14 -10
  77. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  78. package/dist/src/libp2p-key-network.js +28 -58
  79. package/dist/src/libp2p-key-network.js.map +1 -1
  80. package/dist/src/libp2p-node-base.d.ts +38 -7
  81. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  82. package/dist/src/libp2p-node-base.js +148 -63
  83. package/dist/src/libp2p-node-base.js.map +1 -1
  84. package/dist/src/network/open-protocol-stream.d.ts +56 -0
  85. package/dist/src/network/open-protocol-stream.d.ts.map +1 -0
  86. package/dist/src/network/open-protocol-stream.js +60 -0
  87. package/dist/src/network/open-protocol-stream.js.map +1 -0
  88. package/dist/src/network/register-protocol-handler.d.ts +66 -0
  89. package/dist/src/network/register-protocol-handler.d.ts.map +1 -0
  90. package/dist/src/network/register-protocol-handler.js +51 -0
  91. package/dist/src/network/register-protocol-handler.js.map +1 -0
  92. package/dist/src/peer-address-book.d.ts +60 -2
  93. package/dist/src/peer-address-book.d.ts.map +1 -1
  94. package/dist/src/peer-address-book.js +89 -1
  95. package/dist/src/peer-address-book.js.map +1 -1
  96. package/dist/src/pend-validation.d.ts +65 -0
  97. package/dist/src/pend-validation.d.ts.map +1 -0
  98. package/dist/src/pend-validation.js +69 -0
  99. package/dist/src/pend-validation.js.map +1 -0
  100. package/dist/src/reactivity/notify-transport.d.ts.map +1 -1
  101. package/dist/src/reactivity/notify-transport.js +2 -1
  102. package/dist/src/reactivity/notify-transport.js.map +1 -1
  103. package/dist/src/reactivity/push-state-gossip.d.ts.map +1 -1
  104. package/dist/src/reactivity/push-state-gossip.js +2 -1
  105. package/dist/src/reactivity/push-state-gossip.js.map +1 -1
  106. package/dist/src/repo/cluster-coordinator.d.ts +32 -6
  107. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  108. package/dist/src/repo/cluster-coordinator.js +44 -10
  109. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  110. package/dist/src/repo/coordinator-repo.d.ts +181 -25
  111. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  112. package/dist/src/repo/coordinator-repo.js +650 -62
  113. package/dist/src/repo/coordinator-repo.js.map +1 -1
  114. package/dist/src/repo/served-repo-proxy.d.ts +30 -0
  115. package/dist/src/repo/served-repo-proxy.d.ts.map +1 -0
  116. package/dist/src/repo/served-repo-proxy.js +50 -0
  117. package/dist/src/repo/served-repo-proxy.js.map +1 -0
  118. package/dist/src/repo/service.d.ts +16 -1
  119. package/dist/src/repo/service.d.ts.map +1 -1
  120. package/dist/src/repo/service.js +18 -16
  121. package/dist/src/repo/service.js.map +1 -1
  122. package/dist/src/reputation/types.d.ts +3 -1
  123. package/dist/src/reputation/types.d.ts.map +1 -1
  124. package/dist/src/reputation/types.js +3 -1
  125. package/dist/src/reputation/types.js.map +1 -1
  126. package/dist/src/rn.d.ts +24 -2
  127. package/dist/src/rn.d.ts.map +1 -1
  128. package/dist/src/rn.js +33 -4
  129. package/dist/src/rn.js.map +1 -1
  130. package/dist/src/storage/block-archive.d.ts +153 -0
  131. package/dist/src/storage/block-archive.d.ts.map +1 -0
  132. package/dist/src/storage/block-archive.js +194 -0
  133. package/dist/src/storage/block-archive.js.map +1 -0
  134. package/dist/src/storage/block-latch.d.ts +92 -0
  135. package/dist/src/storage/block-latch.d.ts.map +1 -0
  136. package/dist/src/storage/block-latch.js +132 -0
  137. package/dist/src/storage/block-latch.js.map +1 -0
  138. package/dist/src/storage/block-storage.d.ts +169 -12
  139. package/dist/src/storage/block-storage.d.ts.map +1 -1
  140. package/dist/src/storage/block-storage.js +415 -152
  141. package/dist/src/storage/block-storage.js.map +1 -1
  142. package/dist/src/storage/cached-raw-storage.d.ts +5 -1
  143. package/dist/src/storage/cached-raw-storage.d.ts.map +1 -1
  144. package/dist/src/storage/cached-raw-storage.js +16 -1
  145. package/dist/src/storage/cached-raw-storage.js.map +1 -1
  146. package/dist/src/storage/cached-store-driver.d.ts +14 -4
  147. package/dist/src/storage/cached-store-driver.d.ts.map +1 -1
  148. package/dist/src/storage/cached-store-driver.js +39 -5
  149. package/dist/src/storage/cached-store-driver.js.map +1 -1
  150. package/dist/src/storage/i-block-storage.d.ts +70 -19
  151. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  152. package/dist/src/storage/i-block-storage.js +21 -1
  153. package/dist/src/storage/i-block-storage.js.map +1 -1
  154. package/dist/src/storage/i-raw-storage.d.ts +38 -0
  155. package/dist/src/storage/i-raw-storage.d.ts.map +1 -1
  156. package/dist/src/storage/kv-raw-storage.d.ts +14 -8
  157. package/dist/src/storage/kv-raw-storage.d.ts.map +1 -1
  158. package/dist/src/storage/kv-raw-storage.js +29 -9
  159. package/dist/src/storage/kv-raw-storage.js.map +1 -1
  160. package/dist/src/storage/memory-store-driver.d.ts +4 -1
  161. package/dist/src/storage/memory-store-driver.d.ts.map +1 -1
  162. package/dist/src/storage/memory-store-driver.js +22 -2
  163. package/dist/src/storage/memory-store-driver.js.map +1 -1
  164. package/dist/src/storage/raw-store-codec.js +2 -2
  165. package/dist/src/storage/raw-store-driver.d.ts +44 -3
  166. package/dist/src/storage/raw-store-driver.d.ts.map +1 -1
  167. package/dist/src/storage/shared-cache-pool.d.ts +60 -3
  168. package/dist/src/storage/shared-cache-pool.d.ts.map +1 -1
  169. package/dist/src/storage/shared-cache-pool.js +68 -4
  170. package/dist/src/storage/shared-cache-pool.js.map +1 -1
  171. package/dist/src/storage/storage-repo.d.ts +204 -26
  172. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  173. package/dist/src/storage/storage-repo.js +422 -119
  174. package/dist/src/storage/storage-repo.js.map +1 -1
  175. package/dist/src/storage/store-identity.d.ts +21 -0
  176. package/dist/src/storage/store-identity.d.ts.map +1 -0
  177. package/dist/src/storage/store-identity.js +33 -0
  178. package/dist/src/storage/store-identity.js.map +1 -0
  179. package/dist/src/storage/struct.d.ts +15 -0
  180. package/dist/src/storage/struct.d.ts.map +1 -1
  181. package/dist/src/storage/with-read-cache.d.ts +92 -0
  182. package/dist/src/storage/with-read-cache.d.ts.map +1 -0
  183. package/dist/src/storage/with-read-cache.js +132 -0
  184. package/dist/src/storage/with-read-cache.js.map +1 -0
  185. package/dist/src/sync/service.d.ts +13 -3
  186. package/dist/src/sync/service.d.ts.map +1 -1
  187. package/dist/src/sync/service.js +7 -30
  188. package/dist/src/sync/service.js.map +1 -1
  189. package/dist/src/testing/mesh-harness.d.ts +75 -2
  190. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  191. package/dist/src/testing/mesh-harness.js +193 -81
  192. package/dist/src/testing/mesh-harness.js.map +1 -1
  193. package/dist/src/testing/raw-storage-conformance.d.ts +8 -0
  194. package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
  195. package/dist/src/testing/raw-storage-conformance.js +97 -2
  196. package/dist/src/testing/raw-storage-conformance.js.map +1 -1
  197. package/package.json +5 -6
  198. package/src/cluster/block-transfer-service.ts +528 -312
  199. package/src/cluster/block-transfer.ts +547 -424
  200. package/src/cluster/certified-claims.ts +319 -0
  201. package/src/cluster/client-signature-verifier.ts +34 -0
  202. package/src/cluster/cluster-policy.ts +294 -203
  203. package/src/cluster/cluster-repo.ts +2501 -2034
  204. package/src/cluster/commit-proof.ts +344 -0
  205. package/src/cluster/quorum-restore.ts +348 -223
  206. package/src/cluster/rebalance-monitor.ts +539 -254
  207. package/src/cluster/reconcile-block.ts +404 -203
  208. package/src/cluster/service.ts +23 -20
  209. package/src/cluster/spread-on-churn.ts +352 -346
  210. package/src/cohort-topic/host.ts +8 -7
  211. package/src/cohort-topic/stream-util.ts +14 -43
  212. package/src/dispute/cascade.ts +31 -15
  213. package/src/dispute/dispute-service.ts +706 -696
  214. package/src/dispute/invalidation.ts +162 -52
  215. package/src/dispute/service.ts +126 -125
  216. package/src/index.ts +7 -0
  217. package/src/libp2p-key-network.ts +1204 -1235
  218. package/src/libp2p-node-base.ts +185 -70
  219. package/src/network/open-protocol-stream.ts +87 -0
  220. package/src/network/register-protocol-handler.ts +85 -0
  221. package/src/peer-address-book.ts +96 -2
  222. package/src/pend-validation.ts +89 -0
  223. package/src/reactivity/notify-transport.ts +2 -1
  224. package/src/reactivity/push-state-gossip.ts +2 -1
  225. package/src/repo/cluster-coordinator.ts +65 -11
  226. package/src/repo/coordinator-repo.ts +729 -71
  227. package/src/repo/served-repo-proxy.ts +55 -0
  228. package/src/repo/service.ts +25 -16
  229. package/src/reputation/types.ts +3 -1
  230. package/src/rn.ts +34 -4
  231. package/src/storage/block-archive.ts +242 -0
  232. package/src/storage/block-latch.ts +144 -0
  233. package/src/storage/block-storage.ts +787 -498
  234. package/src/storage/cached-raw-storage.ts +21 -1
  235. package/src/storage/cached-store-driver.ts +898 -859
  236. package/src/storage/i-block-storage.ts +172 -116
  237. package/src/storage/i-raw-storage.ts +40 -0
  238. package/src/storage/kv-raw-storage.ts +34 -9
  239. package/src/storage/memory-store-driver.ts +26 -2
  240. package/src/storage/raw-store-codec.ts +42 -42
  241. package/src/storage/raw-store-driver.ts +45 -3
  242. package/src/storage/shared-cache-pool.ts +516 -452
  243. package/src/storage/storage-repo.ts +1421 -1014
  244. package/src/storage/store-identity.ts +41 -0
  245. package/src/storage/struct.ts +52 -34
  246. package/src/storage/with-read-cache.ts +182 -0
  247. package/src/sync/service.ts +178 -198
  248. package/src/testing/mesh-harness.ts +524 -344
  249. package/src/testing/raw-storage-conformance.ts +114 -2
@@ -1,2034 +1,2501 @@
1
- import type { IRepo, ClusterRecord, ClusterPeers, Signature, RepoMessage, ITransactionValidator, ClusterConsensusConfig, CommitResult, BlockId, ActionId, ActionRev, CommitRequest, CommitCert, InvalidateRequest } from "@optimystic/db-core";
2
- import type { ICluster } from "@optimystic/db-core";
3
- import type { IPeerNetwork } from "@optimystic/db-core";
4
- import { blockIdsForTransforms, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
5
- import { computeClusterCommitHash, computeClusterMessageHash, computeClusterPromiseHash, membershipDigest, recordMembershipDigest, clampPriority, clusterVoteSigningPayload, clusterVoteVerificationPayload } from "@optimystic/db-core";
6
- import { verifyInvalidationCertificate, type ArbitratorSetRecompute } from "../dispute/invalidation.js";
7
- import { buildCommitCert, invalidationActionId } from "./commit-cert.js";
8
- import { ClusterClient } from "./client.js";
9
- import type { PeerId, PrivateKey } from "@libp2p/interface";
10
- import { peerIdFromString } from "@libp2p/peer-id";
11
- import { publicKeyFromRaw } from "@libp2p/crypto/keys";
12
- import { peerIdBindsPublicKey } from "./peer-key-binding.js";
13
- import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } from 'uint8arrays';
14
- import { createLogger } from '../logger.js'
15
- import type { PartitionDetector } from "./partition-detector.js";
16
- import type { FretService } from "p2p-fret";
17
- import type { IPeerReputation } from "../reputation/types.js";
18
- import { PenaltyReason } from "../reputation/types.js";
19
- import type { ITransactionStateStore } from "./i-transaction-state-store.js";
20
- import { isMissingBaseRevisionFailure } from "../storage/storage-repo.js";
21
- import { RECONCILE_TIMEOUT_MS } from "./reconcile-block.js";
22
-
23
- const log = createLogger('cluster-member')
24
-
25
- /** State of a transaction in the cluster */
26
- enum TransactionPhase {
27
- Promising, // We have voted; still collecting promises from other peers
28
- OurPromiseNeeded, // We need to provide our promise
29
- OurConflictVoteNeeded, // We hold a conflicting race winner; we must answer with a conflict vote
30
- OurCommitNeeded, // We need to provide our commit
31
- Consensus, // Transaction has reached consensus
32
- Rejected, // Transaction was rejected (validity judgement — enough reject votes)
33
- ConflictSuperseded, // Terminal but retryable: conflict votes make super-majority unreachable
34
- Propagating // Transaction is being propagated
35
- }
36
-
37
- /**
38
- * A phase plus the data its handler needs. Only {@link TransactionPhase.OurConflictVoteNeeded}
39
- * carries any — `conflictsWith`, the winning rival's messageHash — so the conflict-vote handler need
40
- * not re-run conflict detection (whose race resolution has side effects) to learn what blocked it.
41
- * Split by phase rather than an optional field so the handler reads it without an assertion, and so
42
- * a future phase-with-data cannot silently inherit this one's payload.
43
- */
44
- type PhaseResult =
45
- | { phase: TransactionPhase.OurConflictVoteNeeded; conflictsWith: string }
46
- | { phase: Exclude<TransactionPhase, TransactionPhase.OurConflictVoteNeeded> };
47
-
48
- interface TransactionState {
49
- record: ClusterRecord;
50
- promiseTimeout?: NodeJS.Timeout;
51
- resolutionTimeout?: NodeJS.Timeout;
52
- lastUpdate: number;
53
- }
54
-
55
- /**
56
- * Result of verifying one vote signature. `penalize` distinguishes "identity was never proven"
57
- * (no key / not Ed25519 / key not bound to the peer id / malformed input — reject, but do NOT
58
- * report the named peer, whose id may have been attacker-chosen) from "the key IS the one the
59
- * peer id names, yet the signature does not verify" (reject AND reporta genuine bad vote from
60
- * that proven identity). Collapsing both into a bare `false` would let a coordinator get an honest
61
- * peer penalized just by attaching a key it controls under that peer's id.
62
- */
63
- type VerifyOutcome =
64
- | { valid: true }
65
- | { valid: false; penalize: boolean };
66
-
67
- /**
68
- * Actively reconciles a block this member committed without having seen the matching
69
- * pend (cohort drift between the independent pend and commit cluster-transactions).
70
- * Pulls the committed revision from a cohort peer that holds it and restores it into
71
- * local storage. Injected so {@link ClusterMember} stays transport-agnostic mirrors
72
- * how `CoordinatorRepo` receives its `clusterLatestCallback`.
73
- *
74
- * @param blockId the under-replicated block to restore
75
- * @param committed the committed `(actionId, rev)` agreed by consensus
76
- * @param cohortPeerIds cohort members to pull from (self already excluded)
77
- */
78
- export type ReconcileBlockCallback = (blockId: BlockId, committed: ActionRev, cohortPeerIds: string[]) => Promise<void>;
79
-
80
- /**
81
- * Sink for the {@link CommitCert} this member assembled at consensus, fired once per committed action
82
- * **before** the commit is applied to local storage (so it is already retained when
83
- * {@link StorageRepo.commit} emits the matching `CollectionChangeEvent` the reactivity bridge reads).
84
- * The bytes are the cluster's own `approve` commit signatures, forwarded UNCHANGED never re-signed.
85
- * Optional; absent on nodes that do not originate reactivity notifications. A throwing sink is
86
- * isolated + logged (it must never break consensus).
87
- */
88
- export type CommitCertificateSink = (actionId: ActionId, cert: CommitCert) => void;
89
-
90
- /**
91
- * Applies a consensus-ordered {@link InvalidateRequest} to local storage — the deterministic
92
- * reversal every member runs once consensus on the invalidation is reached. The implementation
93
- * recomputes the per-block as-if-`T_inv`-absent content, writes the compensating revisions, and
94
- * appends the durable invalidation log entry (see `applyInvalidation` in the dispute module). It
95
- * is injected so {@link ClusterMember} stays storage/log-agnostic (mirrors {@link ReconcileBlockCallback}).
96
- *
97
- * The certificate is verified by {@link ClusterMember} *before* the sink is invoked, so a sink
98
- * implementation may assume `request.resolution` is already a valid challenger-wins certificate.
99
- * A throwing sink is logged and tolerated (never resets the cluster stream); the in-memory dedup
100
- * marker is rolled back so a re-broadcast can retry.
101
- */
102
- export type InvalidationApplySink = (request: InvalidateRequest) => Promise<void>;
103
-
104
- /**
105
- * Optional **layer-2** capability for invalidation-certificate verification: re-derives the
106
- * legitimately-selected arbitrator set from this member's topology view and judges the carried set
107
- * (see {@link ArbitratorSetRecompute}). Injected so {@link ClusterMember} stays network-agnostic; the
108
- * composition root supplies it from FRET when available. When absent (or when the member cannot
109
- * reconstruct the historical topology), {@link applyConsensusInvalidation} accepts a layer-1-valid
110
- * certificate and logs that it applied an invalidation it could not fully anchor the documented
111
- * interim posture (see `tickets/plan/cohort-topic-membership-cert-trust-anchoring.md`).
112
- */
113
- export type RecomputeArbitratorSetCapability = ArbitratorSetRecompute;
114
-
115
- /**
116
- * The member's own independently-derived view of a block's responsible cluster, plus FRET's confidence in
117
- * the underlying network-size estimate (0..1). Returned by {@link DeriveExpectedClusterCallback} and
118
- * consumed by {@link ClusterMember}'s membership admission gate: the member checks a coordinator-declared
119
- * peer set against `peers` (its expected set `E`) and gates on `confidence` (low confidence ⇒ fail closed
120
- * for any downsizing, the partition posture).
121
- */
122
- export type ExpectedClusterView = {
123
- /** The member's own derived responsible-peer set for the block (its view of the legitimate cluster). */
124
- peers: ClusterPeers;
125
- /** FRET's confidence in the current network-size estimate (0..1); threshold treated as untrusted. */
126
- confidence: number;
127
- };
128
-
129
- /**
130
- * Independently derive this member's own view of a block's responsible cluster. Injected so
131
- * {@link ClusterMember} stays transport-agnostic — the composition root supplies it from
132
- * `IKeyNetwork.findCluster` + FRET (mirroring how the coordinator derives the cluster). Absent on nodes
133
- * that cannot derive a view (no FRET, unit tests): with no derived view AND no asserted
134
- * {@link ClusterConsensusConfig.assumedClusterSize} the gate preserves legacy approve behavior, but an
135
- * asserted size still lets the gate fail closed on an unjustified downsize. See {@link ClusterMember}
136
- * admission gate.
137
- */
138
- export type DeriveExpectedClusterCallback = (blockId: BlockId) => Promise<ExpectedClusterView>;
139
-
140
- /** Stable reject reason a member emits when a declared peer set fails the membership admission gate. */
141
- export const MEMBERSHIP_NOT_ADMITTED = 'membership-not-admitted';
142
-
143
- interface ClusterMemberComponents {
144
- storageRepo: IRepo;
145
- peerNetwork: IPeerNetwork;
146
- peerId: PeerId;
147
- privateKey: PrivateKey;
148
- protocolPrefix?: string;
149
- // Reserved for partition-healing consumers (backlog ticket 6.5-partition-healing); not yet read by ClusterMember.
150
- partitionDetector?: PartitionDetector;
151
- fretService?: FretService;
152
- validator?: ITransactionValidator;
153
- reputation?: IPeerReputation;
154
- consensusConfig?: ClusterConsensusConfig;
155
- stateStore?: ITransactionStateStore;
156
- /** Restores a block under-replicated by cohort drift; see {@link ReconcileBlockCallback}. */
157
- reconcileBlock?: ReconcileBlockCallback;
158
- /** Receives the consensus commit cert per committed action; see {@link CommitCertificateSink}. */
159
- onCommitCertificate?: CommitCertificateSink;
160
- /** Applies a consensus-ordered invalidation to local storage; see {@link InvalidationApplySink}. */
161
- onInvalidate?: InvalidationApplySink;
162
- /** Layer-2 arbitrator-set recompute for invalidation verification; see {@link RecomputeArbitratorSetCapability}. */
163
- recomputeArbitratorSet?: RecomputeArbitratorSetCapability;
164
- /** Member-side cluster derivation for the membership admission gate; see {@link DeriveExpectedClusterCallback}. */
165
- deriveExpectedCluster?: DeriveExpectedClusterCallback;
166
- }
167
-
168
- export function clusterMember(components: ClusterMemberComponents): ClusterMember {
169
- return new ClusterMember(
170
- components.storageRepo,
171
- components.peerNetwork,
172
- components.peerId,
173
- components.privateKey,
174
- components.protocolPrefix,
175
- components.partitionDetector,
176
- components.fretService,
177
- components.validator,
178
- components.reputation,
179
- components.consensusConfig,
180
- components.stateStore,
181
- components.reconcileBlock,
182
- components.onCommitCertificate,
183
- components.onInvalidate,
184
- components.recomputeArbitratorSet,
185
- components.deriveExpectedCluster
186
- );
187
- }
188
-
189
- // How long to keep executed transaction records (10 minutes)
190
- const ExecutedTransactionTtlMs = 10 * 60 * 1000;
191
-
192
- // Upper bound on an awaited active reconciliation of a divergent commit. Bounds the
193
- // consensus path so a slow/unreachable cohort peer can't stall the cluster stream;
194
- // a timeout is logged and tolerated (never thrown — that would reset the stream).
195
- // Shared with the read path's acquisition (see RECONCILE_TIMEOUT_MS) — same operation, same bound.
196
- const ReconcileTimeoutMs = RECONCILE_TIMEOUT_MS;
197
-
198
- /**
199
- * True when a thrown storage error reports a missing pending action — i.e. this
200
- * member reached commit-consensus without having seen the matching pend phase.
201
- * That is recoverable local divergence (reconciled via sync / read-repair), not
202
- * a transaction fault, so it must not reset the cluster stream.
203
- */
204
- function isMissingPendingActionError(err: unknown): boolean {
205
- return err instanceof Error && /pending action .+ not found/i.test(err.message);
206
- }
207
-
208
- /**
209
- * Handles cluster-side operations, managing promises and commits for cluster updates
210
- * and coordinating with the local storage repo.
211
- */
212
- export class ClusterMember implements ICluster {
213
- // Track active transactions by their message hash
214
- private activeTransactions: Map<string, TransactionState> = new Map();
215
- // Track executed consensus transactions to prevent duplicate execution (messageHash -> executedAt timestamp)
216
- private executedTransactions: Map<string, number> = new Map();
217
- // Fast in-memory dedup for applied invalidations, keyed `${invalidatedActionId}:${disputeId}`.
218
- // The durable source of truth is the invalidation log entry (Log.findInvalidation, re-checked
219
- // inside the sink); this map only spares redundant work when the same invalidation reaches
220
- // consensus twice (rebroadcast / sync) under different message hashes. (-> appliedAt timestamp)
221
- private appliedInvalidations: Map<string, number> = new Map();
222
- // Queue of transactions to clean up
223
- private cleanupQueue: string[] = [];
224
- // Serialize concurrent updates for the same transaction
225
- private pendingUpdates: Map<string, Promise<ClusterRecord>> = new Map();
226
- // Interval handles for periodic cleanup (stored so dispose() can clear them)
227
- private readonly expirationInterval: NodeJS.Timeout;
228
- private readonly cleanupInterval: NodeJS.Timeout;
229
-
230
- /**
231
- * Confidence floor at/below which FRET's network-size view is treated as untrustworthy for the
232
- * membership gate. Above it the member trusts its derived view (confident path); at/below it the gate
233
- * fails closed for downsizing. Matches the coordinator's `validateSmallCluster` confidence gate (> 0.5).
234
- */
235
- private static readonly MembershipConfidenceThreshold = 0.5;
236
-
237
- /**
238
- * Effective super-majority threshold this member accepts as sufficient for a commit. Defaults to
239
- * {@link DEFAULT_SUPER_MAJORITY_THRESHOLD} (0.75) when no config is supplied the SAME default the
240
- * coordinator uses, so the two can never silently disagree about whether a transaction is final.
241
- * (Previously defaulted to 1.0/unanimity, which split the member from a coordinator committing at 0.75.)
242
- */
243
- private readonly superMajorityThreshold: number;
244
- // Membership admission gate parameters (see {@link admitMembership}). Read once from consensusConfig
245
- // so the gate has stable thresholds independent of the (untrusted) values a record declares.
246
- private readonly minAbsoluteClusterSize: number;
247
- private readonly clusterSizeTolerance: number;
248
- private readonly membershipAdmissionFraction: number;
249
- /** Operator-asserted smallest genuine cohort size, or undefined when unknown. */
250
- private readonly assumedClusterSize: number | undefined;
251
- private readonly allowUnvalidatedSmallCluster: boolean;
252
-
253
- constructor(
254
- private readonly storageRepo: IRepo,
255
- private readonly peerNetwork: IPeerNetwork,
256
- private readonly peerId: PeerId,
257
- private readonly privateKey: PrivateKey,
258
- private readonly protocolPrefix?: string,
259
- // Reserved for partition-healing (backlog ticket 6.5-partition-healing); held but not yet consumed.
260
- _partitionDetector?: PartitionDetector,
261
- private readonly fretService?: FretService,
262
- private readonly validator?: ITransactionValidator,
263
- private readonly reputation?: IPeerReputation,
264
- consensusConfig?: ClusterConsensusConfig,
265
- private readonly stateStore?: ITransactionStateStore,
266
- private readonly reconcileBlock?: ReconcileBlockCallback,
267
- private readonly onCommitCertificate?: CommitCertificateSink,
268
- private readonly onInvalidate?: InvalidationApplySink,
269
- private readonly recomputeArbitratorSet?: RecomputeArbitratorSetCapability,
270
- private readonly deriveExpectedCluster?: DeriveExpectedClusterCallback
271
- ) {
272
- this.superMajorityThreshold = consensusConfig?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD;
273
- this.minAbsoluteClusterSize = consensusConfig?.minAbsoluteClusterSize ?? 3;
274
- this.clusterSizeTolerance = consensusConfig?.clusterSizeTolerance ?? 0.5;
275
- this.membershipAdmissionFraction = consensusConfig?.membershipAdmissionFraction ?? 0.75;
276
- this.assumedClusterSize = consensusConfig?.assumedClusterSize;
277
- this.allowUnvalidatedSmallCluster = consensusConfig?.allowUnvalidatedSmallCluster ?? false;
278
- // State the resolved gate parameters once, so an operator diagnosing a membership rejection can see
279
- // what this node actually resolved. A fact, not a warning: `assumedClusterSize < clusterSize` is the
280
- // normal default state, so warning on it would fire for every node and be ignored.
281
- log('cluster-member:admission-config', {
282
- assumedClusterSize: this.assumedClusterSize,
283
- minAbsoluteClusterSize: this.minAbsoluteClusterSize,
284
- membershipAdmissionFraction: this.membershipAdmissionFraction,
285
- allowUnvalidatedSmallCluster: this.allowUnvalidatedSmallCluster
286
- });
287
- // Periodically clean up expired transactions (.unref() so tests/short-lived processes can exit)
288
- this.expirationInterval = setInterval(() => this.queueExpiredTransactions(), 60000);
289
- this.expirationInterval.unref();
290
- // Process cleanup queue
291
- this.cleanupInterval = setInterval(() => this.processCleanupQueue(), 1000);
292
- this.cleanupInterval.unref();
293
- }
294
-
295
- /**
296
- * The resolved super-majority threshold this member runs on. Exposed so the composition root can
297
- * fail-fast if the member and the coordinator would run different thresholds (see the coupling
298
- * assertion in `libp2p-node-base.ts`) — a mismatch is a latent phase-disagreement, caught at startup
299
- * rather than mid-consensus.
300
- */
301
- get effectiveSuperMajorityThreshold(): number {
302
- return this.superMajorityThreshold;
303
- }
304
-
305
- /**
306
- * Clears all interval and timeout handles and empties active state.
307
- * Called during node shutdown to prevent leaked timers.
308
- */
309
- dispose(): void {
310
- clearInterval(this.expirationInterval);
311
- clearInterval(this.cleanupInterval);
312
- for (const [, state] of this.activeTransactions) {
313
- if (state.promiseTimeout) clearTimeout(state.promiseTimeout);
314
- if (state.resolutionTimeout) clearTimeout(state.resolutionTimeout);
315
- }
316
- this.activeTransactions.clear();
317
- this.cleanupQueue.length = 0;
318
- }
319
-
320
- /**
321
- * Checks if a transaction's operations were already executed during consensus.
322
- * Used by the coordinator to avoid duplicate execution in CoordinatorRepo.
323
- */
324
- wasTransactionExecuted(messageHash: string): boolean {
325
- return this.executedTransactions.has(messageHash);
326
- }
327
-
328
- /**
329
- * Handles an incoming cluster update, managing the two-phase commit process
330
- * and coordinating with the local storage repo
331
- */
332
- async update(record: ClusterRecord): Promise<ClusterRecord> {
333
- // Serialize concurrent updates for the same transaction
334
- const existingUpdate = this.pendingUpdates.get(record.messageHash);
335
- if (existingUpdate) {
336
- log('cluster-member:concurrent-update-wait', { messageHash: record.messageHash });
337
- await existingUpdate;
338
- // After waiting, continue processing with the new incoming record
339
- // to ensure proper merging of promises/commits from coordinator
340
- }
341
-
342
- // Create a promise for this update operation
343
- const updatePromise = this.processUpdate(record);
344
- this.pendingUpdates.set(record.messageHash, updatePromise);
345
-
346
- try {
347
- const result = await updatePromise;
348
- return result;
349
- } finally {
350
- // Remove from pending updates after a short delay to allow concurrent calls to see it
351
- setTimeout(() => {
352
- this.pendingUpdates.delete(record.messageHash);
353
- }, 100).unref();
354
- }
355
- }
356
-
357
- private async processUpdate(record: ClusterRecord): Promise<ClusterRecord> {
358
- const ourId = this.peerId.toString();
359
- const inboundPhase = record.commits[ourId] ? 'commit' : record.promises[ourId] ? 'promise' : 'initial';
360
- log('cluster-member:incoming', {
361
- messageHash: record.messageHash,
362
- phase: inboundPhase,
363
- peerCount: Object.keys(record.peers).length,
364
- promiseCount: Object.keys(record.promises).length,
365
- commitCount: Object.keys(record.commits).length,
366
- existingTransaction: this.activeTransactions.has(record.messageHash)
367
- });
368
-
369
- // Report network size hint to FRET if provided
370
- if (this.fretService && record.networkSizeHint && record.networkSizeConfidence) {
371
- try {
372
- this.fretService.reportNetworkSize(
373
- record.networkSizeHint,
374
- record.networkSizeConfidence,
375
- 'cluster'
376
- );
377
- } catch (err) {
378
- // Ignore errors reporting to FRET
379
- }
380
- }
381
-
382
- // Validate the incoming record
383
- await this.validateRecord(record);
384
-
385
- const existingState = this.activeTransactions.get(record.messageHash);
386
- let currentRecord = existingState?.record || record;
387
- if (existingState) {
388
- log('cluster-member:merge-start', {
389
- messageHash: record.messageHash,
390
- existingPromises: Object.keys(existingState.record.promises ?? {}),
391
- existingCommits: Object.keys(existingState.record.commits ?? {}),
392
- incomingPromises: Object.keys(record.promises ?? {}),
393
- incomingCommits: Object.keys(record.commits ?? {})
394
- });
395
- }
396
-
397
- // If we have an existing record, merge the signatures
398
- if (existingState) {
399
- currentRecord = await this.mergeRecords(existingState.record, record);
400
- log('cluster-member:merge-complete', {
401
- messageHash: record.messageHash,
402
- mergedPromises: Object.keys(currentRecord.promises ?? {}),
403
- mergedCommits: Object.keys(currentRecord.commits ?? {})
404
- });
405
- }
406
-
407
- // Drive the phase machine to a FIXPOINT rather than handling one phase per delivery. Each
408
- // vote this member adds can put the record straight into the next phase (our promise
409
- // completes super-majority ⇒ our commit is due; our commit completes the majority ⇒
410
- // consensus; our reject/conflict vote makes the record terminal), and any follow-on phase
411
- // not re-handled here would silently wait for the coordinator's next delivery — e.g. a
412
- // member whose promise the coordinator never collected receives the commit-phase record,
413
- // adds its promise, and must then also commit in the SAME delivery. One loop replaces the
414
- // hand-written per-branch re-checks that used to cover only the follow-ons their authors
415
- // thought of. Only the three vote-adding phases continue; each strictly grows the record
416
- // (adds a promise or commit key), so the loop terminates — the cap guards a
417
- // phase-computation bug, not a real bound.
418
- const MaxPhaseSteps = 8;
419
- let shouldPersist = true;
420
- phaseLoop: for (let step = 0; ; step++) {
421
- if (step >= MaxPhaseSteps) {
422
- log('cluster-member:phase-loop-overflow', { messageHash: record.messageHash, steps: step });
423
- break;
424
- }
425
- const phaseResult = await this.getTransactionPhase(currentRecord);
426
- log('cluster-member:phase', {
427
- messageHash: record.messageHash,
428
- phase: phaseResult.phase,
429
- step,
430
- promises: Object.keys(currentRecord.promises ?? {}),
431
- commits: Object.keys(currentRecord.commits ?? {})
432
- });
433
- switch (phaseResult.phase) {
434
- case TransactionPhase.OurPromiseNeeded:
435
- log('cluster-member:action-promise', {
436
- messageHash: record.messageHash
437
- });
438
- currentRecord = await this.handlePromiseNeeded(currentRecord);
439
- log('cluster-member:action-promise-complete', {
440
- messageHash: record.messageHash,
441
- promises: Object.keys(currentRecord.promises ?? {})
442
- });
443
- // Our own vote can be terminal (a reject where maxAllowedRejections is 0) or complete
444
- // the super-majority — recompute rather than guess which.
445
- continue;
446
- case TransactionPhase.OurConflictVoteNeeded:
447
- currentRecord = await this.handleConflictVoteNeeded(currentRecord, phaseResult.conflictsWith);
448
- // Never persist a record we conflict-voted: this member holds the WINNER, and
449
- // persisting the loser would reserve the same blocks a second time — half of what
450
- // made the silent-abstention failure self-sustaining.
451
- shouldPersist = false;
452
- continue;
453
- case TransactionPhase.OurCommitNeeded:
454
- log('cluster-member:action-commit', {
455
- messageHash: record.messageHash
456
- });
457
- currentRecord = await this.handleCommitNeeded(currentRecord);
458
- log('cluster-member:action-commit-complete', {
459
- messageHash: record.messageHash,
460
- commits: Object.keys(currentRecord.commits ?? {})
461
- });
462
- shouldPersist = false;
463
- // Our commit may have completed the majority — recompute; Consensus executes below.
464
- continue;
465
- case TransactionPhase.Consensus:
466
- log('cluster-member:action-consensus', {
467
- messageHash: record.messageHash
468
- });
469
- await this.handleConsensus(currentRecord);
470
- shouldPersist = false;
471
- break phaseLoop;
472
- case TransactionPhase.Rejected:
473
- log('cluster-member:action-rejected', {
474
- messageHash: record.messageHash
475
- });
476
- await this.handleRejection(currentRecord);
477
- shouldPersist = false;
478
- break phaseLoop;
479
- case TransactionPhase.ConflictSuperseded:
480
- // Enough conflict votes that super-majority is unreachable. NOT a rejection the
481
- // callers retry it as a fresh transaction — so it gets its own terminal phase and the
482
- // record is cleared rather than held (holding a provably-dead loser would reserve its
483
- // blocks against the very retry that is supposed to win).
484
- log('cluster-member:action-conflict-superseded', {
485
- messageHash: record.messageHash
486
- });
487
- shouldPersist = false;
488
- break phaseLoop;
489
- case TransactionPhase.Propagating:
490
- // Transaction is complete and propagating - clean it up
491
- log('cluster-member:phase-propagating', {
492
- messageHash: record.messageHash
493
- });
494
- shouldPersist = false;
495
- break phaseLoop;
496
- case TransactionPhase.Promising:
497
- // We have already voted (approve, reject, or conflict); the record is still
498
- // collecting promises from the rest of the cohort. Nothing to add — retain the
499
- // record only if our vote wasn't a conflict (`shouldPersist` already reflects that).
500
- log('cluster-member:phase-promising-waiting', {
501
- messageHash: record.messageHash
502
- });
503
- break phaseLoop;
504
- }
505
- }
506
-
507
- if (shouldPersist) {
508
- // Update transaction state
509
- const timeouts = this.setupTimeouts(currentRecord);
510
- this.activeTransactions.set(record.messageHash, {
511
- record: currentRecord,
512
- lastUpdate: Date.now(),
513
- promiseTimeout: timeouts.promiseTimeout,
514
- resolutionTimeout: timeouts.resolutionTimeout
515
- });
516
- this.persistParticipantState(record.messageHash, currentRecord);
517
- log('cluster-member:state-persist', {
518
- messageHash: record.messageHash,
519
- storedPromises: Object.keys(currentRecord.promises ?? {}),
520
- storedCommits: Object.keys(currentRecord.commits ?? {})
521
- });
522
- } else {
523
- log('cluster-member:state-clear', {
524
- messageHash: record.messageHash
525
- });
526
- this.clearTransaction(record.messageHash);
527
- }
528
-
529
- // Skip propagation - the coordinator manages distribution
530
- // await this.propagateIfNeeded(currentRecord);
531
-
532
- log('cluster-member:update-complete', {
533
- messageHash: record.messageHash,
534
- promiseCount: Object.keys(currentRecord.promises).length,
535
- commitCount: Object.keys(currentRecord.commits).length
536
- });
537
- return currentRecord;
538
- }
539
-
540
- /**
541
- * Merges two records, validating that non-signature fields match.
542
- * Detects equivocation (same peer changing vote type) and applies penalties.
543
- */
544
- private async mergeRecords(existing: ClusterRecord, incoming: ClusterRecord): Promise<ClusterRecord> {
545
- log('cluster-member:merge-records', {
546
- messageHash: existing.messageHash,
547
- existingPromises: Object.keys(existing.promises ?? {}),
548
- existingCommits: Object.keys(existing.commits ?? {}),
549
- incomingPromises: Object.keys(incoming.promises ?? {}),
550
- incomingCommits: Object.keys(incoming.commits ?? {})
551
- });
552
- // Verify that immutable fields match
553
- if (existing.messageHash !== incoming.messageHash) {
554
- throw new Error('Message hash mismatch');
555
- }
556
- if (ClusterMember.canonicalJson(existing.message) !== ClusterMember.canonicalJson(incoming.message)) {
557
- throw new Error('Message content mismatch');
558
- }
559
- if (existing.membershipVersion === 2 || incoming.membershipVersion === 2) {
560
- // v2: the sorted peer-id set (captured by membershipDigest) is bound into messageHash, so equal
561
- // messageHash MUST imply equal membership on any honest path. A mismatch here — different digest
562
- // or version at equal hash — is a protocol violation (a bug or a hash-collision attack), NOT an
563
- // honest divergence: two honest members with different views now hold two DIFFERENT hashes, i.e.
564
- // two competing transactions the race machinery resolves, not one contested record. Log loudly
565
- // and reject; never silently adopt the incoming set. (validateRecord already proved each record's
566
- // own digest matches its own peers, so multiaddr / pubkey churn within the SAME id set — which
567
- // keeps the same digest and hash — does NOT trip this.)
568
- if (existing.membershipVersion !== incoming.membershipVersion || existing.membershipDigest !== incoming.membershipDigest) {
569
- log('cluster-member:peers-mismatch-invariant-violation', {
570
- messageHash: existing.messageHash,
571
- existingVersion: existing.membershipVersion,
572
- incomingVersion: incoming.membershipVersion,
573
- existingDigest: existing.membershipDigest,
574
- incomingDigest: incoming.membershipDigest,
575
- existingPeers: Object.keys(existing.peers ?? {}).sort(),
576
- incomingPeers: Object.keys(incoming.peers ?? {}).sort()
577
- });
578
- throw new Error('Peers mismatch');
579
- }
580
- } else if (ClusterMember.canonicalJson(existing.peers) !== ClusterMember.canonicalJson(incoming.peers)) {
581
- // v1 (legacy, membership unbound): full peer-object equality is the only available guard.
582
- throw new Error('Peers mismatch');
583
- }
584
-
585
- // Merge signatures with equivocation detection
586
- const mergedPromises = this.detectEquivocation(
587
- existing.promises, incoming.promises, 'promise', existing.messageHash
588
- );
589
- const mergedCommits = this.detectEquivocation(
590
- existing.commits, incoming.commits, 'commit', existing.messageHash
591
- );
592
-
593
- return {
594
- ...existing,
595
- promises: mergedPromises,
596
- commits: mergedCommits
597
- };
598
- }
599
-
600
- /**
601
- * Compares existing vs incoming signatures for the same peers.
602
- * If a peer's vote type changed (approve↔reject), that's equivocation:
603
- * report a penalty and keep the first-seen signature.
604
- * New peers are accepted normally.
605
- */
606
- private detectEquivocation(
607
- existing: Record<string, Signature>,
608
- incoming: Record<string, Signature>,
609
- phase: 'promise' | 'commit',
610
- messageHash: string
611
- ): Record<string, Signature> {
612
- // NOTE: relies on validateSignatures() (via validateRecord in processUpdate) having already run on
613
- // every record reaching here, so each peerId's signature is key-bound. Without that guarantee the
614
- // Equivocation penalty below would act on self-asserted, unverified peer ids and could frame an
615
- // honest peer. Do not call this on unvalidated signatures.
616
- const merged = { ...existing };
617
-
618
- for (const [peerId, incomingSig] of Object.entries(incoming)) {
619
- const existingSig = existing[peerId];
620
- if (existingSig) {
621
- if (existingSig.type !== incomingSig.type) {
622
- // Equivocation detected: peer changed their vote type
623
- log('cluster-member:equivocation-detected', {
624
- peerId,
625
- phase,
626
- messageHash,
627
- existingType: existingSig.type,
628
- incomingType: incomingSig.type
629
- });
630
- this.reputation?.reportPeer(
631
- peerId,
632
- PenaltyReason.Equivocation,
633
- `${phase}:${messageHash}:${existingSig.type}->${incomingSig.type}`
634
- );
635
- // Keep first-seen signature — do not let the peer flip their vote
636
- }
637
- // Same type: keep existing (no-op, already in merged)
638
- } else {
639
- // New peer — accept normally
640
- merged[peerId] = incomingSig;
641
- }
642
- }
643
-
644
- return merged;
645
- }
646
-
647
- private async validateRecord(record: ClusterRecord): Promise<void> {
648
- // Reject a record whose membership-binding version this code does not implement. The cluster
649
- // consensus code is a single deployable unit (all cluster members upgrade together), so a version
650
- // we don't understand is rejected rather than cross-version-consensus'd.
651
- if (record.membershipVersion !== undefined && record.membershipVersion !== 1 && record.membershipVersion !== 2) {
652
- throw new Error(`Unsupported membershipVersion: ${record.membershipVersion}`);
653
- }
654
-
655
- // v2: the declared membership digest must match the record's own peer set. A record whose declared
656
- // digest doesn't match its peers is malformed (and its messageHash — computed over that digest —
657
- // would not bind the real membership).
658
- // NOTE: recomputes membershipDigest (one SHA256 over the sorted peer-id list) on every incoming v2
659
- // record; if a hot cluster ever shows this as a cost, memoize per (messageHash → digest).
660
- if (record.membershipVersion === 2) {
661
- const expectedDigest = await membershipDigest(record.peers);
662
- if (expectedDigest !== record.membershipDigest) {
663
- throw new Error(`Membership digest mismatch: expected=${expectedDigest}, received=${record.membershipDigest ?? 'undefined'}`);
664
- }
665
- }
666
-
667
- // Validate message hash matches the message content (v2 folds in the membership digest)
668
- const expectedHash = await this.computeMessageHash(record);
669
- if (expectedHash !== record.messageHash) {
670
- throw new Error(`Message hash mismatch: expected=${expectedHash}, received=${record.messageHash}`);
671
- }
672
-
673
- // Validate signatures
674
- await this.validateSignatures(record);
675
-
676
- // Validate expiration
677
- if (record.message.expiration && record.message.expiration < Date.now()) {
678
- throw new Error('Transaction expired');
679
- }
680
- }
681
-
682
- /**
683
- * Compute message hash using the same algorithm as the coordinator. Version-dispatched: a v2 record
684
- * folds its membership digest into the preimage, a v1 / unversioned record hashes byte-identically to
685
- * before this change. Must match cluster-coordinator.ts createMessageHash().
686
- */
687
- private async computeMessageHash(record: Pick<ClusterRecord, 'message' | 'membershipVersion' | 'membershipDigest'>): Promise<string> {
688
- return computeClusterMessageHash(record.message, recordMembershipDigest(record));
689
- }
690
-
691
- private async validateSignatures(record: ClusterRecord): Promise<void> {
692
- // Validate promise signatures. Reject on any failure, but only report an InvalidSignature
693
- // penalty when the key was proven to belong to `peerId` (outcome.penalize) — otherwise the id
694
- // is attacker-chosen and reporting it would let a coordinator frame an honest peer.
695
- const promiseHash = await this.computePromiseHash(record);
696
- for (const [peerId, signature] of Object.entries(record.promises)) {
697
- const outcome = await this.verifySignature(record, peerId, promiseHash, signature);
698
- if (!outcome.valid) {
699
- if (outcome.penalize) {
700
- this.reputation?.reportPeer(peerId, PenaltyReason.InvalidSignature, `promise:${record.messageHash}`);
701
- }
702
- throw new Error(`Invalid promise signature from ${peerId}`);
703
- }
704
- }
705
-
706
- // Validate commit signatures
707
- const commitHash = await this.computeCommitHash(record);
708
- for (const [peerId, signature] of Object.entries(record.commits)) {
709
- const outcome = await this.verifySignature(record, peerId, commitHash, signature);
710
- if (!outcome.valid) {
711
- if (outcome.penalize) {
712
- this.reputation?.reportPeer(peerId, PenaltyReason.InvalidSignature, `commit:${record.messageHash}`);
713
- }
714
- throw new Error(`Invalid commit signature from ${peerId}`);
715
- }
716
- }
717
- }
718
-
719
- /**
720
- * Deterministic JSON: sorts object keys so comparisons are order-independent.
721
- * NOTE: this is a second copy of the canonicalization in db-core `membership.ts` (which feeds the hash
722
- * preimages). It is used here only for equality checks (message-content / v1-peers compare), and those
723
- * run *after* a messageHash-equality gate, so a drift between the two can't silently forge agreement —
724
- * but keep them byte-identical. If a third caller appears, promote to a single exported helper.
725
- */
726
- private static canonicalJson(value: unknown): string {
727
- return JSON.stringify(value, (_, v) =>
728
- v && typeof v === 'object' && !Array.isArray(v)
729
- ? Object.keys(v).sort().reduce((o: Record<string, unknown>, k) => { o[k] = v[k]; return o; }, {})
730
- : v
731
- );
732
- }
733
-
734
- private async computePromiseHash(record: ClusterRecord): Promise<string> {
735
- return computeClusterPromiseHash(record.messageHash, record.message, recordMembershipDigest(record));
736
- }
737
-
738
- private async computeCommitHash(record: ClusterRecord): Promise<string> {
739
- return computeClusterCommitHash(record.messageHash, record.message, record.promises, recordMembershipDigest(record));
740
- }
741
-
742
- private async signVote(hash: string, type: Signature['type'], extra?: string): Promise<string> {
743
- const sigBytes = await this.privateKey.sign(clusterVoteSigningPayload(hash, type, extra));
744
- return uint8ArrayToString(sigBytes, 'base64url');
745
- }
746
-
747
- /**
748
- * Verify one vote signature and classify the outcome (see {@link VerifyOutcome}). Total on hostile
749
- * input: a missing/empty key, a non-Ed25519 id, a key not bound to `peerId`, or malformed bytes all
750
- * yield `{ valid:false, penalize:false }` (reject without penalizing an unproven identity) rather
751
- * than throwing. Only after the key is proven to be the one `peerId` names does a failed
752
- * cryptographic verify yield `{ valid:false, penalize:true }`.
753
- *
754
- * NOTE: the binding check (`peerIdBindsPublicKey`) proves the vote was signed by the key `peerId`
755
- * names — it does NOT establish that `peerId` is legitimately in the cohort. A coordinator minting
756
- * fresh keypairs and using each key's own derived id passes this for every one. Sybil/cohort
757
- * membership is a separate layer (cohort-topic membership certificates), not solved here.
758
- */
759
- private async verifySignature(record: ClusterRecord, peerId: string, hash: string, signature: Signature): Promise<VerifyOutcome> {
760
- const peerInfo = record.peers[peerId];
761
- if (!peerInfo?.publicKey?.length) {
762
- // No key to check against — identity not proven. Reject without penalty.
763
- return { valid: false, penalize: false };
764
- }
765
- let keyBytes: Uint8Array;
766
- try {
767
- // publicKey is base64url-encoded string (JSON-serialization safe)
768
- keyBytes = uint8ArrayFromString(peerInfo.publicKey, 'base64url');
769
- } catch {
770
- return { valid: false, penalize: false };
771
- }
772
- // The key must be the one `peerId` provably names, else the vote could be attributed to any peer
773
- // id while signed by a key the coordinator controls. Binding failure ⇒ identity unproven ⇒ no penalty.
774
- if (!peerIdBindsPublicKey(peerId, keyBytes)) {
775
- return { valid: false, penalize: false };
776
- }
777
- try {
778
- const pubKey = publicKeyFromRaw(keyBytes);
779
- const payload = clusterVoteVerificationPayload(hash, signature);
780
- const sigBytes = uint8ArrayFromString(signature.signature, 'base64url');
781
- const ok = await pubKey.verify(payload, sigBytes);
782
- // Key is bound to peerId: a failed verify is a genuine bad vote from a proven identity → penalize.
783
- // NOTE: residual — an Ed25519 peer's public key is derivable from its (public) id, so an attacker
784
- // can attach a victim's REAL key with a garbage signature and still trip this InvalidSignature
785
- // penalty on the victim. Binding narrows framing (the attacker must use the victim's own key, not
786
- // an arbitrary one) but cannot eliminate it here: a single signature can't distinguish "victim
787
- // signed badly" from "someone pasted the victim's public key + junk". Fully closing it needs an
788
- // authenticated membership/channel layer (cohort-topic membership certs), out of scope for this fix.
789
- return ok ? { valid: true } : { valid: false, penalize: true };
790
- } catch {
791
- // Malformed signature bytes / key decode failure: reject, but do not penalize on unparseable input.
792
- return { valid: false, penalize: false };
793
- }
794
- }
795
-
796
- private async getTransactionPhase(record: ClusterRecord): Promise<PhaseResult> {
797
- const peerCount = Object.keys(record.peers).length;
798
- const promiseCount = Object.keys(record.promises).length;
799
- const ourId = this.peerId.toString();
800
-
801
- const superMajority = Math.ceil(peerCount * this.superMajorityThreshold);
802
- const maxAllowedRejections = peerCount - superMajority;
803
-
804
- // Check for rejections — rejected if too many rejections to ever reach super-majority.
805
- // ONLY `reject` votes count here: a `conflict` vote is "not now", never a validity
806
- // judgement, so it must not push a record into the permanent `Rejected` phase.
807
- const rejectedPromises = Object.values(record.promises).filter(s => s.type === 'reject');
808
- const conflictPromises = Object.values(record.promises).filter(s => s.type === 'conflict');
809
- const rejectedCommits = Object.values(record.commits).filter(s => s.type === 'reject');
810
- if (rejectedPromises.length > maxAllowedRejections || this.hasMajority(rejectedCommits.length, peerCount)) {
811
- return { phase: TransactionPhase.Rejected };
812
- }
813
-
814
- // Conflict votes don't judge validity, but enough of them still make super-majority
815
- // unreachable a distinct terminal outcome (retryable as a fresh transaction) so logs and
816
- // reputation-adjacent paths keep meaning what they say.
817
- if (conflictPromises.length > 0 && rejectedPromises.length + conflictPromises.length > maxAllowedRejections) {
818
- return { phase: TransactionPhase.ConflictSuperseded };
819
- }
820
-
821
- // Check if we need to vote. A lost race is answered with a conflict vote, not silence:
822
- // absence used to mean both "unreachable" and "refusing in favour of a rival", and the
823
- // coordinator could not tell the two apart. Once our conflict vote is merged into
824
- // `promises`, this branch is skipped forever a conflict vote is terminal for this record;
825
- // a retry must be a fresh transaction (new messageHash), which `CoordinatorRepo.pend`
826
- // already mints per call.
827
- if (!record.promises[ourId]) {
828
- const conflict = this.findConflict(record);
829
- if (conflict) {
830
- return { phase: TransactionPhase.OurConflictVoteNeeded, conflictsWith: conflict.blockedBy };
831
- }
832
- return { phase: TransactionPhase.OurPromiseNeeded };
833
- }
834
-
835
- // Check if we have enough approved promises to proceed to commit. Deliberately blind to what
836
- // OUR own vote was: the rest of the cohort reaching super-majority is the commit rule
837
- // (Theorem 1 Case 2), so a member that rejected or conflict-voted still signs the commit
838
- // the cohort decided on rather than stalling it. Only the rejection/superseded thresholds
839
- // above can stop a record here, and both are checked first.
840
- //
841
- // NOTE: signing the commit drops this member's reservation on the record
842
- // (`shouldPersist = false` in the caller), and the phase fixpoint means that can now happen on
843
- // the FIRST delivery when the record already arrives at super-majority, rather than a
844
- // round-trip later. The safety argument is quorum intersection (Theorem 9: no rival can
845
- // assemble its own super-majority once this one has), NOT the reservation — the reservation
846
- // only orders *concurrently-pending* rivals. If a lost-update between commit-signing and
847
- // consensus-apply ever shows up, hold the reservation until `handleConsensus` instead of
848
- // releasing it here.
849
- const approvedPromises = Object.values(record.promises).filter(s => s.type === 'approve');
850
- if (approvedPromises.length >= superMajority && !record.commits[ourId]) {
851
- return { phase: TransactionPhase.OurCommitNeeded };
852
- }
853
-
854
- // Check if still collecting promises
855
- if (promiseCount < peerCount && approvedPromises.length < superMajority) {
856
- return { phase: TransactionPhase.Promising };
857
- }
858
-
859
- // Check for consensus
860
- const approvedCommits = Object.values(record.commits).filter(s => s.type === 'approve');
861
- if (this.hasMajority(approvedCommits.length, peerCount)) {
862
- return { phase: TransactionPhase.Consensus };
863
- }
864
-
865
- return { phase: TransactionPhase.Propagating };
866
- }
867
-
868
- private hasMajority(count: number, total: number): boolean {
869
- return count > total / 2;
870
- }
871
-
872
- private async handlePromiseNeeded(record: ClusterRecord): Promise<ClusterRecord> {
873
- // Membership admission gate runs BEFORE pend validation: a member independently checks the declared
874
- // peer set is a legitimate cluster it belongs to, and refuses (reject vote) rather than rubber-stamping
875
- // a set the coordinator chose (e.g. a self-shrunk minority-partition set). On admission failure we skip
876
- // pend validation entirely and emit the membership rejection.
877
- const validationResult = await this.evaluatePromise(record);
878
-
879
- const promiseHash = await this.computePromiseHash(record);
880
- const type = validationResult.valid ? 'approve' as const : 'reject' as const;
881
- const rejectReason = validationResult.valid ? undefined : validationResult.reason;
882
- const sig = await this.signVote(promiseHash, type, rejectReason);
883
-
884
- const signature: Signature = validationResult.valid
885
- ? { type: 'approve', signature: sig }
886
- : { type: 'reject', signature: sig, rejectReason };
887
-
888
- if (!validationResult.valid) {
889
- log('cluster-member:validation-rejected', {
890
- messageHash: record.messageHash,
891
- reason: validationResult.reason
892
- });
893
- }
894
-
895
- return {
896
- ...record,
897
- promises: {
898
- ...record.promises,
899
- [this.peerId.toString()]: signature
900
- }
901
- };
902
- }
903
-
904
- /**
905
- * Answer a record that lost the deterministic race to a transaction this member already holds
906
- * (`docs/correctness.md` Theorems 1 & 9: the loser is TOLD it lost, not ignored — an unanswered
907
- * loss is indistinguishable from an unreachable cohort at the coordinator). `conflictWith` the
908
- * winning rival's messageHash is folded into the signed payload, so the claim is
909
- * integrity-protected in transit and readable without parsing prose. NOT a validity judgement:
910
- * {@link getTransactionPhase} never counts conflict votes toward the permanent-rejection
911
- * threshold, and the coordinator surfaces them as a retryable loss, never a validator rejection.
912
- */
913
- private async handleConflictVoteNeeded(record: ClusterRecord, conflictWith: string): Promise<ClusterRecord> {
914
- log('cluster-member:action-conflict-vote', {
915
- messageHash: record.messageHash,
916
- conflictWith
917
- });
918
- const promiseHash = await this.computePromiseHash(record);
919
- const sig = await this.signVote(promiseHash, 'conflict', conflictWith);
920
- const signature: Signature = { type: 'conflict', signature: sig, conflictWith };
921
-
922
- return {
923
- ...record,
924
- promises: {
925
- ...record.promises,
926
- [this.peerId.toString()]: signature
927
- }
928
- };
929
- }
930
-
931
- /**
932
- * The full promise-phase decision for a record: admit the declared membership FIRST, then (only if
933
- * admitted) validate its pend operations. Failing either yields a `{ valid:false, reason }` the caller
934
- * turns into a `reject` vote. Splitting membership from pend validation keeps the reason strings
935
- * distinct a `membership-not-admitted` reject is a different signal (feeds the dispute path) than a
936
- * stale-revision / custom-validator reject.
937
- */
938
- private async evaluatePromise(record: ClusterRecord): Promise<{ valid: boolean; reason?: string }> {
939
- const admission = await this.admitMembership(record);
940
- if (!admission.admit) {
941
- return { valid: false, reason: admission.reason ?? MEMBERSHIP_NOT_ADMITTED };
942
- }
943
- return await this.validatePendOperations(record);
944
- }
945
-
946
- /**
947
- * Membership admission gate. Decides whether the coordinator-declared peer set (`record.peers`, call it
948
- * `D`) is a *legitimate* cluster this member may vote inside, judged against the member's OWN
949
- * independently-derived view not against anything the (untrusted) record declares about its size.
950
- * Evaluated on the promise path before the member signs an approve.
951
- *
952
- * The predicate admits `D` iff ALL hold:
953
- * 1. **Self-membership** — this member's id ∈ `D`; else this block is not its responsibility (and a
954
- * coordinator must not route a record to a non-member to pad approval counts).
955
- * 2. **Not a self-shrink below the floor** with a confident derived view `E`, `|D| ≥ ⌈fraction·|E|⌉`
956
- * (and minAbsoluteClusterSize). `|E|` is the member's own confident cluster-size estimate `K_est`,
957
- * so a minority-partition set (small `D`) is rejected against the member's larger view.
958
- * 3. **Consistency with the derived view** — `|D △ E|` within `clusterSizeTolerance·|E|`; honest churn
959
- * of a peer or two is absorbed, a wholesale-disjoint or half-size set is not.
960
- *
961
- * **Fail-closed posture.** When the member cannot confidently derive `E` (no capability, low FRET
962
- * confidence — exactly what a partition induces), it must refuse any *downsizing* decision — but it
963
- * needs a size reference to judge "downsize" against, and it may NOT borrow `clusterSize` for that:
964
- * `clusterSize` is the replication factor (what a cohort should aim for), not a claim about how many
965
- * peers exist, so a small deployment configured with the default 10 would refuse every write. The
966
- * fallback yardstick is instead {@link ClusterConsensusConfig.assumedClusterSize} — the operator's own
967
- * assertion of the smallest cohort this deployment can genuinely field — run through the SAME
968
- * {@link admissionFloor} as the confident path, so the fallback can never be stricter than the measured
969
- * path (it was: it demanded the full configured size, with no fraction and no slack for churn or a peer
970
- * not yet discovered). With NEITHER a confident view NOR an asserted size the gate cannot judge a
971
- * downsize at all, so it preserves the legacy approve behavior (backward-compatible for nodes/tests with
972
- * no derivation wired). `allowUnvalidatedSmallCluster` is the explicit opt-in (single-node / local dev
973
- * knowingly below the safe floor), matching the coordinator's `validateSmallCluster` semantics.
974
- */
975
- private async admitMembership(record: ClusterRecord): Promise<{ admit: boolean; reason?: string }> {
976
- const ourId = this.peerId.toString();
977
- const declared = Object.keys(record.peers ?? {});
978
-
979
- // Predicate 1: self-membership. Always enforced (independent of any opt-in): a member does not vote
980
- // in a cluster it is not part of.
981
- if (!declared.includes(ourId)) {
982
- log('cluster-member:admission-reject', { messageHash: record.messageHash, reason: 'self-not-member', declaredSize: declared.length });
983
- return { admit: false, reason: `${MEMBERSHIP_NOT_ADMITTED}:self-not-member` };
984
- }
985
-
986
- // Explicit opt-in: knowingly transact below the safe floor (single-node / local dev). Skips the
987
- // size/consistency gates but not self-membership above.
988
- if (this.allowUnvalidatedSmallCluster) {
989
- return { admit: true };
990
- }
991
-
992
- const derived = await this.deriveExpectedClusterView(record);
993
- // An empty derived view (kEst === 0) carries no usable reference set: measured against it every
994
- // non-empty declared set is wholly "inconsistent" (maxDiff = ceil(tol·0) = 0), which would spuriously
995
- // reject a legitimate full cluster a stricter, worse outcome than an absent view. Treat empty as
996
- // not-confident so it takes the fail-closed-or-legacy branch below instead. (Not normally reachable:
997
- // a responsible member's findCluster includes at least itself; this guards a transient empty read.)
998
- const derivedSize = derived !== undefined ? Object.keys(derived.peers ?? {}).length : 0;
999
- const confident = derived !== undefined
1000
- && derived.confidence > ClusterMember.MembershipConfidenceThreshold
1001
- && derivedSize > 0;
1002
-
1003
- if (!confident) {
1004
- // Fail closed for downsizing under low/absent confidence, measured against the operator's asserted
1005
- // cohort size rather than the replication factor. With no asserted size the gate cannot tell a
1006
- // downsize from a legitimately small cluster at all, so it preserves legacy approve behavior.
1007
- if (this.assumedClusterSize === undefined) {
1008
- return { admit: true };
1009
- }
1010
- const floor = this.admissionFloor(this.assumedClusterSize);
1011
- if (declared.length >= floor) {
1012
- return { admit: true };
1013
- }
1014
- log('cluster-member:admission-reject', {
1015
- messageHash: record.messageHash,
1016
- reason: 'low-confidence-downsize',
1017
- declaredSize: declared.length,
1018
- floor,
1019
- assumedClusterSize: this.assumedClusterSize,
1020
- confidence: derived?.confidence
1021
- });
1022
- // The numbers ride along in the reason: this rejection is caused by *local* configuration, and
1023
- // without them a coordinator (or an operator reading a dispute record) has no hint which knob did it.
1024
- // NOTE: two honest members with different local config now emit *different* reason strings for the
1025
- // same record. Nothing compares reasons across peers today (`disputeEvidence.rejectReasons` is a
1026
- // per-peer map, and the signed payload hashes the string each vote carries); if anything ever groups
1027
- // or dedupes dispute reasons by string equality, group on the `membership-not-admitted:<variant>`
1028
- // prefix, not the whole string.
1029
- return {
1030
- admit: false,
1031
- reason: `${MEMBERSHIP_NOT_ADMITTED}:low-confidence-downsize (declared=${declared.length}, floor=${floor}, assumedClusterSize=${this.assumedClusterSize})`
1032
- };
1033
- }
1034
-
1035
- const expected = Object.keys(derived!.peers ?? {});
1036
- const kEst = expected.length;
1037
-
1038
- // Predicate 2: floor derived from the member's OWN confident estimate.
1039
- const floor = this.admissionFloor(kEst);
1040
- if (declared.length < floor) {
1041
- log('cluster-member:admission-reject', {
1042
- messageHash: record.messageHash,
1043
- reason: 'below-floor',
1044
- declaredSize: declared.length,
1045
- floor,
1046
- kEst
1047
- });
1048
- return {
1049
- admit: false,
1050
- reason: `${MEMBERSHIP_NOT_ADMITTED}:below-floor (declared=${declared.length}, floor=${floor}, kEst=${kEst})`
1051
- };
1052
- }
1053
-
1054
- // Predicate 3: consistency with the derived view within tolerance.
1055
- const symmetricDiff = ClusterMember.symmetricDifferenceSize(declared, expected);
1056
- const maxDiff = Math.ceil(this.clusterSizeTolerance * kEst);
1057
- if (symmetricDiff > maxDiff) {
1058
- log('cluster-member:admission-reject', {
1059
- messageHash: record.messageHash,
1060
- reason: 'inconsistent-with-derived-view',
1061
- declaredSize: declared.length,
1062
- kEst,
1063
- symmetricDiff,
1064
- maxDiff
1065
- });
1066
- return { admit: false, reason: `${MEMBERSHIP_NOT_ADMITTED}:inconsistent-with-derived-view` };
1067
- }
1068
-
1069
- return { admit: true };
1070
- }
1071
-
1072
- /**
1073
- * The smallest declared peer set admissible against a cohort-size reference `k`, whether `k` is
1074
- * measured (the confident path's `kEst`) or asserted (`assumedClusterSize`). One function so the
1075
- * fallback can never be stricter than the measured path which it was, demanding the full configured
1076
- * size with no fraction and no slack. Clamped at `minAbsoluteClusterSize`, so a degenerate `k` of 0, 1
1077
- * or negative yields the absolute floor rather than a floor that admits everything. A non-finite scaled
1078
- * size (a `NaN` or `Infinity` config value) is likewise treated as no usable reference rather than
1079
- * propagating: an unguarded `NaN` floor fails EVERY comparison, which would silently make the node
1080
- * reject every unconfident write.
1081
- *
1082
- * NOTE: partition safety needs `2 · membershipAdmissionFraction · superMajorityThreshold > 1` each
1083
- * side of a split must recruit `fraction · threshold · K` distinct honest members, and two sides cannot
1084
- * both find them in one K-peer cluster. At the shipped defaults (0.75 · 0.75 both default to
1085
- * `DEFAULT_SUPER_MAJORITY_THRESHOLD` / `membershipAdmissionFraction`'s own default) that product is
1086
- * 1.125. If either default is ever lowered, re-check Theorem 2 in `docs/correctness.md` before shipping it.
1087
- */
1088
- private admissionFloor(k: number): number {
1089
- const scaled = Math.ceil(this.membershipAdmissionFraction * k);
1090
- return Math.max(this.minAbsoluteClusterSize, Number.isFinite(scaled) ? scaled : 0);
1091
- }
1092
-
1093
- /**
1094
- * Derive this member's own view of the record's block cluster via the injected capability, or
1095
- * `undefined` when it cannot (no capability, no coordinating block id, or a derivation error all of
1096
- * which the gate treats as "not confident"). Derived from the record's coordinating block, the same key
1097
- * the coordinator used to select the cluster.
1098
- */
1099
- private async deriveExpectedClusterView(record: ClusterRecord): Promise<ExpectedClusterView | undefined> {
1100
- if (!this.deriveExpectedCluster) {
1101
- return undefined;
1102
- }
1103
- const blockId = record.coordinatingBlockIds?.[0];
1104
- if (blockId === undefined) {
1105
- return undefined;
1106
- }
1107
- try {
1108
- // NOTE: derives (findCluster) once per inbound record on the promise path — one routing lookup
1109
- // per vote. If this shows up as hot, cache the derived view per (blockId, short TTL): it is a
1110
- // pure read of current topology, so a few-seconds-stale view is safe for admission.
1111
- return await this.deriveExpectedCluster(blockId as BlockId);
1112
- } catch (err) {
1113
- log('cluster-member:derive-expected-cluster-error', { messageHash: record.messageHash, error: (err as Error).message });
1114
- return undefined;
1115
- }
1116
- }
1117
-
1118
- /** |A △ B| over two id lists (order-independent set symmetric difference). */
1119
- private static symmetricDifferenceSize(a: string[], b: string[]): number {
1120
- const setA = new Set(a);
1121
- const setB = new Set(b);
1122
- let count = 0;
1123
- for (const x of setA) if (!setB.has(x)) count++;
1124
- for (const x of setB) if (!setA.has(x)) count++;
1125
- return count;
1126
- }
1127
-
1128
- /**
1129
- * Validates pend operations in a cluster record using the transaction validator.
1130
- * Also checks for stale revisions to prevent consensus on operations that would fail.
1131
- * Returns success if no validator is configured (backwards compatibility).
1132
- */
1133
- private async validatePendOperations(record: ClusterRecord): Promise<{ valid: boolean; reason?: string }> {
1134
- // Find pend operations in the message
1135
- for (const operation of record.message.operations) {
1136
- if ('pend' in operation) {
1137
- const pendRequest = operation.pend;
1138
-
1139
- // Check for stale revisions before allowing consensus
1140
- if (pendRequest.rev !== undefined) {
1141
- const blockIds = blockIdsForTransforms(pendRequest.transforms);
1142
- // Get block states to check latest revisions
1143
- const blockResults = await this.storageRepo.get({ blockIds });
1144
- for (const blockId of blockIds) {
1145
- const blockResult = blockResults[blockId];
1146
- if (blockResult?.unavailable !== undefined) {
1147
- // This member cannot establish the block's revision, so it cannot judge
1148
- // staleness. Vote reject rather than approve on an answer it knows is a
1149
- // guess — approving would let a stale pend reach consensus on the strength
1150
- // of a member that could not check it. (Before StorageRepo caught
1151
- // materialization faults per block, this read threw out of the promise
1152
- // handler; rejecting keeps the fail-closed posture with a signed reason.)
1153
- log('cluster-member:validation-block-unavailable', {
1154
- messageHash: record.messageHash,
1155
- blockId,
1156
- reason: blockResult.unavailable
1157
- });
1158
- return { valid: false, reason: `block ${blockId} unavailable (${blockResult.unavailable}): cannot verify revision` };
1159
- }
1160
- const latestRev = blockResult?.state?.latest?.rev;
1161
- if (latestRev !== undefined && latestRev >= pendRequest.rev) {
1162
- log('cluster-member:validation-stale-revision', {
1163
- messageHash: record.messageHash,
1164
- blockId,
1165
- requestedRev: pendRequest.rev,
1166
- latestRev
1167
- });
1168
- // Deliberately prose-only: this reason is fed to computeSigningPayload, signed,
1169
- // and carried as Signature.rejectReason, so adding a structured revision here
1170
- // would change the signed byte layout and the Signature type — every peer would
1171
- // have to agree on the new format or verification breaks across versions. This
1172
- // is NOT a StaleFailure producer, so StaleFailure.staleAt does not apply; the
1173
- // coordinator's own local re-read (CoordinatorRepo.classifyStaleRejection)
1174
- // supplies that number when it can confirm the revision itself.
1175
- return { valid: false, reason: `stale revision: block ${blockId} at rev ${latestRev}, requested rev ${pendRequest.rev}` };
1176
- }
1177
- }
1178
- }
1179
-
1180
- // Run custom validator if configured
1181
- if (this.validator && pendRequest.transaction && pendRequest.operationsHash) {
1182
- const result = await this.validator.validate(pendRequest.transaction, pendRequest.operationsHash);
1183
- if (!result.valid) {
1184
- return { valid: false, reason: result.reason };
1185
- }
1186
- }
1187
- }
1188
- }
1189
-
1190
- return { valid: true };
1191
- }
1192
-
1193
- private async handleCommitNeeded(record: ClusterRecord): Promise<ClusterRecord> {
1194
- if (this.hasLocalCommit(record)) {
1195
- return record;
1196
- }
1197
- const commitHash = await this.computeCommitHash(record);
1198
- const sig = await this.signVote(commitHash, 'approve');
1199
- const signature: Signature = {
1200
- type: 'approve',
1201
- signature: sig
1202
- };
1203
-
1204
- return {
1205
- ...record,
1206
- commits: {
1207
- ...record.commits,
1208
- [this.peerId.toString()]: signature
1209
- }
1210
- };
1211
- }
1212
-
1213
- /**
1214
- * Executes operations after consensus is reached.
1215
- *
1216
- * @warning This method executes on ALL cluster peers, not just the coordinator.
1217
- * Each peer independently applies the operations to its local storage.
1218
- *
1219
- * @pitfall **Check-then-act race** - The in-memory guard must be checked AND set
1220
- * atomically (before any `await`) to prevent duplicate execution; JavaScript's
1221
- * single-threaded nature makes that synchronous check-and-set atomic. The durable
1222
- * marker, by contrast, is persisted only *after* apply succeeds — writing it eagerly
1223
- * would leave a stuck marker on a caught fault or a crash mid-apply, silently dropping
1224
- * the transaction on this member on redelivery.
1225
- *
1226
- * @pitfall **Independent node storage** - Each node has its own storage. After consensus,
1227
- * each node applies operations locally. Nodes must fetch missing blocks from cluster
1228
- * peers via `restoreCallback` if they don't have prior revisions.
1229
- *
1230
- * @see docs/internals.md "Check-Then-Act Race in Consensus" and "Independent Node Storage" pitfalls
1231
- */
1232
- private async handleConsensus(record: ClusterRecord): Promise<void> {
1233
- // Check persistent store first for post-recovery dedup (in-memory map is cleared on restart).
1234
- // wasTransactionExecutedAsync also checks the in-memory map as a fast path.
1235
- if (await this.wasTransactionExecutedAsync(record.messageHash)) {
1236
- log('cluster-member:consensus-already-executed', { messageHash: record.messageHash });
1237
- return;
1238
- }
1239
- // Check-and-set ATOMICALLY to prevent race condition where multiple calls
1240
- // pass the async check before any completes. Since JavaScript is single-threaded,
1241
- // this synchronous check-and-set is atomic before any await.
1242
- if (this.executedTransactions.has(record.messageHash)) {
1243
- log('cluster-member:consensus-already-executed', { messageHash: record.messageHash });
1244
- return;
1245
- }
1246
- // Set the in-memory guard IMMEDIATELY, before any async operations: its synchronous
1247
- // check-and-set (line above's `has` + this `set`) is what prevents the concurrent
1248
- // apply-window race where two handleConsensus calls for the same hash both pass the
1249
- // async check. The durable marker is deliberately NOT written here — see below.
1250
- const executedAt = Date.now();
1251
- this.executedTransactions.set(record.messageHash, executedAt);
1252
-
1253
- try {
1254
- for (const operation of record.message.operations) {
1255
- await this.applyConsensusOperation(record, operation);
1256
- }
1257
- } catch (err) {
1258
- // A genuinely unexpected fault (e.g. storage I/O) — roll back the in-memory
1259
- // marker so a corrected retry can re-run, and propagate so the caller learns
1260
- // the real cause. The durable marker was never written (it lands only after
1261
- // apply succeeds, below), so there is nothing to roll back. Recoverable local
1262
- // divergence is absorbed inside applyConsensusOperation and never reaches here.
1263
- this.executedTransactions.delete(record.messageHash);
1264
- throw err;
1265
- }
1266
-
1267
- // Persist the durable marker only now that apply has actually succeeded. Writing it
1268
- // eagerly (before the loop) would leave a stuck marker on a caught fault OR a crash
1269
- // mid-apply, and on redelivery handleConsensus short-circuits at the async
1270
- // wasTransactionExecuted check — silently dropping the transaction on this member
1271
- // forever. The durable marker exists only for post-restart dedup (the in-memory map
1272
- // is empty after restart), and the narrow window between "apply succeeded" and
1273
- // "durable write landed" is safe to re-run on restart: re-applying an
1274
- // already-applied consensus transaction is idempotent (the "ahead" divergence path
1275
- // in applyConsensusOperation tolerates it as a no-op), so it converges rather than
1276
- // dropping. Fire-and-forget: a persist failure must not fail the apply that succeeded.
1277
- this.stateStore?.markExecuted(record.messageHash, executedAt)
1278
- .catch(err => log('cluster-member:persist-executed-error', { messageHash: record.messageHash, error: (err as Error).message }));
1279
- }
1280
-
1281
- /**
1282
- * Applies one consensus-approved operation to local storage.
1283
- *
1284
- * By the time we reach here the cluster has *already* reached consensus, so the
1285
- * operation is authoritative cluster-wide. A failure applying it to THIS member's
1286
- * local store therefore does not mean the operation is invalid — it means our
1287
- * local state has diverged from the agreed history:
1288
- *
1289
- * - **ahead**: we already hold a newer revision, so a stale pend/commit is a
1290
- * no-op for us (`StorageRepo.commit` returns `success:false` with `missing`);
1291
- * - **behind**: we missed the prior `pend` cluster-transaction (cohort drift
1292
- * between the independent pend and commit phases, or transient unreachability),
1293
- * so we lack the pending action`StorageRepo.commit` *throws* "Pending
1294
- * action not found";
1295
- * - **behind (no base)**: we DID see the pend, but we never saw the revision that
1296
- * created the block, so the transform has nothing to apply to —
1297
- * `StorageRepo.commit` returns `success:false` with a `missing-base-revision`
1298
- * reason rather than recording a revision it could not materialize.
1299
- *
1300
- * For both **behind** cases we hold no usable revision of the committed blocks, so we
1301
- * actively reconcile: pull the committed revision from a cohort peer that holds it
1302
- * (`reconcileBlock`) and restore it locally. Lazy read-repair on a later read cannot
1303
- * recover it on its own when cohort drift has left the block under-replicated (no
1304
- * reachable peer the reader sees holds the newer rev), so reconciling here is what
1305
- * keeps cross-cohort transactions converging. For the **ahead** case we already hold
1306
- * the committed rev, so we tolerate the no-op without reconciling downward.
1307
- *
1308
- * Whatever happens, we must NOT reset the stream: throwing here would reset the
1309
- * cluster stream the coordinator is awaiting and surface as a spurious
1310
- * `StreamResetError`, sinking an otherwise-successful transaction. So divergence
1311
- * and any reconciliation failure is tolerated (logged for observability). A
1312
- * genuinely *invalid* pend can never get this far: it is rejected during the promise
1313
- * phase (`validatePendOperations`, which validates pend ops only commits carry no
1314
- * promise-phase validation).
1315
- *
1316
- * The propagate-vs-tolerate split keys off the failure's *nature* via `CommitResult`,
1317
- * not throw-vs-return: a missing pend (thrown "not found") or a stale/ahead commit
1318
- * (`success:false` with `missing`) is divergence and tolerated, whereas a genuine
1319
- * mid-commit `internalCommit` fault (`success:false` with a bare `reason`, no
1320
- * `missing`) is propagated so {@link handleConsensus} rolls back the executed marker
1321
- * and rethrows exactly like an unexpected thrown fault.
1322
- */
1323
- private async applyConsensusOperation(record: ClusterRecord, operation: RepoMessage['operations'][number]): Promise<void> {
1324
- const messageHash = record.messageHash;
1325
- if ('get' in operation) {
1326
- await this.storageRepo.get(operation.get);
1327
- return;
1328
- }
1329
- if ('cancel' in operation) {
1330
- await this.storageRepo.cancel(operation.cancel.actionRef);
1331
- return;
1332
- }
1333
- if ('pend' in operation) {
1334
- const result = await this.storageRepo.pend(operation.pend);
1335
- if (!result.success) {
1336
- log('cluster-member:consensus-pend-diverged', {
1337
- messageHash,
1338
- actionId: operation.pend.actionId,
1339
- reason: result.reason,
1340
- hasMissing: !!result.missing?.length,
1341
- hasPending: !!result.pending?.length
1342
- });
1343
- }
1344
- return;
1345
- }
1346
- if ('commit' in operation) {
1347
- const commit = operation.commit;
1348
- // Capture the consensus commit cert BEFORE applying to storage: StorageRepo.commit emits the
1349
- // CollectionChangeEvent synchronously at the end of the call below, and the reactivity bridge
1350
- // resolves the cert from that event — so it must already be retained when commit() returns.
1351
- //
1352
- // The commit-vote signed preimage is computed HERE, before the synchronous capture+commit
1353
- // sequence below: it is the exact bytes each approving signer endorsed
1354
- // (`utf8(commitHash + ":approve")`), reproduced from this already-validated `record` (its
1355
- // commit signatures were verified against the same `computeCommitHash`). Reactivity sets a
1356
- // notification's `digest` from it so a subscriber's threshold-verify over `digest` succeeds.
1357
- // The one `await` happens before `captureCommitCert` runs, so the cert is still retained
1358
- // synchronously before `commit()` emits its change event (do not move this past the commit).
1359
- // Gated on the sink: with no reactivity wired the preimage has no consumer, so a sink-less
1360
- // node pays neither the extra `sha256` nor the extra microtask — the true zero-cost default.
1361
- if (this.onCommitCertificate) {
1362
- const commitSignedPayload = clusterVoteSigningPayload(await this.computeCommitHash(record), 'approve');
1363
- this.captureCommitCert(record, commit.actionId, commitSignedPayload);
1364
- }
1365
- let result: CommitResult;
1366
- try {
1367
- result = await this.storageRepo.commit(commit);
1368
- } catch (err) {
1369
- // `StorageRepo.commit` throws (rather than returning success:false) when
1370
- // the pending action is missing the canonical "behind" signal: this
1371
- // member reached commit-consensus without the matching pend (cohort drift).
1372
- if (isMissingPendingActionError(err)) {
1373
- log('cluster-member:consensus-commit-diverged', {
1374
- messageHash,
1375
- actionId: commit.actionId,
1376
- divergence: 'behind',
1377
- reason: (err as Error).message
1378
- });
1379
- // We hold no revision of these blocks; pull the committed revision from a
1380
- // cohort peer so the block is no longer under-replicated. Best-effort:
1381
- // failures are logged inside, never thrown (a throw would reset the stream).
1382
- await this.reconcileDivergentCommit(record, commit);
1383
- return;
1384
- }
1385
- throw err;
1386
- }
1387
- if (!result.success) {
1388
- // success:false is a StaleFailure. `missing` ⇒ ahead/stale divergence
1389
- // (we already hold this rev): tolerate, do NOT reconcile downward. A
1390
- // missing-base reason behind divergence, reconcile (below). Any other bare
1391
- // `reason` with no `missing` a genuine internalCommit fault: propagate so
1392
- // handleConsensus rolls back the executed marker and rethrows.
1393
- if (result.missing?.length) {
1394
- log('cluster-member:consensus-commit-diverged', {
1395
- messageHash,
1396
- actionId: commit.actionId,
1397
- divergence: 'ahead',
1398
- reason: result.reason,
1399
- hasMissing: true
1400
- });
1401
- return;
1402
- }
1403
- // This member holds no materializable base for one of the blocks, so
1404
- // `StorageRepo.commit` REFUSED rather than record a revision it could never serve.
1405
- // Same "behind" divergence as a missing pend — and the same cure: pull the committed
1406
- // revision from a cohort peer. Reconciling here (after commit released its per-block
1407
- // latches) is what makes refusing safe; fetching inside the commit path would deadlock
1408
- // against the latch `saveReplicatedBlock` needs to persist what it fetched.
1409
- if (isMissingBaseRevisionFailure(result)) {
1410
- log('cluster-member:consensus-commit-diverged', {
1411
- messageHash,
1412
- actionId: commit.actionId,
1413
- divergence: 'behind',
1414
- reason: result.reason
1415
- });
1416
- await this.reconcileDivergentCommit(record, commit);
1417
- return;
1418
- }
1419
- throw new Error(`Consensus commit for action ${commit.actionId} failed: ${result.reason ?? 'unknown reason'}`);
1420
- }
1421
- return;
1422
- }
1423
- if ('invalidate' in operation) {
1424
- await this.applyConsensusInvalidation(record, operation.invalidate);
1425
- return;
1426
- }
1427
- }
1428
-
1429
- /**
1430
- * Applies a consensus-ordered invalidation on this member: dedup → certificate verification →
1431
- * capture the invalidation's commit cert (for reactivity reuse) → delegate the compensating write +
1432
- * durable log append to the injected {@link InvalidationApplySink}.
1433
- *
1434
- * Like every other branch of {@link applyConsensusOperation}, a failure here is tolerated rather
1435
- * than thrown — a throw would reset the cluster stream. A forged/sub-threshold certificate is
1436
- * rejected (and never reaches the sink); a sink fault is logged and its dedup marker rolled back
1437
- * so a re-broadcast can retry. The durable, authoritative dedup is the invalidation log entry the
1438
- * sink consults; the in-memory map is only a fast path.
1439
- */
1440
- private async applyConsensusInvalidation(record: ClusterRecord, request: InvalidateRequest): Promise<void> {
1441
- const messageHash = record.messageHash;
1442
- const dedupKey = `${request.invalidatedActionId}:${request.resolution.disputeId}`;
1443
- if (this.appliedInvalidations.has(dedupKey)) {
1444
- log('cluster-member:consensus-invalidate-duplicate', { messageHash, dedupKey });
1445
- return;
1446
- }
1447
-
1448
- // Certificate verification BEFORE apply — never trust the originator's say-so. Verify the proof
1449
- // against THIS request's own target: the votes are bound to the transaction the dispute resolved,
1450
- // so a genuine proof carried in a request that points at a different (innocent) action/blocks fails
1451
- // here (the network-facing replay boundary, #2). The proof is also bound to the legitimately-selected
1452
- // arbitrator set (#1): membership + the challenger's set signature gate it (layer 1), and when a
1453
- // recompute capability is wired we re-derive the eligible set from our topology and reject a forged
1454
- // one (layer 2). When we cannot reconstruct the historical topology, we accept on layer 1 and LOG
1455
- // that the invalidation was applied without full anchoring — the documented interim posture.
1456
- const certified = await verifyInvalidationCertificate(
1457
- request.resolution,
1458
- { invalidatedActionId: request.invalidatedActionId, blockIds: request.blockIds },
1459
- {
1460
- recomputeArbitratorSet: this.recomputeArbitratorSet,
1461
- onUnanchored: (info) => log('cluster-member:consensus-invalidate-unanchored', {
1462
- messageHash,
1463
- invalidatedActionId: request.invalidatedActionId,
1464
- disputeId: info.disputeId,
1465
- reason: info.reason,
1466
- arbitratorSetSize: info.arbitratorSet.length
1467
- })
1468
- }
1469
- );
1470
- if (!certified) {
1471
- log('cluster-member:consensus-invalidate-reject-certificate', {
1472
- messageHash,
1473
- invalidatedActionId: request.invalidatedActionId,
1474
- disputeId: request.resolution.disputeId,
1475
- outcome: request.resolution.outcome
1476
- });
1477
- return;
1478
- }
1479
-
1480
- // Capture the invalidation's own commit cert — the threshold signature the cohort produced for
1481
- // THIS consensus-ordered `invalidate` op (over `computeCommitHash(record)`), the reversal analogue
1482
- // of the commit-cert capture in {@link applyConsensusOperation}. Reactivity reuses it bit-for-bit
1483
- // as the invalidation notification's `sig` (never re-signed). Keyed on the deterministic
1484
- // {@link invalidationActionId} the invalidation's change event also carries, so the bridge's
1485
- // cert extractor resolves it. Gated on the sink a node with no reactivity wired pays nothing.
1486
- if (this.onCommitCertificate) {
1487
- const invSignedPayload = clusterVoteSigningPayload(await this.computeCommitHash(record), 'approve');
1488
- this.captureCommitCert(record, invalidationActionId(request.invalidatedActionId, request.resolution.disputeId), invSignedPayload);
1489
- }
1490
-
1491
- if (!this.onInvalidate) {
1492
- log('cluster-member:consensus-invalidate-no-sink', { messageHash, dedupKey });
1493
- return;
1494
- }
1495
-
1496
- this.appliedInvalidations.set(dedupKey, Date.now());
1497
- try {
1498
- await this.onInvalidate(request);
1499
- log('cluster-member:consensus-invalidate-applied', {
1500
- messageHash,
1501
- invalidatedActionId: request.invalidatedActionId,
1502
- disputeId: request.resolution.disputeId,
1503
- blockCount: request.blockIds.length
1504
- });
1505
- } catch (err) {
1506
- // Tolerate (don't reset the stream); roll back the marker so a re-broadcast retries.
1507
- this.appliedInvalidations.delete(dedupKey);
1508
- log('cluster-member:consensus-invalidate-sink-error', { messageHash, dedupKey, error: (err as Error).message });
1509
- }
1510
- }
1511
-
1512
- /**
1513
- * Record the consensus commit cert for `actionId` into the injected {@link CommitCertificateSink},
1514
- * if one is wired. The cert is built from the agreed `record.commits` (the per-member `approve`
1515
- * commit signatures), forwarded UNCHANGED for reactivity to reuse. `signedPayload` is the exact
1516
- * commit-vote preimage those signatures were produced over (`utf8(commitHash + ":approve")`),
1517
- * computed by the caller from the same `record` *before* the synchronous commit — reactivity sets a
1518
- * notification's `digest` from it so a real threshold-verify over `digest` succeeds. Stays
1519
- * **synchronous** (no `await` here) so the cert is retained before `StorageRepo.commit` emits its
1520
- * change event. No-op (zero cost) when no sink is configured; a throwing sink is isolated + logged so
1521
- * it can never break consensus.
1522
- */
1523
- private captureCommitCert(record: ClusterRecord, actionId: ActionId, signedPayload: Uint8Array): void {
1524
- if (!this.onCommitCertificate) {
1525
- return;
1526
- }
1527
- const minSigs = Math.ceil(Object.keys(record.peers).length * this.superMajorityThreshold);
1528
- try {
1529
- this.onCommitCertificate(actionId, buildCommitCert(record, minSigs, signedPayload));
1530
- } catch (err) {
1531
- log('cluster-member:commit-cert-sink-error', { actionId, error: (err as Error).message });
1532
- }
1533
- }
1534
-
1535
- /**
1536
- * After tolerating a "behind" commit divergence, pull the committed revision of
1537
- * each block from a cohort peer that holds it and restore it locally. Best-effort:
1538
- * a missing callback, an empty cohort, or a per-block failure/timeout is logged and
1539
- * tolerated never thrown, since a throw out of consensus execution resets the
1540
- * cluster stream.
1541
- */
1542
- private async reconcileDivergentCommit(record: ClusterRecord, commit: CommitRequest): Promise<void> {
1543
- if (!this.reconcileBlock) {
1544
- log('cluster-member:consensus-commit-reconcile-skip', { messageHash: record.messageHash, reason: 'no-callback' });
1545
- return;
1546
- }
1547
- const cohortPeerIds = Object.keys(record.peers).filter(id => id !== this.peerId.toString());
1548
- if (cohortPeerIds.length === 0) {
1549
- log('cluster-member:consensus-commit-reconcile-skip', { messageHash: record.messageHash, reason: 'no-cohort-peers' });
1550
- return;
1551
- }
1552
- const committed: ActionRev = { actionId: commit.actionId, rev: commit.rev };
1553
- await Promise.all(
1554
- commit.blockIds.map(blockId => this.reconcileOneBlock(record.messageHash, blockId, committed, cohortPeerIds))
1555
- );
1556
- }
1557
-
1558
- /** Reconcile a single block, bounding the awaited callback and swallowing failures. */
1559
- private async reconcileOneBlock(messageHash: string, blockId: BlockId, committed: ActionRev, cohortPeerIds: string[]): Promise<void> {
1560
- try {
1561
- await this.withReconcileTimeout(this.reconcileBlock!(blockId, committed, cohortPeerIds), blockId);
1562
- // "attempted", not "reconciled": the callback returns void, and a quorum decline is a
1563
- // normal, non-throwing outcome — so reaching here means the pass ran to completion, NOT
1564
- // that anything was restored. `reconcile:restored` (reconcile-block.ts) is the line that
1565
- // says the bytes actually landed; `reconcile:no-rev-quorum` / `reconcile:no-content-quorum`
1566
- // say they did not.
1567
- log('cluster-member:consensus-commit-reconcile-attempted', { messageHash, blockId, rev: committed.rev });
1568
- } catch (err) {
1569
- log('cluster-member:consensus-commit-reconcile-failed', {
1570
- messageHash,
1571
- blockId,
1572
- rev: committed.rev,
1573
- error: (err as Error).message
1574
- });
1575
- }
1576
- }
1577
-
1578
- /** Bound an awaited reconcile so a slow/unreachable cohort peer can't stall consensus. */
1579
- private withReconcileTimeout<T>(promise: Promise<T>, blockId: BlockId): Promise<T> {
1580
- let timer: NodeJS.Timeout | undefined;
1581
- const timeout = new Promise<never>((_, reject) => {
1582
- timer = setTimeout(
1583
- () => reject(new Error(`reconcile for block ${blockId} timed out after ${ReconcileTimeoutMs}ms`)),
1584
- ReconcileTimeoutMs
1585
- );
1586
- timer.unref();
1587
- });
1588
- return Promise.race([promise, timeout]).finally(() => {
1589
- if (timer) clearTimeout(timer);
1590
- });
1591
- }
1592
-
1593
- private async handleRejection(_record: ClusterRecord): Promise<void> {
1594
- // Clean up any resources - will be cleared by shouldPersist = false in the main flow
1595
- }
1596
-
1597
- private setupTimeouts(record: ClusterRecord): { promiseTimeout?: NodeJS.Timeout; resolutionTimeout?: NodeJS.Timeout } {
1598
- if (!record.message.expiration) {
1599
- return {};
1600
- }
1601
-
1602
- return {
1603
- promiseTimeout: setTimeout(
1604
- () => this.handleExpiration(record.messageHash),
1605
- record.message.expiration - Date.now()
1606
- ).unref(),
1607
- resolutionTimeout: setTimeout(
1608
- () => this.resolveWithPeers(record.messageHash),
1609
- record.message.expiration + 5000 - Date.now()
1610
- ).unref()
1611
- };
1612
- }
1613
-
1614
- /**
1615
- * Scan this member's reservation table (`activeTransactions`) for a held transaction that
1616
- * conflicts with `record` AND wins the deterministic race against it. Returns the winner's
1617
- * identity — `{ blockedBy: messageHash }` — rather than a bare boolean, because that identity is
1618
- * exactly what the resulting conflict vote must name (`Signature.conflictWith`); the old boolean
1619
- * lost it. `undefined` means no blocking conflict. Side-effectful on the way through: stale
1620
- * entries are swept, and a held transaction that LOSES the race to `record` is cleared.
1621
- */
1622
- private findConflict(record: ClusterRecord): { blockedBy: string } | undefined {
1623
- const now = Date.now();
1624
- const staleThresholdMs = 2000; // 2 seconds - allow more time for distributed consensus
1625
-
1626
- const incomingBlockIds = this.getAffectedBlockIds(record.message.operations);
1627
- log('cluster-member:findConflict-check', {
1628
- messageHash: record.messageHash,
1629
- activeCount: this.activeTransactions.size,
1630
- incomingBlockIds
1631
- });
1632
-
1633
- for (const [existingHash, state] of Array.from(this.activeTransactions.entries())) {
1634
- if (existingHash === record.messageHash) {
1635
- continue;
1636
- }
1637
-
1638
- const existingBlockIds = this.getAffectedBlockIds(state.record.message.operations);
1639
- log('cluster-member:findConflict-compare', {
1640
- existing: existingHash,
1641
- incoming: record.messageHash,
1642
- existingBlockIds,
1643
- incomingBlockIds
1644
- });
1645
-
1646
- // Clean up stale transactions that have been around too long
1647
- if (now - state.lastUpdate > staleThresholdMs) {
1648
- log('cluster-member:stale-cleanup', {
1649
- messageHash: existingHash,
1650
- age: now - state.lastUpdate
1651
- });
1652
- this.clearTransaction(existingHash);
1653
- continue;
1654
- }
1655
-
1656
- if (this.operationsConflict(state.record.message.operations, record.message.operations)) {
1657
- // Use race resolution to determine winner
1658
- const resolution = this.resolveRace(state.record, record);
1659
-
1660
- if (resolution === 'keep-existing') {
1661
- log('cluster-member:race-keep-existing', {
1662
- existing: existingHash,
1663
- incoming: record.messageHash
1664
- });
1665
- return { blockedBy: existingHash }; // Reject incoming, naming the winner
1666
- } else {
1667
- // Accept incoming, abort existing
1668
- log('cluster-member:race-accept-incoming', {
1669
- existing: existingHash,
1670
- incoming: record.messageHash
1671
- });
1672
- this.clearTransaction(existingHash);
1673
- continue; // Check other conflicts
1674
- }
1675
- }
1676
- }
1677
-
1678
- return undefined; // No blocking conflicts
1679
- }
1680
-
1681
- /** Number of *approve* promise votes on a record the count the commit rule uses. */
1682
- private static approvalCount(record: ClusterRecord): number {
1683
- return Object.values(record.promises).filter(s => s.type === 'approve').length;
1684
- }
1685
-
1686
- /**
1687
- * Resolve a race between two conflicting transactions. Total and deterministic, so every honest
1688
- * member computes the identical winner (the Theorem 1 Case-2 premise). Order:
1689
- * 1. more *approve* promise signatures wins (progress monotonicity see safety note below);
1690
- * 2. equal approval counts → higher aged priority wins (fairness — see {@link recordPriority});
1691
- * 3. still tied → higher message hash wins.
1692
- *
1693
- * The count is APPROVALS, not `promises` keys. `promises` is the vote map — a reject occupies a key
1694
- * there exactly as an approve does — so counting keys would treat a rejection as progress, letting a
1695
- * record that can never commit outrank (and therefore block, via {@link findConflict}) a fresh rival
1696
- * for the whole staleness window. Approvals is also the count the invariant below actually needs:
1697
- * the commit rule is `approvedPromises >= superMajority`, which never looks at rejections.
1698
- *
1699
- * Approval count is FIRST so this comparison never displaces a transaction that is further along.
1700
- * That restores the pre-priority safety invariant: a member commits purely on promise supermajority
1701
- * (`handleCommitNeeded` signs whenever `approvedPromises >= superMajority`; the commit path has NO
1702
- * conflict re-check), so `resolveRace` is the ONLY arbiter among concurrently-pending conflicts.
1703
- * With approvals-first, once transaction X holds a promise supermajority every conflicting rival Y has
1704
- * strictly fewer approvals Y can only match X's count by getting the intersecting quorum member to
1705
- * approve it, but that member already holds X at supermajority and `resolveRace(X, Y)` returns
1706
- * `keep-existing` on X's higher count, so it never does. By quorum intersection any Y-supermajority
1707
- * overlaps X's in ≥1 honest member, and that member rejects Y. One winner (docs/correctness.md
1708
- * Theorem 9). Priority-first would break this: it could displace an already-quorum-reached X for a
1709
- * higher-priority Y with fewer approvals, letting BOTH commit (split brain) — the regression fixed by
1710
- * ticket occ-priority-first-breaks-promise-monotonicity.
1711
- *
1712
- * Priority is now a tie-break that runs only at EQUAL approval counts, which is exactly the
1713
- * concurrent-starvation case aging targets (two fresh rivals, 0 promises each, otherwise coin-flipping
1714
- * on the hash). Priority still breaks those ties deterministically, so aging still solves the stated
1715
- * fairness problem in its common case. It only orders two *concurrently-pending* conflicts; it does NOT
1716
- * defer a fresh pend for an absent aged transaction (that residual — sequential sub-window starvation —
1717
- * is the deferred feat-occ-priority-reservation).
1718
- *
1719
- * NOTE: residual-fairness tripwire. Under approvals-first an aged transaction can still lose to a fresh
1720
- * rival that has *legitimately* gathered even one more approval that is not the pure-coin-flip
1721
- * starvation aging targets (equal counts, priority wins), it is the monotonicity behaviour we WANT (a
1722
- * more-progressed rival is never displaced). If deeper fairness against a genuinely-more-progressed
1723
- * rival is ever needed, it belongs to feat-occ-priority-reservation (reserve/defer at pend time), NOT
1724
- * to this race tie-break.
1725
- *
1726
- * NOTE: Byzantine self-assert is a fairness DoS, not a safety hole. A coordinator can stamp
1727
- * priority == MaxPriority on every transaction; recordPriority clamps to the cap so it cannot
1728
- * exceed it, and priority never influences validity/operationsHash/stale-read checks and now sits
1729
- * below the approval count, so it can only break equal-count ties it might have ~50% won anyway,
1730
- * degrading to at-worst-status-quo fairness (the same graceful-degradation class as spam under
1731
- * honest-majority). Binding priority to provable age is out of scope (feat-occ-priority-reservation).
1732
- *
1733
- * NOTE: keep priority a self-contained additive message field + this one comparison key so it
1734
- * composes with does not block — a future HLC/crdt-sync redesign of this same path
1735
- * (design-hot-log-tail-sharding-guidance).
1736
- */
1737
- private resolveRace(existing: ClusterRecord, incoming: ClusterRecord): 'keep-existing' | 'accept-incoming' {
1738
- // 1. Transaction with more APPROVALS wins — never displace a more-progressed rival (safety, see
1739
- // above). Counting `promises` keys instead would count reject votes as progress: a record holding
1740
- // one rejection would outrank an untouched rival and reserve its blocks for the whole staleness
1741
- // window, and the commit rule this ordering protects (`approvedPromises >= superMajority`) never
1742
- // looks at rejections anyway.
1743
- const existingCount = ClusterMember.approvalCount(existing);
1744
- const incomingCount = ClusterMember.approvalCount(incoming);
1745
- if (existingCount !== incomingCount) {
1746
- return existingCount > incomingCount ? 'keep-existing' : 'accept-incoming';
1747
- }
1748
-
1749
- // 2. Equal approval counts higher aged priority wins (fairness tie-break).
1750
- const existingPriority = this.recordPriority(existing);
1751
- const incomingPriority = this.recordPriority(incoming);
1752
- if (existingPriority !== incomingPriority) {
1753
- return existingPriority > incomingPriority ? 'keep-existing' : 'accept-incoming';
1754
- }
1755
-
1756
- // 3. Tie-breaker: higher message hash wins (deterministic).
1757
- return existing.messageHash > incoming.messageHash ? 'keep-existing' : 'accept-incoming';
1758
- }
1759
-
1760
- /**
1761
- * Aged advisory priority carried by a record's pend operation, clamped to [0, MaxPriority].
1762
- * The multi-collection path carries it on `pend.transaction.priority`; the single-collection
1763
- * (`Collection.sync`) path carries it as top-level `pend.priority`; a record with neither — a
1764
- * legacy/unversioned coordinator's transaction, or a non-pend operation — is priority 0
1765
- * (backward compatible: such transactions simply never age). Both carriers live inside the signed
1766
- * `message`, so priority is integrity-protected in transit; clamping here bounds a self-asserted
1767
- * out-of-range value to the cap.
1768
- *
1769
- * NOTE: `message` is fixed for a transaction's whole lifecycle (promises/commits accrue in the
1770
- * separate `promises`/`commits` maps, never in `message`), so a transaction keeps its rank through
1771
- * the commit phase — there is no "priority drops to 0 at commit" asymmetry. resolveRace is only
1772
- * consulted at the promise decision (findConflict), i.e. between two still-open conflicting
1773
- * transactions, which is exactly the concurrent-contention case priority is meant to order.
1774
- */
1775
- private recordPriority(record: ClusterRecord): number {
1776
- for (const op of record.message.operations) {
1777
- if ('pend' in op) {
1778
- return clampPriority(op.pend.transaction?.priority ?? op.pend.priority);
1779
- }
1780
- }
1781
- return 0;
1782
- }
1783
-
1784
- private operationsConflict(ops1: RepoMessage['operations'], ops2: RepoMessage['operations']): boolean {
1785
- // Check if one is a commit for the same action as a pend - these don't conflict
1786
- const actionId1 = this.getActionId(ops1);
1787
- const actionId2 = this.getActionId(ops2);
1788
- if (actionId1 && actionId2 && actionId1 === actionId2) {
1789
- // Same action - commit is resolving the pend, not conflicting
1790
- return false;
1791
- }
1792
-
1793
- const blocks1 = new Set(this.getAffectedBlockIds(ops1));
1794
- const blocks2 = new Set(this.getAffectedBlockIds(ops2));
1795
-
1796
- for (const block of Array.from(blocks1)) {
1797
- if (blocks2.has(block)) {
1798
- log('cluster-member:conflict-detected', {
1799
- blocks1: Array.from(blocks1),
1800
- blocks2: Array.from(blocks2),
1801
- conflictingBlock: block
1802
- });
1803
- return true;
1804
- }
1805
- }
1806
-
1807
- return false;
1808
- }
1809
-
1810
- private getActionId(operations: RepoMessage['operations']): string | undefined {
1811
- for (const operation of operations) {
1812
- if ('pend' in operation) {
1813
- return operation.pend.actionId;
1814
- } else if ('commit' in operation) {
1815
- return operation.commit.actionId;
1816
- } else if ('cancel' in operation) {
1817
- return operation.cancel.actionRef.actionId;
1818
- }
1819
- }
1820
- return undefined;
1821
- }
1822
-
1823
- private getAffectedBlockIds(operations: RepoMessage['operations']): string[] {
1824
- const blockIds = new Set<string>();
1825
-
1826
- for (const operation of operations) {
1827
- if ('get' in operation) {
1828
- operation.get.blockIds.forEach(id => blockIds.add(id));
1829
- } else if ('pend' in operation) {
1830
- // Use blockIdsForTransforms to correctly extract block IDs from Transforms structure
1831
- blockIdsForTransforms(operation.pend.transforms).forEach(id => blockIds.add(id));
1832
- } else if ('commit' in operation) {
1833
- operation.commit.blockIds.forEach(id => blockIds.add(id));
1834
- } else if ('cancel' in operation) {
1835
- operation.cancel.actionRef.blockIds.forEach(id => blockIds.add(id));
1836
- } else if ('invalidate' in operation) {
1837
- // The invalidation writes compensating revisions to these blocks; surfacing them lets
1838
- // conflict detection serialize a concurrent commit racing the invalidation on a block.
1839
- operation.invalidate.blockIds.forEach(id => blockIds.add(id));
1840
- }
1841
- }
1842
-
1843
- return Array.from(blockIds);
1844
- }
1845
-
1846
- private async propagateIfNeeded(record: ClusterRecord): Promise<void> {
1847
- const promises = [];
1848
- for (const peerId of Object.keys(record.peers)) {
1849
- if (peerId === this.peerId.toString()) continue;
1850
-
1851
- try {
1852
- const client = ClusterClient.create(peerIdFromString(peerId), this.peerNetwork, this.protocolPrefix);
1853
- promises.push(client.update(record));
1854
- } catch (error) {
1855
- log('ERROR: Failed to propagate to peer %s: %o', peerId, error);
1856
- }
1857
- }
1858
- await Promise.allSettled(promises);
1859
- }
1860
-
1861
- private async handleExpiration(messageHash: string): Promise<void> {
1862
- const state = this.activeTransactions.get(messageHash);
1863
- if (!state) return;
1864
-
1865
- if (!state.record.promises[this.peerId.toString()]) {
1866
- const rejectReason = 'Transaction expired';
1867
- const promiseHash = await this.computePromiseHash(state.record);
1868
- const sig = await this.signVote(promiseHash, 'reject', rejectReason);
1869
- const signature: Signature = {
1870
- type: 'reject',
1871
- signature: sig,
1872
- rejectReason
1873
- };
1874
-
1875
- const updatedRecord = {
1876
- ...state.record,
1877
- promises: {
1878
- ...state.record.promises,
1879
- [this.peerId.toString()]: signature
1880
- }
1881
- };
1882
-
1883
- this.activeTransactions.set(messageHash, {
1884
- ...state,
1885
- record: updatedRecord
1886
- });
1887
-
1888
- await this.propagateIfNeeded(updatedRecord);
1889
- }
1890
- }
1891
-
1892
- private async resolveWithPeers(messageHash: string): Promise<void> {
1893
- // This method is disabled - the coordinator handles all retry logic
1894
- // Keeping the skeleton in case we need peer-initiated recovery in the future
1895
- log('cluster-member:resolve-skipped', { messageHash, reason: 'coordinator-handles-retry' });
1896
- }
1897
-
1898
- private queueExpiredTransactions(): void {
1899
- const now = Date.now();
1900
- for (const [messageHash, state] of Array.from(this.activeTransactions.entries())) {
1901
- if (state.record.message.expiration && state.record.message.expiration < now) {
1902
- this.cleanupQueue.push(messageHash);
1903
- }
1904
- }
1905
- // Also clean up old executed transaction records
1906
- const expirationThreshold = now - ExecutedTransactionTtlMs;
1907
- for (const [messageHash, executedAt] of Array.from(this.executedTransactions.entries())) {
1908
- if (executedAt < expirationThreshold) {
1909
- this.executedTransactions.delete(messageHash);
1910
- }
1911
- }
1912
- // Prune old applied-invalidation dedup markers on the same TTL.
1913
- for (const [dedupKey, appliedAt] of Array.from(this.appliedInvalidations.entries())) {
1914
- if (appliedAt < expirationThreshold) {
1915
- this.appliedInvalidations.delete(dedupKey);
1916
- }
1917
- }
1918
- this.stateStore?.pruneExecuted(expirationThreshold)
1919
- .catch(err => log('cluster-member:prune-executed-error', { error: (err as Error).message }));
1920
- }
1921
-
1922
- private async processCleanupQueue(): Promise<void> {
1923
- while (this.cleanupQueue.length > 0) {
1924
- const messageHash = this.cleanupQueue.shift();
1925
- if (!messageHash) continue;
1926
-
1927
- const state = this.activeTransactions.get(messageHash);
1928
- if (!state) continue;
1929
-
1930
- // NOTE: an expired entry already in a terminal phase is deliberately left alone here —
1931
- // `processUpdate` clears those on the update that made them terminal, so reaching this point
1932
- // in one means that update never arrived. It is not stranded: `findConflict`'s 2 s staleness
1933
- // sweep drops it on the next conflicting arrival. But on a member that then goes idle the
1934
- // entry lingers until traffic returns. If member memory ever shows entries outliving their
1935
- // expiration, delete unconditionally here instead of exempting the terminal phases.
1936
- const { phase } = await this.getTransactionPhase(state.record);
1937
- if (phase !== TransactionPhase.Consensus && phase !== TransactionPhase.Rejected) {
1938
- this.activeTransactions.delete(messageHash);
1939
- }
1940
- }
1941
- }
1942
-
1943
- private hasLocalCommit(record: ClusterRecord): boolean {
1944
- const ourId = this.peerId.toString();
1945
- return Boolean(record.commits[ourId]);
1946
- }
1947
-
1948
- private clearTransaction(messageHash: string): void {
1949
- const state = this.activeTransactions.get(messageHash);
1950
- if (!state) {
1951
- log('cluster-member:clear-miss', { messageHash });
1952
- return;
1953
- }
1954
- if (state.promiseTimeout) {
1955
- clearTimeout(state.promiseTimeout);
1956
- }
1957
- if (state.resolutionTimeout) {
1958
- clearTimeout(state.resolutionTimeout);
1959
- }
1960
- this.activeTransactions.delete(messageHash);
1961
- this.stateStore?.deleteParticipantState(messageHash)
1962
- .catch(err => log('cluster-member:persist-delete-error', { messageHash, error: (err as Error).message }));
1963
- log('cluster-member:clear-done', {
1964
- messageHash,
1965
- remaining: Array.from(this.activeTransactions.keys())
1966
- });
1967
- }
1968
-
1969
- /** Fire-and-forget persist errors are logged, never thrown. */
1970
- private persistParticipantState(messageHash: string, record: ClusterRecord): void {
1971
- if (!this.stateStore) return;
1972
- this.stateStore.saveParticipantState(messageHash, {
1973
- messageHash,
1974
- record,
1975
- lastUpdate: Date.now()
1976
- }).catch(err => log('cluster-member:persist-error', { messageHash, error: (err as Error).message }));
1977
- }
1978
-
1979
- /**
1980
- * Recover member transactions from persistent store after a restart.
1981
- * Called during node startup, before accepting new requests.
1982
- */
1983
- async recoverTransactions(): Promise<void> {
1984
- if (!this.stateStore) return;
1985
- const now = Date.now();
1986
-
1987
- // 1. Prune expired executed entries from persistent store
1988
- await this.stateStore.pruneExecuted(now - ExecutedTransactionTtlMs);
1989
- // Note: executed transactions are checked via wasTransactionExecutedAsync() at runtime,
1990
- // which falls back to the persistent store when the in-memory map misses.
1991
-
1992
- // 2. Restore active participant states
1993
- const participantStates = await this.stateStore.getAllParticipantStates();
1994
- for (const state of participantStates) {
1995
- const { messageHash } = state;
1996
- // Expired clean up
1997
- if (state.record.message.expiration && state.record.message.expiration < now) {
1998
- log('cluster-member:recovery-expired', { messageHash });
1999
- await this.stateStore.deleteParticipantState(messageHash);
2000
- continue;
2001
- }
2002
- // Restore into activeTransactions with fresh timeouts
2003
- log('cluster-member:recovery-restore', { messageHash });
2004
- const timeouts = this.setupTimeouts(state.record);
2005
- this.activeTransactions.set(messageHash, {
2006
- record: state.record,
2007
- lastUpdate: state.lastUpdate,
2008
- promiseTimeout: timeouts.promiseTimeout,
2009
- resolutionTimeout: timeouts.resolutionTimeout
2010
- });
2011
- }
2012
-
2013
- log('cluster-member:recovery-complete', {
2014
- restoredActive: this.activeTransactions.size,
2015
- restoredExecuted: this.executedTransactions.size
2016
- });
2017
- }
2018
-
2019
- /**
2020
- * Checks if a transaction's operations were already executed during consensus.
2021
- * Falls back to the persistent store when the in-memory map misses.
2022
- */
2023
- async wasTransactionExecutedAsync(messageHash: string): Promise<boolean> {
2024
- if (this.executedTransactions.has(messageHash)) return true;
2025
- if (!this.stateStore) return false;
2026
- const persisted = await this.stateStore.wasExecuted(messageHash);
2027
- if (persisted) {
2028
- // Re-populate in-memory map for future synchronous checks
2029
- this.executedTransactions.set(messageHash, Date.now());
2030
- }
2031
- return persisted;
2032
- }
2033
- }
2034
-
1
+ import type { IRepo, ClusterRecord, ClusterPeers, Signature, RepoMessage, ITransactionValidator, ClusterConsensusConfig, UnvalidatablePendPolicy, CommitResult, PendResult, BlockId, ActionId, ActionRev, CommitRequest, CommitCert, InvalidateRequest } from "@optimystic/db-core";
2
+ import type { ICluster } from "@optimystic/db-core";
3
+ import type { IPeerNetwork } from "@optimystic/db-core";
4
+ import { blockIdsForTransforms, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
5
+ import { computeClusterCommitHash, computeClusterMessageHash, computeClusterPromiseHash, membershipDigest, recordMembershipDigest, clampPriority, clusterVoteSigningPayload, clusterVoteVerificationPayload } from "@optimystic/db-core";
6
+ import { verifyInvalidationCertificate, type ArbitratorSetRecompute } from "../dispute/invalidation.js";
7
+ import { buildCommitCert, invalidationActionId } from "./commit-cert.js";
8
+ import { ClusterClient } from "./client.js";
9
+ import type { PeerId, PrivateKey } from "@libp2p/interface";
10
+ import { peerIdFromString } from "@libp2p/peer-id";
11
+ import { publicKeyFromRaw } from "@libp2p/crypto/keys";
12
+ import { peerIdBindsPublicKey } from "./peer-key-binding.js";
13
+ import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } from 'uint8arrays';
14
+ import { createLogger } from '../logger.js'
15
+ import type { PartitionDetector } from "./partition-detector.js";
16
+ import type { FretService } from "p2p-fret";
17
+ import type { IPeerReputation } from "../reputation/types.js";
18
+ import { PenaltyReason } from "../reputation/types.js";
19
+ import type { ITransactionStateStore } from "./i-transaction-state-store.js";
20
+ import { isMissingBaseRevisionFailure, type CommitDigestPreview, type ICommitDigestPreviewer, type ICommitProofPersister, type IRevisionActionReader } from "../storage/storage-repo.js";
21
+ import { checkPendValidation } from "../pend-validation.js";
22
+ import { buildBlockCommitProof } from "./commit-proof.js";
23
+ import { RECONCILE_TIMEOUT_MS } from "./reconcile-block.js";
24
+
25
+ const log = createLogger('cluster-member')
26
+
27
+ /** State of a transaction in the cluster */
28
+ enum TransactionPhase {
29
+ Promising, // We have voted; still collecting promises from other peers
30
+ OurPromiseNeeded, // We need to provide our promise
31
+ OurConflictVoteNeeded, // We hold a conflicting race winner; we must answer with a conflict vote
32
+ OurCommitNeeded, // We need to provide our commit
33
+ Consensus, // Transaction has reached consensus
34
+ Rejected, // Transaction was rejected (validity judgement — enough reject votes)
35
+ ConflictSuperseded, // Terminal but retryable: conflict votes make super-majority unreachable
36
+ Propagating // Transaction is being propagated
37
+ }
38
+
39
+ /**
40
+ * A phase plus the data its handler needs. Only {@link TransactionPhase.OurConflictVoteNeeded}
41
+ * carries any `conflictsWith`, the winning rival's messageHash so the conflict-vote handler need
42
+ * not re-run conflict detection (whose race resolution has side effects) to learn what blocked it.
43
+ * Split by phase rather than an optional field so the handler reads it without an assertion, and so
44
+ * a future phase-with-data cannot silently inherit this one's payload.
45
+ */
46
+ type PhaseResult =
47
+ | { phase: TransactionPhase.OurConflictVoteNeeded; conflictsWith: string }
48
+ | { phase: Exclude<TransactionPhase, TransactionPhase.OurConflictVoteNeeded> };
49
+
50
+ interface TransactionState {
51
+ record: ClusterRecord;
52
+ promiseTimeout?: NodeJS.Timeout;
53
+ resolutionTimeout?: NodeJS.Timeout;
54
+ lastUpdate: number;
55
+ }
56
+
57
+ /**
58
+ * Result of verifying one vote signature. `penalize` distinguishes "identity was never proven"
59
+ * (no key / not Ed25519 / key not bound to the peer id / malformed input reject, but do NOT
60
+ * report the named peer, whose id may have been attacker-chosen) from "the key IS the one the
61
+ * peer id names, yet the signature does not verify" (reject AND report — a genuine bad vote from
62
+ * that proven identity). Collapsing both into a bare `false` would let a coordinator get an honest
63
+ * peer penalized just by attaching a key it controls under that peer's id.
64
+ */
65
+ type VerifyOutcome =
66
+ | { valid: true }
67
+ | { valid: false; penalize: boolean };
68
+
69
+ /**
70
+ * Actively reconciles a block this member committed without having seen the matching
71
+ * pend (cohort drift between the independent pend and commit cluster-transactions).
72
+ * Pulls the committed revision from a cohort peer that holds it and restores it into
73
+ * local storage. Injected so {@link ClusterMember} stays transport-agnostic — mirrors
74
+ * how `CoordinatorRepo` receives its `clusterLatestCallback`.
75
+ *
76
+ * @param blockId the under-replicated block to restore
77
+ * @param committed the committed `(actionId, rev)` agreed by consensus
78
+ * @param cohortPeerIds cohort members to pull from (self already excluded)
79
+ */
80
+ export type ReconcileBlockCallback = (blockId: BlockId, committed: ActionRev, cohortPeerIds: string[]) => Promise<void>;
81
+
82
+ /**
83
+ * Sink for the {@link CommitCert} this member assembled at consensus, fired once per committed action
84
+ * **before** the commit is applied to local storage (so it is already retained when
85
+ * {@link StorageRepo.commit} emits the matching `CollectionChangeEvent` the reactivity bridge reads).
86
+ * The bytes are the cluster's own `approve` commit signatures, forwarded UNCHANGED — never re-signed.
87
+ * Optional; absent on nodes that do not originate reactivity notifications. A throwing sink is
88
+ * isolated + logged (it must never break consensus).
89
+ */
90
+ export type CommitCertificateSink = (actionId: ActionId, cert: CommitCert) => void;
91
+
92
+ /**
93
+ * Applies a consensus-ordered {@link InvalidateRequest} to local storage the deterministic
94
+ * reversal every member runs once consensus on the invalidation is reached. The implementation
95
+ * recomputes the per-block as-if-`T_inv`-absent content, writes the compensating revisions, and
96
+ * appends the durable invalidation log entry (see `applyInvalidation` in the dispute module). It
97
+ * is injected so {@link ClusterMember} stays storage/log-agnostic (mirrors {@link ReconcileBlockCallback}).
98
+ *
99
+ * The certificate is verified by {@link ClusterMember} *before* the sink is invoked, so a sink
100
+ * implementation may assume `request.resolution` is already a valid challenger-wins certificate.
101
+ * A throwing sink is logged and tolerated (never resets the cluster stream); the in-memory dedup
102
+ * marker is rolled back so a re-broadcast can retry.
103
+ */
104
+ export type InvalidationApplySink = (request: InvalidateRequest) => Promise<void>;
105
+
106
+ /**
107
+ * Optional **layer-2** capability for invalidation-certificate verification: re-derives the
108
+ * legitimately-selected arbitrator set from this member's topology view and judges the carried set
109
+ * (see {@link ArbitratorSetRecompute}). Injected so {@link ClusterMember} stays network-agnostic; the
110
+ * composition root supplies it from FRET when available. When absent (or when the member cannot
111
+ * reconstruct the historical topology), {@link applyConsensusInvalidation} accepts a layer-1-valid
112
+ * certificate and logs that it applied an invalidation it could not fully anchor — the documented
113
+ * interim posture (see `tickets/plan/cohort-topic-membership-cert-trust-anchoring.md`).
114
+ */
115
+ export type RecomputeArbitratorSetCapability = ArbitratorSetRecompute;
116
+
117
+ /**
118
+ * The member's own independently-derived view of a block's responsible cluster, plus FRET's confidence in
119
+ * the underlying network-size estimate (0..1). Returned by {@link DeriveExpectedClusterCallback} and
120
+ * consumed by {@link ClusterMember}'s membership admission gate: the member checks a coordinator-declared
121
+ * peer set against `peers` (its expected set `E`) and gates on `confidence` (low confidence ⇒ fail closed
122
+ * for any downsizing, the partition posture).
123
+ */
124
+ export type ExpectedClusterView = {
125
+ /** The member's own derived responsible-peer set for the block (its view of the legitimate cluster). */
126
+ peers: ClusterPeers;
127
+ /** FRET's confidence in the current network-size estimate (0..1); ≤ threshold ⇒ treated as untrusted. */
128
+ confidence: number;
129
+ };
130
+
131
+ /**
132
+ * Independently derive this member's own view of a block's responsible cluster. Injected so
133
+ * {@link ClusterMember} stays transport-agnostic the composition root supplies it from
134
+ * `IKeyNetwork.findCluster` + FRET (mirroring how the coordinator derives the cluster). Absent on nodes
135
+ * that cannot derive a view (no FRET, unit tests): with no derived view AND no asserted
136
+ * {@link ClusterConsensusConfig.assumedClusterSize} the gate preserves legacy approve behavior, but an
137
+ * asserted size still lets the gate fail closed on an unjustified downsize. See {@link ClusterMember}
138
+ * admission gate.
139
+ */
140
+ export type DeriveExpectedClusterCallback = (blockId: BlockId) => Promise<ExpectedClusterView>;
141
+
142
+ /** Stable reject reason a member emits when a declared peer set fails the membership admission gate. */
143
+ export const MEMBERSHIP_NOT_ADMITTED = 'membership-not-admitted';
144
+
145
+ /**
146
+ * Stable reject reason a member emits when its own materialization of a commit's block disagrees
147
+ * with the content digest the transaction author declared (`CommitRequest.blockDigests`). Rides in
148
+ * the reject vote's `rejectReason`, which `clusterVoteSigningPayload` folds into the signed bytes —
149
+ * so the rejection itself is integrity-protected.
150
+ */
151
+ export const CONTENT_DIGEST_MISMATCH = 'content-digest-mismatch';
152
+
153
+ /**
154
+ * The two stable reject reasons a validator-configured member emits from the shared
155
+ * {@link checkPendValidation}: `PEND_NOT_VALIDATABLE` for a pend carrying no `validation` payload
156
+ * under `ClusterConsensusConfig.unvalidatablePendPolicy: 'reject'`, and `VALIDATOR_FAULT` for a
157
+ * checker that threw. Defined in `pend-validation.ts` (which the storage tier runs too, so both
158
+ * tiers refuse with the same prefixes) and re-exported here next to its siblings above.
159
+ */
160
+ export { PEND_NOT_VALIDATABLE, VALIDATOR_FAULT } from "../pend-validation.js";
161
+
162
+ interface ClusterMemberComponents {
163
+ storageRepo: IRepo;
164
+ peerNetwork: IPeerNetwork;
165
+ peerId: PeerId;
166
+ privateKey: PrivateKey;
167
+ protocolPrefix?: string;
168
+ // Reserved for partition-healing consumers (backlog ticket 6.5-partition-healing); not yet read by ClusterMember.
169
+ partitionDetector?: PartitionDetector;
170
+ fretService?: FretService;
171
+ validator?: ITransactionValidator;
172
+ reputation?: IPeerReputation;
173
+ consensusConfig?: ClusterConsensusConfig;
174
+ stateStore?: ITransactionStateStore;
175
+ /** Restores a block under-replicated by cohort drift; see {@link ReconcileBlockCallback}. */
176
+ reconcileBlock?: ReconcileBlockCallback;
177
+ /** Receives the consensus commit cert per committed action; see {@link CommitCertificateSink}. */
178
+ onCommitCertificate?: CommitCertificateSink;
179
+ /** Applies a consensus-ordered invalidation to local storage; see {@link InvalidationApplySink}. */
180
+ onInvalidate?: InvalidationApplySink;
181
+ /** Layer-2 arbitrator-set recompute for invalidation verification; see {@link RecomputeArbitratorSetCapability}. */
182
+ recomputeArbitratorSet?: RecomputeArbitratorSetCapability;
183
+ /** Member-side cluster derivation for the membership admission gate; see {@link DeriveExpectedClusterCallback}. */
184
+ deriveExpectedCluster?: DeriveExpectedClusterCallback;
185
+ }
186
+
187
+ export function clusterMember(components: ClusterMemberComponents): ClusterMember {
188
+ return new ClusterMember(
189
+ components.storageRepo,
190
+ components.peerNetwork,
191
+ components.peerId,
192
+ components.privateKey,
193
+ components.protocolPrefix,
194
+ components.partitionDetector,
195
+ components.fretService,
196
+ components.validator,
197
+ components.reputation,
198
+ components.consensusConfig,
199
+ components.stateStore,
200
+ components.reconcileBlock,
201
+ components.onCommitCertificate,
202
+ components.onInvalidate,
203
+ components.recomputeArbitratorSet,
204
+ components.deriveExpectedCluster
205
+ );
206
+ }
207
+
208
+ // How long to keep executed transaction records (10 minutes)
209
+ const ExecutedTransactionTtlMs = 10 * 60 * 1000;
210
+
211
+ // Upper bound on an awaited active reconciliation of a divergent commit. Bounds the
212
+ // consensus path so a slow/unreachable cohort peer can't stall the cluster stream;
213
+ // a timeout is logged and tolerated (never thrown — that would reset the stream).
214
+ // Shared with the read path's acquisition (see RECONCILE_TIMEOUT_MS) — same operation, same bound.
215
+ const ReconcileTimeoutMs = RECONCILE_TIMEOUT_MS;
216
+
217
+ /**
218
+ * True when a thrown storage error reports a missing pending action — i.e. this
219
+ * member reached commit-consensus without having seen the matching pend phase.
220
+ * That is recoverable local divergence (reconciled via sync / read-repair), not
221
+ * a transaction fault, so it must not reset the cluster stream.
222
+ */
223
+ function isMissingPendingActionError(err: unknown): boolean {
224
+ return err instanceof Error && /pending action .+ not found/i.test(err.message);
225
+ }
226
+
227
+ /**
228
+ * Handles cluster-side operations, managing promises and commits for cluster updates
229
+ * and coordinating with the local storage repo.
230
+ */
231
+ export class ClusterMember implements ICluster {
232
+ // Track active transactions by their message hash
233
+ private activeTransactions: Map<string, TransactionState> = new Map();
234
+ // Track executed consensus transactions to prevent duplicate execution (messageHash -> executedAt timestamp)
235
+ private executedTransactions: Map<string, number> = new Map();
236
+ // Local storage's verdict for a pend operation applied during consensus (messageHash -> PendResult).
237
+ // Retained so the coordinator can return storage's real answer instead of fabricating success —
238
+ // a pend that every member refused (rival pending action, or the revision already taken) must
239
+ // reach the writer as a conflict, not a win. Pruned alongside executedTransactions (same TTL).
240
+ private executedPendResults: Map<string, PendResult> = new Map();
241
+ // Local storage's verdict for a COMMIT operation applied during consensus (messageHash ->
242
+ // CommitResult). Retained so the coordinator can detect when the ahead-divergence tolerance in
243
+ // applyConsensusOperation swallowed a refusal whose real cause was a RIVAL action holding the
244
+ // requested revision the commit-tier acknowledgement hole: a commit that assembled consensus
245
+ // inside every member's signed-but-not-yet-applied window is refused by every member's storage
246
+ // at apply, and without this verdict the coordinator fabricates a success no member durably
247
+ // stored. Pruned alongside executedTransactions (same TTL).
248
+ private executedCommitResults: Map<string, CommitResult> = new Map();
249
+ // Fast in-memory dedup for applied invalidations, keyed `${invalidatedActionId}:${disputeId}`.
250
+ // The durable source of truth is the invalidation log entry (Log.findInvalidation, re-checked
251
+ // inside the sink); this map only spares redundant work when the same invalidation reaches
252
+ // consensus twice (rebroadcast / sync) under different message hashes. (-> appliedAt timestamp)
253
+ private appliedInvalidations: Map<string, number> = new Map();
254
+ // Queue of transactions to clean up
255
+ private cleanupQueue: string[] = [];
256
+ // Serialize concurrent updates for the same transaction
257
+ private pendingUpdates: Map<string, Promise<ClusterRecord>> = new Map();
258
+ // Interval handles for periodic cleanup (stored so dispose() can clear them)
259
+ private readonly expirationInterval: NodeJS.Timeout;
260
+ private readonly cleanupInterval: NodeJS.Timeout;
261
+
262
+ /**
263
+ * Confidence floor at/below which FRET's network-size view is treated as untrustworthy for the
264
+ * membership gate. Above it the member trusts its derived view (confident path); at/below it the gate
265
+ * fails closed for downsizing. Matches the coordinator's `validateSmallCluster` confidence gate (> 0.5).
266
+ */
267
+ private static readonly MembershipConfidenceThreshold = 0.5;
268
+
269
+ /**
270
+ * Effective super-majority threshold this member accepts as sufficient for a commit. Defaults to
271
+ * {@link DEFAULT_SUPER_MAJORITY_THRESHOLD} (0.75) when no config is supplied — the SAME default the
272
+ * coordinator uses, so the two can never silently disagree about whether a transaction is final.
273
+ * (Previously defaulted to 1.0/unanimity, which split the member from a coordinator committing at 0.75.)
274
+ */
275
+ private readonly superMajorityThreshold: number;
276
+ // Membership admission gate parameters (see {@link admitMembership}). Read once from consensusConfig
277
+ // so the gate has stable thresholds independent of the (untrusted) values a record declares.
278
+ private readonly minAbsoluteClusterSize: number;
279
+ private readonly clusterSizeTolerance: number;
280
+ private readonly membershipAdmissionFraction: number;
281
+ /** Operator-asserted smallest genuine cohort size, or undefined when unknown. */
282
+ private readonly assumedClusterSize: number | undefined;
283
+ private readonly allowUnvalidatedSmallCluster: boolean;
284
+ /** What a validator-configured member does with a pend carrying no `validation` payload — see
285
+ * {@link ClusterConsensusConfig.unvalidatablePendPolicy}. Read once, like the gate parameters. */
286
+ private readonly unvalidatablePendPolicy: UnvalidatablePendPolicy;
287
+
288
+ constructor(
289
+ private readonly storageRepo: IRepo,
290
+ private readonly peerNetwork: IPeerNetwork,
291
+ private readonly peerId: PeerId,
292
+ private readonly privateKey: PrivateKey,
293
+ private readonly protocolPrefix?: string,
294
+ // Reserved for partition-healing (backlog ticket 6.5-partition-healing); held but not yet consumed.
295
+ _partitionDetector?: PartitionDetector,
296
+ private readonly fretService?: FretService,
297
+ private readonly validator?: ITransactionValidator,
298
+ private readonly reputation?: IPeerReputation,
299
+ consensusConfig?: ClusterConsensusConfig,
300
+ private readonly stateStore?: ITransactionStateStore,
301
+ private readonly reconcileBlock?: ReconcileBlockCallback,
302
+ private readonly onCommitCertificate?: CommitCertificateSink,
303
+ private readonly onInvalidate?: InvalidationApplySink,
304
+ private readonly recomputeArbitratorSet?: RecomputeArbitratorSetCapability,
305
+ private readonly deriveExpectedCluster?: DeriveExpectedClusterCallback
306
+ ) {
307
+ this.superMajorityThreshold = consensusConfig?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD;
308
+ this.minAbsoluteClusterSize = consensusConfig?.minAbsoluteClusterSize ?? 3;
309
+ this.clusterSizeTolerance = consensusConfig?.clusterSizeTolerance ?? 0.5;
310
+ this.membershipAdmissionFraction = consensusConfig?.membershipAdmissionFraction ?? 0.75;
311
+ this.assumedClusterSize = consensusConfig?.assumedClusterSize;
312
+ this.allowUnvalidatedSmallCluster = consensusConfig?.allowUnvalidatedSmallCluster ?? false;
313
+ this.unvalidatablePendPolicy = consensusConfig?.unvalidatablePendPolicy ?? 'accept';
314
+ // State the resolved gate parameters once, so an operator diagnosing a membership rejection can see
315
+ // what this node actually resolved. A fact, not a warning: `assumedClusterSize < clusterSize` is the
316
+ // normal default state, so warning on it would fire for every node and be ignored.
317
+ log('cluster-member:admission-config', {
318
+ assumedClusterSize: this.assumedClusterSize,
319
+ minAbsoluteClusterSize: this.minAbsoluteClusterSize,
320
+ membershipAdmissionFraction: this.membershipAdmissionFraction,
321
+ allowUnvalidatedSmallCluster: this.allowUnvalidatedSmallCluster
322
+ });
323
+ // Periodically clean up expired transactions (.unref() so tests/short-lived processes can exit)
324
+ this.expirationInterval = setInterval(() => this.queueExpiredTransactions(), 60000);
325
+ this.expirationInterval.unref();
326
+ // Process cleanup queue
327
+ this.cleanupInterval = setInterval(() => this.processCleanupQueue(), 1000);
328
+ this.cleanupInterval.unref();
329
+ }
330
+
331
+ /**
332
+ * The resolved super-majority threshold this member runs on. Exposed so the composition root can
333
+ * fail-fast if the member and the coordinator would run different thresholds (see the coupling
334
+ * assertion in `libp2p-node-base.ts`) — a mismatch is a latent phase-disagreement, caught at startup
335
+ * rather than mid-consensus.
336
+ */
337
+ get effectiveSuperMajorityThreshold(): number {
338
+ return this.superMajorityThreshold;
339
+ }
340
+
341
+ /**
342
+ * Clears all interval and timeout handles and empties active state.
343
+ * Called during node shutdown to prevent leaked timers.
344
+ */
345
+ dispose(): void {
346
+ clearInterval(this.expirationInterval);
347
+ clearInterval(this.cleanupInterval);
348
+ for (const [, state] of this.activeTransactions) {
349
+ if (state.promiseTimeout) clearTimeout(state.promiseTimeout);
350
+ if (state.resolutionTimeout) clearTimeout(state.resolutionTimeout);
351
+ }
352
+ this.activeTransactions.clear();
353
+ this.cleanupQueue.length = 0;
354
+ this.executedPendResults.clear();
355
+ this.executedCommitResults.clear();
356
+ }
357
+
358
+ /**
359
+ * Checks if a transaction's operations were already executed during consensus.
360
+ * Used by the coordinator to avoid duplicate execution in CoordinatorRepo.
361
+ */
362
+ wasTransactionExecuted(messageHash: string): boolean {
363
+ return this.executedTransactions.has(messageHash);
364
+ }
365
+
366
+ /**
367
+ * Local storage's verdict for the pend operation this member applied at consensus for
368
+ * `messageHash`, when one was retained. The coordinator reads this so the answer a writer gets
369
+ * is the answer storage gave the cluster path must not fabricate a success the single-node
370
+ * path (`CoordinatorRepo.pend`'s `peerCount <= 1` short-circuit) would never produce. Absent for
371
+ * transactions carrying no pend operation, for transactions applied before this member restarted
372
+ * (the map is in-memory only; the coordinator then falls back to its fabricated-success shape,
373
+ * with the promise-phase pending check in {@link validatePendOperations} narrowing that window),
374
+ * and after the executed-transaction TTL prunes it.
375
+ */
376
+ getExecutedPendResult(messageHash: string): PendResult | undefined {
377
+ return this.executedPendResults.get(messageHash);
378
+ }
379
+
380
+ /**
381
+ * Commit-shaped sibling of {@link getExecutedPendResult}: local storage's verdict for a commit
382
+ * operation applied during consensus, when this member retained one. `CoordinatorRepo.commit`
383
+ * consults it after a locally-executed commit-consensus — a retained refusal whose cause a local
384
+ * re-read confirms as a rival holding the requested revision is returned to the writer as a
385
+ * retryable conflict instead of the fabricated success the ahead-divergence tolerance would
386
+ * otherwise imply. Same availability caveats as the pend accessor: in-memory only, absent for
387
+ * pre-restart applies, pruned on the executed-transaction TTL.
388
+ */
389
+ getExecutedCommitResult(messageHash: string): CommitResult | undefined {
390
+ return this.executedCommitResults.get(messageHash);
391
+ }
392
+
393
+ /**
394
+ * Handles an incoming cluster update, managing the two-phase commit process
395
+ * and coordinating with the local storage repo
396
+ */
397
+ async update(record: ClusterRecord): Promise<ClusterRecord> {
398
+ // Serialize concurrent updates for the same transaction
399
+ const existingUpdate = this.pendingUpdates.get(record.messageHash);
400
+ if (existingUpdate) {
401
+ log('cluster-member:concurrent-update-wait', { messageHash: record.messageHash });
402
+ await existingUpdate;
403
+ // After waiting, continue processing with the new incoming record
404
+ // to ensure proper merging of promises/commits from coordinator
405
+ }
406
+
407
+ // Create a promise for this update operation
408
+ const updatePromise = this.processUpdate(record);
409
+ this.pendingUpdates.set(record.messageHash, updatePromise);
410
+
411
+ try {
412
+ const result = await updatePromise;
413
+ return result;
414
+ } finally {
415
+ // Remove from pending updates after a short delay to allow concurrent calls to see it
416
+ setTimeout(() => {
417
+ this.pendingUpdates.delete(record.messageHash);
418
+ }, 100).unref();
419
+ }
420
+ }
421
+
422
+ private async processUpdate(record: ClusterRecord): Promise<ClusterRecord> {
423
+ const ourId = this.peerId.toString();
424
+ const inboundPhase = record.commits[ourId] ? 'commit' : record.promises[ourId] ? 'promise' : 'initial';
425
+ log('cluster-member:incoming', {
426
+ messageHash: record.messageHash,
427
+ phase: inboundPhase,
428
+ peerCount: Object.keys(record.peers).length,
429
+ promiseCount: Object.keys(record.promises).length,
430
+ commitCount: Object.keys(record.commits).length,
431
+ existingTransaction: this.activeTransactions.has(record.messageHash)
432
+ });
433
+
434
+ // Report network size hint to FRET if provided
435
+ if (this.fretService && record.networkSizeHint && record.networkSizeConfidence) {
436
+ try {
437
+ this.fretService.reportNetworkSize(
438
+ record.networkSizeHint,
439
+ record.networkSizeConfidence,
440
+ 'cluster'
441
+ );
442
+ } catch (err) {
443
+ // Ignore errors reporting to FRET
444
+ }
445
+ }
446
+
447
+ // Validate the incoming record
448
+ await this.validateRecord(record);
449
+
450
+ const existingState = this.activeTransactions.get(record.messageHash);
451
+ let currentRecord = existingState?.record || record;
452
+ if (existingState) {
453
+ log('cluster-member:merge-start', {
454
+ messageHash: record.messageHash,
455
+ existingPromises: Object.keys(existingState.record.promises ?? {}),
456
+ existingCommits: Object.keys(existingState.record.commits ?? {}),
457
+ incomingPromises: Object.keys(record.promises ?? {}),
458
+ incomingCommits: Object.keys(record.commits ?? {})
459
+ });
460
+ }
461
+
462
+ // If we have an existing record, merge the signatures
463
+ if (existingState) {
464
+ currentRecord = await this.mergeRecords(existingState.record, record);
465
+ log('cluster-member:merge-complete', {
466
+ messageHash: record.messageHash,
467
+ mergedPromises: Object.keys(currentRecord.promises ?? {}),
468
+ mergedCommits: Object.keys(currentRecord.commits ?? {})
469
+ });
470
+ }
471
+
472
+ // Drive the phase machine to a FIXPOINT rather than handling one phase per delivery. Each
473
+ // vote this member adds can put the record straight into the next phase (our promise
474
+ // completes super-majority ⇒ our commit is due; our commit completes the majority ⇒
475
+ // consensus; our reject/conflict vote makes the record terminal), and any follow-on phase
476
+ // not re-handled here would silently wait for the coordinator's next delivery — e.g. a
477
+ // member whose promise the coordinator never collected receives the commit-phase record,
478
+ // adds its promise, and must then also commit in the SAME delivery. One loop replaces the
479
+ // hand-written per-branch re-checks that used to cover only the follow-ons their authors
480
+ // thought of. Only the three vote-adding phases continue; each strictly grows the record
481
+ // (adds a promise or commit key), so the loop terminates the cap guards a
482
+ // phase-computation bug, not a real bound.
483
+ const MaxPhaseSteps = 8;
484
+ let shouldPersist = true;
485
+ phaseLoop: for (let step = 0; ; step++) {
486
+ if (step >= MaxPhaseSteps) {
487
+ log('cluster-member:phase-loop-overflow', { messageHash: record.messageHash, steps: step });
488
+ break;
489
+ }
490
+ const phaseResult = await this.getTransactionPhase(currentRecord);
491
+ log('cluster-member:phase', {
492
+ messageHash: record.messageHash,
493
+ phase: phaseResult.phase,
494
+ step,
495
+ promises: Object.keys(currentRecord.promises ?? {}),
496
+ commits: Object.keys(currentRecord.commits ?? {})
497
+ });
498
+ switch (phaseResult.phase) {
499
+ case TransactionPhase.OurPromiseNeeded:
500
+ log('cluster-member:action-promise', {
501
+ messageHash: record.messageHash
502
+ });
503
+ currentRecord = await this.handlePromiseNeeded(currentRecord);
504
+ log('cluster-member:action-promise-complete', {
505
+ messageHash: record.messageHash,
506
+ promises: Object.keys(currentRecord.promises ?? {})
507
+ });
508
+ // Our own vote can be terminal (a reject where maxAllowedRejections is 0) or complete
509
+ // the super-majority — recompute rather than guess which.
510
+ continue;
511
+ case TransactionPhase.OurConflictVoteNeeded:
512
+ currentRecord = await this.handleConflictVoteNeeded(currentRecord, phaseResult.conflictsWith);
513
+ // Never persist a record we conflict-voted: this member holds the WINNER, and
514
+ // persisting the loser would reserve the same blocks a second time — half of what
515
+ // made the silent-abstention failure self-sustaining.
516
+ shouldPersist = false;
517
+ continue;
518
+ case TransactionPhase.OurCommitNeeded:
519
+ log('cluster-member:action-commit', {
520
+ messageHash: record.messageHash
521
+ });
522
+ currentRecord = await this.handleCommitNeeded(currentRecord);
523
+ log('cluster-member:action-commit-complete', {
524
+ messageHash: record.messageHash,
525
+ commits: Object.keys(currentRecord.commits ?? {})
526
+ });
527
+ shouldPersist = false;
528
+ // Our commit may have completed the majority — recompute; Consensus executes below.
529
+ continue;
530
+ case TransactionPhase.Consensus:
531
+ log('cluster-member:action-consensus', {
532
+ messageHash: record.messageHash
533
+ });
534
+ await this.handleConsensus(currentRecord);
535
+ shouldPersist = false;
536
+ break phaseLoop;
537
+ case TransactionPhase.Rejected:
538
+ log('cluster-member:action-rejected', {
539
+ messageHash: record.messageHash
540
+ });
541
+ await this.handleRejection(currentRecord);
542
+ shouldPersist = false;
543
+ break phaseLoop;
544
+ case TransactionPhase.ConflictSuperseded:
545
+ // Enough conflict votes that super-majority is unreachable. NOT a rejection — the
546
+ // callers retry it as a fresh transaction — so it gets its own terminal phase and the
547
+ // record is cleared rather than held (holding a provably-dead loser would reserve its
548
+ // blocks against the very retry that is supposed to win).
549
+ log('cluster-member:action-conflict-superseded', {
550
+ messageHash: record.messageHash
551
+ });
552
+ shouldPersist = false;
553
+ break phaseLoop;
554
+ case TransactionPhase.Propagating:
555
+ // Transaction is complete and propagating - clean it up
556
+ log('cluster-member:phase-propagating', {
557
+ messageHash: record.messageHash
558
+ });
559
+ shouldPersist = false;
560
+ break phaseLoop;
561
+ case TransactionPhase.Promising:
562
+ // We have already voted (approve, reject, or conflict); the record is still
563
+ // collecting promises from the rest of the cohort. Nothing to add retain the
564
+ // record only if our vote wasn't a conflict (`shouldPersist` already reflects that).
565
+ log('cluster-member:phase-promising-waiting', {
566
+ messageHash: record.messageHash
567
+ });
568
+ break phaseLoop;
569
+ }
570
+ }
571
+
572
+ if (shouldPersist) {
573
+ // Update transaction state
574
+ const timeouts = this.setupTimeouts(currentRecord);
575
+ this.activeTransactions.set(record.messageHash, {
576
+ record: currentRecord,
577
+ lastUpdate: Date.now(),
578
+ promiseTimeout: timeouts.promiseTimeout,
579
+ resolutionTimeout: timeouts.resolutionTimeout
580
+ });
581
+ this.persistParticipantState(record.messageHash, currentRecord);
582
+ log('cluster-member:state-persist', {
583
+ messageHash: record.messageHash,
584
+ storedPromises: Object.keys(currentRecord.promises ?? {}),
585
+ storedCommits: Object.keys(currentRecord.commits ?? {})
586
+ });
587
+ } else {
588
+ log('cluster-member:state-clear', {
589
+ messageHash: record.messageHash
590
+ });
591
+ this.clearTransaction(record.messageHash);
592
+ }
593
+
594
+ // Skip propagation - the coordinator manages distribution
595
+ // await this.propagateIfNeeded(currentRecord);
596
+
597
+ log('cluster-member:update-complete', {
598
+ messageHash: record.messageHash,
599
+ promiseCount: Object.keys(currentRecord.promises).length,
600
+ commitCount: Object.keys(currentRecord.commits).length
601
+ });
602
+ return currentRecord;
603
+ }
604
+
605
+ /**
606
+ * Merges two records, validating that non-signature fields match.
607
+ * Detects equivocation (same peer changing vote type) and applies penalties.
608
+ */
609
+ private async mergeRecords(existing: ClusterRecord, incoming: ClusterRecord): Promise<ClusterRecord> {
610
+ log('cluster-member:merge-records', {
611
+ messageHash: existing.messageHash,
612
+ existingPromises: Object.keys(existing.promises ?? {}),
613
+ existingCommits: Object.keys(existing.commits ?? {}),
614
+ incomingPromises: Object.keys(incoming.promises ?? {}),
615
+ incomingCommits: Object.keys(incoming.commits ?? {})
616
+ });
617
+ // Verify that immutable fields match
618
+ if (existing.messageHash !== incoming.messageHash) {
619
+ throw new Error('Message hash mismatch');
620
+ }
621
+ if (ClusterMember.canonicalJson(existing.message) !== ClusterMember.canonicalJson(incoming.message)) {
622
+ throw new Error('Message content mismatch');
623
+ }
624
+ if (existing.membershipVersion === 2 || incoming.membershipVersion === 2) {
625
+ // v2: the sorted peer-id set (captured by membershipDigest) is bound into messageHash, so equal
626
+ // messageHash MUST imply equal membership on any honest path. A mismatch here — different digest
627
+ // or version at equal hash — is a protocol violation (a bug or a hash-collision attack), NOT an
628
+ // honest divergence: two honest members with different views now hold two DIFFERENT hashes, i.e.
629
+ // two competing transactions the race machinery resolves, not one contested record. Log loudly
630
+ // and reject; never silently adopt the incoming set. (validateRecord already proved each record's
631
+ // own digest matches its own peers, so multiaddr / pubkey churn within the SAME id set — which
632
+ // keeps the same digest and hash — does NOT trip this.)
633
+ if (existing.membershipVersion !== incoming.membershipVersion || existing.membershipDigest !== incoming.membershipDigest) {
634
+ log('cluster-member:peers-mismatch-invariant-violation', {
635
+ messageHash: existing.messageHash,
636
+ existingVersion: existing.membershipVersion,
637
+ incomingVersion: incoming.membershipVersion,
638
+ existingDigest: existing.membershipDigest,
639
+ incomingDigest: incoming.membershipDigest,
640
+ existingPeers: Object.keys(existing.peers ?? {}).sort(),
641
+ incomingPeers: Object.keys(incoming.peers ?? {}).sort()
642
+ });
643
+ throw new Error('Peers mismatch');
644
+ }
645
+ } else if (ClusterMember.canonicalJson(existing.peers) !== ClusterMember.canonicalJson(incoming.peers)) {
646
+ // v1 (legacy, membership unbound): full peer-object equality is the only available guard.
647
+ throw new Error('Peers mismatch');
648
+ }
649
+
650
+ // Merge signatures with equivocation detection
651
+ const mergedPromises = this.detectEquivocation(
652
+ existing.promises, incoming.promises, 'promise', existing.messageHash
653
+ );
654
+ const mergedCommits = this.detectEquivocation(
655
+ existing.commits, incoming.commits, 'commit', existing.messageHash
656
+ );
657
+
658
+ return {
659
+ ...existing,
660
+ promises: mergedPromises,
661
+ commits: mergedCommits
662
+ };
663
+ }
664
+
665
+ /**
666
+ * Compares existing vs incoming signatures for the same peers.
667
+ * If a peer's vote type changed (approve↔reject), that's equivocation:
668
+ * report a penalty and keep the first-seen signature.
669
+ * New peers are accepted normally.
670
+ */
671
+ private detectEquivocation(
672
+ existing: Record<string, Signature>,
673
+ incoming: Record<string, Signature>,
674
+ phase: 'promise' | 'commit',
675
+ messageHash: string
676
+ ): Record<string, Signature> {
677
+ // NOTE: relies on validateSignatures() (via validateRecord in processUpdate) having already run on
678
+ // every record reaching here, so each peerId's signature is key-bound. Without that guarantee the
679
+ // Equivocation penalty below would act on self-asserted, unverified peer ids and could frame an
680
+ // honest peer. Do not call this on unvalidated signatures.
681
+ const merged = { ...existing };
682
+
683
+ for (const [peerId, incomingSig] of Object.entries(incoming)) {
684
+ const existingSig = existing[peerId];
685
+ if (existingSig) {
686
+ if (existingSig.type !== incomingSig.type) {
687
+ // Equivocation detected: peer changed their vote type
688
+ log('cluster-member:equivocation-detected', {
689
+ peerId,
690
+ phase,
691
+ messageHash,
692
+ existingType: existingSig.type,
693
+ incomingType: incomingSig.type
694
+ });
695
+ this.reputation?.reportPeer(
696
+ peerId,
697
+ PenaltyReason.Equivocation,
698
+ `${phase}:${messageHash}:${existingSig.type}->${incomingSig.type}`
699
+ );
700
+ // Keep first-seen signature — do not let the peer flip their vote
701
+ }
702
+ // Same type: keep existing (no-op, already in merged)
703
+ } else {
704
+ // New peer — accept normally
705
+ merged[peerId] = incomingSig;
706
+ }
707
+ }
708
+
709
+ return merged;
710
+ }
711
+
712
+ private async validateRecord(record: ClusterRecord): Promise<void> {
713
+ // Reject a record whose membership-binding version this code does not implement. The cluster
714
+ // consensus code is a single deployable unit (all cluster members upgrade together), so a version
715
+ // we don't understand is rejected rather than cross-version-consensus'd.
716
+ if (record.membershipVersion !== undefined && record.membershipVersion !== 1 && record.membershipVersion !== 2) {
717
+ throw new Error(`Unsupported membershipVersion: ${record.membershipVersion}`);
718
+ }
719
+
720
+ // v2: the declared membership digest must match the record's own peer set. A record whose declared
721
+ // digest doesn't match its peers is malformed (and its messageHash computed over that digest
722
+ // would not bind the real membership).
723
+ // NOTE: recomputes membershipDigest (one SHA256 over the sorted peer-id list) on every incoming v2
724
+ // record; if a hot cluster ever shows this as a cost, memoize per (messageHash digest).
725
+ if (record.membershipVersion === 2) {
726
+ const expectedDigest = await membershipDigest(record.peers);
727
+ if (expectedDigest !== record.membershipDigest) {
728
+ throw new Error(`Membership digest mismatch: expected=${expectedDigest}, received=${record.membershipDigest ?? 'undefined'}`);
729
+ }
730
+ }
731
+
732
+ // Validate message hash matches the message content (v2 folds in the membership digest)
733
+ const expectedHash = await this.computeMessageHash(record);
734
+ if (expectedHash !== record.messageHash) {
735
+ throw new Error(`Message hash mismatch: expected=${expectedHash}, received=${record.messageHash}`);
736
+ }
737
+
738
+ // Validate signatures
739
+ await this.validateSignatures(record);
740
+
741
+ // Validate expiration
742
+ if (record.message.expiration && record.message.expiration < Date.now()) {
743
+ throw new Error('Transaction expired');
744
+ }
745
+ }
746
+
747
+ /**
748
+ * Compute message hash using the same algorithm as the coordinator. Version-dispatched: a v2 record
749
+ * folds its membership digest into the preimage, a v1 / unversioned record hashes byte-identically to
750
+ * before this change. Must match cluster-coordinator.ts createMessageHash().
751
+ */
752
+ private async computeMessageHash(record: Pick<ClusterRecord, 'message' | 'membershipVersion' | 'membershipDigest'>): Promise<string> {
753
+ return computeClusterMessageHash(record.message, recordMembershipDigest(record));
754
+ }
755
+
756
+ private async validateSignatures(record: ClusterRecord): Promise<void> {
757
+ // Validate promise signatures. Reject on any failure, but only report an InvalidSignature
758
+ // penalty when the key was proven to belong to `peerId` (outcome.penalize) — otherwise the id
759
+ // is attacker-chosen and reporting it would let a coordinator frame an honest peer.
760
+ const promiseHash = await this.computePromiseHash(record);
761
+ for (const [peerId, signature] of Object.entries(record.promises)) {
762
+ const outcome = await this.verifySignature(record, peerId, promiseHash, signature);
763
+ if (!outcome.valid) {
764
+ if (outcome.penalize) {
765
+ this.reputation?.reportPeer(peerId, PenaltyReason.InvalidSignature, `promise:${record.messageHash}`);
766
+ }
767
+ throw new Error(`Invalid promise signature from ${peerId}`);
768
+ }
769
+ }
770
+
771
+ // Validate commit signatures
772
+ const commitHash = await this.computeCommitHash(record);
773
+ for (const [peerId, signature] of Object.entries(record.commits)) {
774
+ const outcome = await this.verifySignature(record, peerId, commitHash, signature);
775
+ if (!outcome.valid) {
776
+ if (outcome.penalize) {
777
+ this.reputation?.reportPeer(peerId, PenaltyReason.InvalidSignature, `commit:${record.messageHash}`);
778
+ }
779
+ throw new Error(`Invalid commit signature from ${peerId}`);
780
+ }
781
+ }
782
+ }
783
+
784
+ /**
785
+ * Deterministic JSON: sorts object keys so comparisons are order-independent.
786
+ * NOTE: this is a second copy of the canonicalization in db-core `membership.ts` (which feeds the hash
787
+ * preimages). It is used here only for equality checks (message-content / v1-peers compare), and those
788
+ * run *after* a messageHash-equality gate, so a drift between the two can't silently forge agreement —
789
+ * but keep them byte-identical. If a third caller appears, promote to a single exported helper.
790
+ */
791
+ private static canonicalJson(value: unknown): string {
792
+ return JSON.stringify(value, (_, v) =>
793
+ v && typeof v === 'object' && !Array.isArray(v)
794
+ ? Object.keys(v).sort().reduce((o: Record<string, unknown>, k) => { o[k] = v[k]; return o; }, {})
795
+ : v
796
+ );
797
+ }
798
+
799
+ private async computePromiseHash(record: ClusterRecord): Promise<string> {
800
+ return computeClusterPromiseHash(record.messageHash, record.message, recordMembershipDigest(record));
801
+ }
802
+
803
+ private async computeCommitHash(record: ClusterRecord): Promise<string> {
804
+ return computeClusterCommitHash(record.messageHash, record.message, record.promises, recordMembershipDigest(record));
805
+ }
806
+
807
+ private async signVote(hash: string, type: Signature['type'], extra?: string): Promise<string> {
808
+ const sigBytes = await this.privateKey.sign(clusterVoteSigningPayload(hash, type, extra));
809
+ return uint8ArrayToString(sigBytes, 'base64url');
810
+ }
811
+
812
+ /**
813
+ * Verify one vote signature and classify the outcome (see {@link VerifyOutcome}). Total on hostile
814
+ * input: a missing/empty key, a non-Ed25519 id, a key not bound to `peerId`, or malformed bytes all
815
+ * yield `{ valid:false, penalize:false }` (reject without penalizing an unproven identity) rather
816
+ * than throwing. Only after the key is proven to be the one `peerId` names does a failed
817
+ * cryptographic verify yield `{ valid:false, penalize:true }`.
818
+ *
819
+ * NOTE: the binding check (`peerIdBindsPublicKey`) proves the vote was signed by the key `peerId`
820
+ * names — it does NOT establish that `peerId` is legitimately in the cohort. A coordinator minting
821
+ * fresh keypairs and using each key's own derived id passes this for every one. Sybil/cohort
822
+ * membership is a separate layer (cohort-topic membership certificates), not solved here.
823
+ */
824
+ private async verifySignature(record: ClusterRecord, peerId: string, hash: string, signature: Signature): Promise<VerifyOutcome> {
825
+ const peerInfo = record.peers[peerId];
826
+ if (!peerInfo?.publicKey?.length) {
827
+ // No key to check against — identity not proven. Reject without penalty.
828
+ return { valid: false, penalize: false };
829
+ }
830
+ let keyBytes: Uint8Array;
831
+ try {
832
+ // publicKey is base64url-encoded string (JSON-serialization safe)
833
+ keyBytes = uint8ArrayFromString(peerInfo.publicKey, 'base64url');
834
+ } catch {
835
+ return { valid: false, penalize: false };
836
+ }
837
+ // The key must be the one `peerId` provably names, else the vote could be attributed to any peer
838
+ // id while signed by a key the coordinator controls. Binding failure identity unproven ⇒ no penalty.
839
+ if (!peerIdBindsPublicKey(peerId, keyBytes)) {
840
+ return { valid: false, penalize: false };
841
+ }
842
+ try {
843
+ const pubKey = publicKeyFromRaw(keyBytes);
844
+ const payload = clusterVoteVerificationPayload(hash, signature);
845
+ const sigBytes = uint8ArrayFromString(signature.signature, 'base64url');
846
+ const ok = await pubKey.verify(payload, sigBytes);
847
+ // Key is bound to peerId: a failed verify is a genuine bad vote from a proven identity → penalize.
848
+ // NOTE: residual — an Ed25519 peer's public key is derivable from its (public) id, so an attacker
849
+ // can attach a victim's REAL key with a garbage signature and still trip this InvalidSignature
850
+ // penalty on the victim. Binding narrows framing (the attacker must use the victim's own key, not
851
+ // an arbitrary one) but cannot eliminate it here: a single signature can't distinguish "victim
852
+ // signed badly" from "someone pasted the victim's public key + junk". Fully closing it needs an
853
+ // authenticated membership/channel layer (cohort-topic membership certs), out of scope for this fix.
854
+ return ok ? { valid: true } : { valid: false, penalize: true };
855
+ } catch {
856
+ // Malformed signature bytes / key decode failure: reject, but do not penalize on unparseable input.
857
+ return { valid: false, penalize: false };
858
+ }
859
+ }
860
+
861
+ private async getTransactionPhase(record: ClusterRecord): Promise<PhaseResult> {
862
+ const peerCount = Object.keys(record.peers).length;
863
+ const promiseCount = Object.keys(record.promises).length;
864
+ const ourId = this.peerId.toString();
865
+
866
+ const superMajority = Math.ceil(peerCount * this.superMajorityThreshold);
867
+ const maxAllowedRejections = peerCount - superMajority;
868
+
869
+ // Check for rejections rejected if too many rejections to ever reach super-majority.
870
+ // ONLY `reject` votes count here: a `conflict` vote is "not now", never a validity
871
+ // judgement, so it must not push a record into the permanent `Rejected` phase.
872
+ const rejectedPromises = Object.values(record.promises).filter(s => s.type === 'reject');
873
+ const conflictPromises = Object.values(record.promises).filter(s => s.type === 'conflict');
874
+ const rejectedCommits = Object.values(record.commits).filter(s => s.type === 'reject');
875
+ if (rejectedPromises.length > maxAllowedRejections || this.hasMajority(rejectedCommits.length, peerCount)) {
876
+ return { phase: TransactionPhase.Rejected };
877
+ }
878
+
879
+ // Conflict votes don't judge validity, but enough of them still make super-majority
880
+ // unreachable a distinct terminal outcome (retryable as a fresh transaction) so logs and
881
+ // reputation-adjacent paths keep meaning what they say.
882
+ if (conflictPromises.length > 0 && rejectedPromises.length + conflictPromises.length > maxAllowedRejections) {
883
+ return { phase: TransactionPhase.ConflictSuperseded };
884
+ }
885
+
886
+ // Check if we need to vote. A lost race is answered with a conflict vote, not silence:
887
+ // absence used to mean both "unreachable" and "refusing in favour of a rival", and the
888
+ // coordinator could not tell the two apart. Once our conflict vote is merged into
889
+ // `promises`, this branch is skipped forever — a conflict vote is terminal for this record;
890
+ // a retry must be a fresh transaction (new messageHash), which `CoordinatorRepo.pend`
891
+ // already mints per call.
892
+ if (!record.promises[ourId]) {
893
+ const conflict = this.findConflict(record);
894
+ if (conflict) {
895
+ return { phase: TransactionPhase.OurConflictVoteNeeded, conflictsWith: conflict.blockedBy };
896
+ }
897
+ return { phase: TransactionPhase.OurPromiseNeeded };
898
+ }
899
+
900
+ // Check if we have enough approved promises to proceed to commit. Deliberately blind to what
901
+ // OUR own vote was: the rest of the cohort reaching super-majority is the commit rule
902
+ // (Theorem 1 Case 2), so a member that rejected — or conflict-voted — still signs the commit
903
+ // the cohort decided on rather than stalling it. Only the rejection/superseded thresholds
904
+ // above can stop a record here, and both are checked first.
905
+ //
906
+ // NOTE: signing the commit drops this member's reservation on the record
907
+ // (`shouldPersist = false` in the caller), and the phase fixpoint means that can now happen on
908
+ // the FIRST delivery when the record already arrives at super-majority, rather than a
909
+ // round-trip later. The safety argument is quorum intersection (Theorem 9: no rival can
910
+ // assemble its own super-majority once this one has), NOT the reservation — the reservation
911
+ // only orders *concurrently-pending* rivals.
912
+ //
913
+ // The lost update this predicted WAS observed (a pend admitted between a rival's
914
+ // pend-consensus and commit-consensus, then refused by every member's storage at apply and
915
+ // still reported to the writer as a success). Holding the reservation until `handleConsensus`
916
+ // would not have closed it — the loser was approved before the winner's apply had even
917
+ // reached most members — so the cure went elsewhere: `validatePendOperations` now rejects a
918
+ // pend whose blocks are held by a different unresolved STORAGE pending record (the durable
919
+ // reservation that spans pend-apply → commit/cancel), and the coordinator returns storage's
920
+ // retained apply verdict instead of fabricating success (`getExecutedPendResult`). Residual:
921
+ // a member that has not yet applied the rival's pend abstains from that vote, and only the
922
+ // coordinating node's own verdict is threaded back — see the handoff notes on
923
+ // `CoordinatorRepo.pend`.
924
+ const approvedPromises = Object.values(record.promises).filter(s => s.type === 'approve');
925
+ if (approvedPromises.length >= superMajority && !record.commits[ourId]) {
926
+ return { phase: TransactionPhase.OurCommitNeeded };
927
+ }
928
+
929
+ // Check if still collecting promises
930
+ if (promiseCount < peerCount && approvedPromises.length < superMajority) {
931
+ return { phase: TransactionPhase.Promising };
932
+ }
933
+
934
+ // Check for consensus
935
+ const approvedCommits = Object.values(record.commits).filter(s => s.type === 'approve');
936
+ if (this.hasMajority(approvedCommits.length, peerCount)) {
937
+ return { phase: TransactionPhase.Consensus };
938
+ }
939
+
940
+ return { phase: TransactionPhase.Propagating };
941
+ }
942
+
943
+ private hasMajority(count: number, total: number): boolean {
944
+ return count > total / 2;
945
+ }
946
+
947
+ private async handlePromiseNeeded(record: ClusterRecord): Promise<ClusterRecord> {
948
+ // Membership admission gate runs BEFORE pend validation: a member independently checks the declared
949
+ // peer set is a legitimate cluster it belongs to, and refuses (reject vote) rather than rubber-stamping
950
+ // a set the coordinator chose (e.g. a self-shrunk minority-partition set). On admission failure we skip
951
+ // pend validation entirely and emit the membership rejection.
952
+ const validationResult = await this.evaluatePromise(record);
953
+
954
+ const promiseHash = await this.computePromiseHash(record);
955
+ const type = validationResult.valid ? 'approve' as const : 'reject' as const;
956
+ const rejectReason = validationResult.valid ? undefined : validationResult.reason;
957
+ const sig = await this.signVote(promiseHash, type, rejectReason);
958
+
959
+ const signature: Signature = validationResult.valid
960
+ ? { type: 'approve', signature: sig }
961
+ : { type: 'reject', signature: sig, rejectReason };
962
+
963
+ if (!validationResult.valid) {
964
+ log('cluster-member:validation-rejected', {
965
+ messageHash: record.messageHash,
966
+ reason: validationResult.reason
967
+ });
968
+ }
969
+
970
+ return {
971
+ ...record,
972
+ promises: {
973
+ ...record.promises,
974
+ [this.peerId.toString()]: signature
975
+ }
976
+ };
977
+ }
978
+
979
+ /**
980
+ * Answer a record that lost the deterministic race to a transaction this member already holds
981
+ * (`docs/correctness.md` Theorems 1 & 9: the loser is TOLD it lost, not ignored — an unanswered
982
+ * loss is indistinguishable from an unreachable cohort at the coordinator). `conflictWith` — the
983
+ * winning rival's messageHash — is folded into the signed payload, so the claim is
984
+ * integrity-protected in transit and readable without parsing prose. NOT a validity judgement:
985
+ * {@link getTransactionPhase} never counts conflict votes toward the permanent-rejection
986
+ * threshold, and the coordinator surfaces them as a retryable loss, never a validator rejection.
987
+ */
988
+ private async handleConflictVoteNeeded(record: ClusterRecord, conflictWith: string): Promise<ClusterRecord> {
989
+ log('cluster-member:action-conflict-vote', {
990
+ messageHash: record.messageHash,
991
+ conflictWith
992
+ });
993
+ const promiseHash = await this.computePromiseHash(record);
994
+ const sig = await this.signVote(promiseHash, 'conflict', conflictWith);
995
+ const signature: Signature = { type: 'conflict', signature: sig, conflictWith };
996
+
997
+ return {
998
+ ...record,
999
+ promises: {
1000
+ ...record.promises,
1001
+ [this.peerId.toString()]: signature
1002
+ }
1003
+ };
1004
+ }
1005
+
1006
+ /**
1007
+ * The full promise-phase decision for a record: admit the declared membership FIRST, then (only if
1008
+ * admitted) validate its pend operations, then its commit operations. Failing any yields a
1009
+ * `{ valid:false, reason }` the caller turns into a `reject` vote. Keeping the three separate keeps
1010
+ * the reason strings distinct — a `membership-not-admitted` reject is a different signal (feeds the
1011
+ * dispute path) than a stale-revision / custom-validator reject, which is different again from a
1012
+ * `content-digest-mismatch` (see {@link validateCommitOperations}). A record carries pend OR commit
1013
+ * operations, so in practice exactly one of the latter two has anything to inspect.
1014
+ */
1015
+ private async evaluatePromise(record: ClusterRecord): Promise<{ valid: boolean; reason?: string }> {
1016
+ const admission = await this.admitMembership(record);
1017
+ if (!admission.admit) {
1018
+ return { valid: false, reason: admission.reason ?? MEMBERSHIP_NOT_ADMITTED };
1019
+ }
1020
+ const pendValidation = await this.validatePendOperations(record);
1021
+ if (!pendValidation.valid) {
1022
+ return pendValidation;
1023
+ }
1024
+ // Revision staleness before content digests: a commit whose revision a rival already took can
1025
+ // never win, and the sharper stale reject also skips the per-block digest previews (the digest
1026
+ // check would abstain on such a block anyway its local base rev no longer matches the
1027
+ // declared one).
1028
+ const commitRevValidation = await this.validateCommitRevisions(record);
1029
+ if (!commitRevValidation.valid) {
1030
+ return commitRevValidation;
1031
+ }
1032
+ return await this.validateCommitOperations(record);
1033
+ }
1034
+
1035
+ /**
1036
+ * Membership admission gate. Decides whether the coordinator-declared peer set (`record.peers`, call it
1037
+ * `D`) is a *legitimate* cluster this member may vote inside, judged against the member's OWN
1038
+ * independently-derived view not against anything the (untrusted) record declares about its size.
1039
+ * Evaluated on the promise path before the member signs an approve.
1040
+ *
1041
+ * The predicate admits `D` iff ALL hold:
1042
+ * 1. **Self-membership** — this member's id ∈ `D`; else this block is not its responsibility (and a
1043
+ * coordinator must not route a record to a non-member to pad approval counts).
1044
+ * 2. **Not a self-shrink below the floor** — with a confident derived view `E`, `|D| ≥ ⌈fraction·|E|⌉`
1045
+ * (and ≥ minAbsoluteClusterSize). `|E|` is the member's own confident cluster-size estimate `K_est`,
1046
+ * so a minority-partition set (small `D`) is rejected against the member's larger view.
1047
+ * 3. **Consistency with the derived view** — `|D △ E|` within `clusterSizeTolerance·|E|`; honest churn
1048
+ * of a peer or two is absorbed, a wholesale-disjoint or half-size set is not.
1049
+ *
1050
+ * **Fail-closed posture.** When the member cannot confidently derive `E` (no capability, low FRET
1051
+ * confidence — exactly what a partition induces), it must refuse any *downsizing* decision — but it
1052
+ * needs a size reference to judge "downsize" against, and it may NOT borrow `clusterSize` for that:
1053
+ * `clusterSize` is the replication factor (what a cohort should aim for), not a claim about how many
1054
+ * peers exist, so a small deployment configured with the default 10 would refuse every write. The
1055
+ * fallback yardstick is instead {@link ClusterConsensusConfig.assumedClusterSize} — the operator's own
1056
+ * assertion of the smallest cohort this deployment can genuinely field — run through the SAME
1057
+ * {@link admissionFloor} as the confident path, so the fallback can never be stricter than the measured
1058
+ * path (it was: it demanded the full configured size, with no fraction and no slack for churn or a peer
1059
+ * not yet discovered). With NEITHER a confident view NOR an asserted size the gate cannot judge a
1060
+ * downsize at all, so it preserves the legacy approve behavior (backward-compatible for nodes/tests with
1061
+ * no derivation wired). `allowUnvalidatedSmallCluster` is the explicit opt-in (single-node / local dev
1062
+ * knowingly below the safe floor), matching the coordinator's `validateSmallCluster` semantics.
1063
+ */
1064
+ private async admitMembership(record: ClusterRecord): Promise<{ admit: boolean; reason?: string }> {
1065
+ const ourId = this.peerId.toString();
1066
+ const declared = Object.keys(record.peers ?? {});
1067
+
1068
+ // Predicate 1: self-membership. Always enforced (independent of any opt-in): a member does not vote
1069
+ // in a cluster it is not part of.
1070
+ if (!declared.includes(ourId)) {
1071
+ log('cluster-member:admission-reject', { messageHash: record.messageHash, reason: 'self-not-member', declaredSize: declared.length });
1072
+ return { admit: false, reason: `${MEMBERSHIP_NOT_ADMITTED}:self-not-member` };
1073
+ }
1074
+
1075
+ // Explicit opt-in: knowingly transact below the safe floor (single-node / local dev). Skips the
1076
+ // size/consistency gates but not self-membership above.
1077
+ if (this.allowUnvalidatedSmallCluster) {
1078
+ return { admit: true };
1079
+ }
1080
+
1081
+ const derived = await this.deriveExpectedClusterView(record);
1082
+ // An empty derived view (kEst === 0) carries no usable reference set: measured against it every
1083
+ // non-empty declared set is wholly "inconsistent" (maxDiff = ceil(tol·0) = 0), which would spuriously
1084
+ // reject a legitimate full cluster a stricter, worse outcome than an absent view. Treat empty as
1085
+ // not-confident so it takes the fail-closed-or-legacy branch below instead. (Not normally reachable:
1086
+ // a responsible member's findCluster includes at least itself; this guards a transient empty read.)
1087
+ const derivedSize = derived !== undefined ? Object.keys(derived.peers ?? {}).length : 0;
1088
+ const confident = derived !== undefined
1089
+ && derived.confidence > ClusterMember.MembershipConfidenceThreshold
1090
+ && derivedSize > 0;
1091
+
1092
+ if (!confident) {
1093
+ // Fail closed for downsizing under low/absent confidence, measured against the operator's asserted
1094
+ // cohort size rather than the replication factor. With no asserted size the gate cannot tell a
1095
+ // downsize from a legitimately small cluster at all, so it preserves legacy approve behavior.
1096
+ if (this.assumedClusterSize === undefined) {
1097
+ return { admit: true };
1098
+ }
1099
+ const floor = this.admissionFloor(this.assumedClusterSize);
1100
+ if (declared.length >= floor) {
1101
+ return { admit: true };
1102
+ }
1103
+ log('cluster-member:admission-reject', {
1104
+ messageHash: record.messageHash,
1105
+ reason: 'low-confidence-downsize',
1106
+ declaredSize: declared.length,
1107
+ floor,
1108
+ assumedClusterSize: this.assumedClusterSize,
1109
+ confidence: derived?.confidence
1110
+ });
1111
+ // The numbers ride along in the reason: this rejection is caused by *local* configuration, and
1112
+ // without them a coordinator (or an operator reading a dispute record) has no hint which knob did it.
1113
+ // NOTE: two honest members with different local config now emit *different* reason strings for the
1114
+ // same record. Nothing compares reasons across peers today (`disputeEvidence.rejectReasons` is a
1115
+ // per-peer map, and the signed payload hashes the string each vote carries); if anything ever groups
1116
+ // or dedupes dispute reasons by string equality, group on the `membership-not-admitted:<variant>`
1117
+ // prefix, not the whole string.
1118
+ return {
1119
+ admit: false,
1120
+ reason: `${MEMBERSHIP_NOT_ADMITTED}:low-confidence-downsize (declared=${declared.length}, floor=${floor}, assumedClusterSize=${this.assumedClusterSize})`
1121
+ };
1122
+ }
1123
+
1124
+ const expected = Object.keys(derived!.peers ?? {});
1125
+ const kEst = expected.length;
1126
+
1127
+ // Predicate 2: floor derived from the member's OWN confident estimate.
1128
+ const floor = this.admissionFloor(kEst);
1129
+ if (declared.length < floor) {
1130
+ log('cluster-member:admission-reject', {
1131
+ messageHash: record.messageHash,
1132
+ reason: 'below-floor',
1133
+ declaredSize: declared.length,
1134
+ floor,
1135
+ kEst
1136
+ });
1137
+ return {
1138
+ admit: false,
1139
+ reason: `${MEMBERSHIP_NOT_ADMITTED}:below-floor (declared=${declared.length}, floor=${floor}, kEst=${kEst})`
1140
+ };
1141
+ }
1142
+
1143
+ // Predicate 3: consistency with the derived view within tolerance.
1144
+ const symmetricDiff = ClusterMember.symmetricDifferenceSize(declared, expected);
1145
+ const maxDiff = Math.ceil(this.clusterSizeTolerance * kEst);
1146
+ if (symmetricDiff > maxDiff) {
1147
+ log('cluster-member:admission-reject', {
1148
+ messageHash: record.messageHash,
1149
+ reason: 'inconsistent-with-derived-view',
1150
+ declaredSize: declared.length,
1151
+ kEst,
1152
+ symmetricDiff,
1153
+ maxDiff
1154
+ });
1155
+ return { admit: false, reason: `${MEMBERSHIP_NOT_ADMITTED}:inconsistent-with-derived-view` };
1156
+ }
1157
+
1158
+ return { admit: true };
1159
+ }
1160
+
1161
+ /**
1162
+ * The smallest declared peer set admissible against a cohort-size reference `k`, whether `k` is
1163
+ * measured (the confident path's `kEst`) or asserted (`assumedClusterSize`). One function so the
1164
+ * fallback can never be stricter than the measured path — which it was, demanding the full configured
1165
+ * size with no fraction and no slack. Clamped at `minAbsoluteClusterSize`, so a degenerate `k` of 0, 1
1166
+ * or negative yields the absolute floor rather than a floor that admits everything. A non-finite scaled
1167
+ * size (a `NaN` or `Infinity` config value) is likewise treated as no usable reference rather than
1168
+ * propagating: an unguarded `NaN` floor fails EVERY comparison, which would silently make the node
1169
+ * reject every unconfident write.
1170
+ *
1171
+ * NOTE: partition safety needs `2 · membershipAdmissionFraction · superMajorityThreshold > 1` each
1172
+ * side of a split must recruit `fraction · threshold · K` distinct honest members, and two sides cannot
1173
+ * both find them in one K-peer cluster. At the shipped defaults (0.75 · 0.75 — both default to
1174
+ * `DEFAULT_SUPER_MAJORITY_THRESHOLD` / `membershipAdmissionFraction`'s own default) that product is
1175
+ * 1.125. If either default is ever lowered, re-check Theorem 2 in `docs/correctness.md` before shipping it.
1176
+ */
1177
+ private admissionFloor(k: number): number {
1178
+ const scaled = Math.ceil(this.membershipAdmissionFraction * k);
1179
+ return Math.max(this.minAbsoluteClusterSize, Number.isFinite(scaled) ? scaled : 0);
1180
+ }
1181
+
1182
+ /**
1183
+ * Derive this member's own view of the record's block cluster via the injected capability, or
1184
+ * `undefined` when it cannot (no capability, no coordinating block id, a coordinating block not bound
1185
+ * to the record's own operations, or a derivation error — all of which the gate treats as "not
1186
+ * confident"). Derived from the record's coordinating block, the same key the coordinator used to
1187
+ * select the cluster.
1188
+ *
1189
+ * Read off `record.message`, NOT a top-level record field: `messageHash` covers the message only, so
1190
+ * only the in-message copy is tamper-evident to a relaying peer. (There is no top-level copy any more —
1191
+ * see {@link ClusterRecord.message}.)
1192
+ */
1193
+ private async deriveExpectedClusterView(record: ClusterRecord): Promise<ExpectedClusterView | undefined> {
1194
+ if (!this.deriveExpectedCluster) {
1195
+ return undefined;
1196
+ }
1197
+ const blockId = record.message.coordinatingBlockIds?.[0];
1198
+ if (blockId === undefined) {
1199
+ return undefined;
1200
+ }
1201
+ // Hashing the field makes it tamper-evident to RELAYS, but the coordinator is the party this gate
1202
+ // exists to check and it picks the field before it computes the hash. Unbound, a Byzantine
1203
+ // coordinator declares a shrunken cohort `D` and names a coordinating block whose real cohort
1204
+ // resembles `D`: every member then derives that block's cohort, finds kEst = |D|, symmetric
1205
+ // difference 0, and admits — the gate fully defeated. Binding the coordinating block to a block the
1206
+ // record's OWN operations touch removes that free choice. `getAffectedBlockIds` is the same block
1207
+ // extraction conflict detection already runs on this message — one definition, so the set a
1208
+ // coordinating id must come from cannot drift from the set the record is judged to touch.
1209
+ if (!this.getAffectedBlockIds(record.message.operations).includes(blockId)) {
1210
+ log('cluster-member:coordinating-block-unbound', {
1211
+ messageHash: record.messageHash,
1212
+ coordinatingBlockId: blockId
1213
+ });
1214
+ // Fail closed into the branch that already exists rather than throwing: a hard reject would
1215
+ // change `validateRecord`'s failure surface, and "not confident" already refuses any downsize.
1216
+ return undefined;
1217
+ }
1218
+ // Which block a member derived its cohort view from is the single most useful fact when an
1219
+ // admission decision has to be explained after the fact and the only externally visible sign
1220
+ // that the confident predicates ran at all rather than the fallback floor.
1221
+ log('cluster-member:derive-expected-cluster', { messageHash: record.messageHash, blockId });
1222
+ try {
1223
+ // NOTE: derives (findCluster) once per inbound record on the promise path one routing lookup
1224
+ // per vote. If this shows up as hot, cache the derived view per (blockId, short TTL): it is a
1225
+ // pure read of current topology, so a few-seconds-stale view is safe for admission.
1226
+ return await this.deriveExpectedCluster(blockId as BlockId);
1227
+ } catch (err) {
1228
+ log('cluster-member:derive-expected-cluster-error', { messageHash: record.messageHash, error: (err as Error).message });
1229
+ return undefined;
1230
+ }
1231
+ }
1232
+
1233
+ /** |A B| over two id lists (order-independent set symmetric difference). */
1234
+ private static symmetricDifferenceSize(a: string[], b: string[]): number {
1235
+ const setA = new Set(a);
1236
+ const setB = new Set(b);
1237
+ let count = 0;
1238
+ for (const x of setA) if (!setB.has(x)) count++;
1239
+ for (const x of setB) if (!setA.has(x)) count++;
1240
+ return count;
1241
+ }
1242
+
1243
+ /**
1244
+ * Validates pend operations in a cluster record using the transaction validator.
1245
+ * Also checks for stale revisions, and for blocks held by a different unresolved pending
1246
+ * action, to prevent consensus on operations that storage would refuse at apply.
1247
+ * Returns success if no validator is configured (backwards compatibility).
1248
+ */
1249
+ private async validatePendOperations(record: ClusterRecord): Promise<{ valid: boolean; reason?: string }> {
1250
+ // Find pend operations in the message
1251
+ for (const operation of record.message.operations) {
1252
+ if ('pend' in operation) {
1253
+ const pendRequest = operation.pend;
1254
+ const blockIds = blockIdsForTransforms(pendRequest.transforms);
1255
+ // One state read serves both checks below: `latest` for staleness, `pendings` for the
1256
+ // unresolved-rival check.
1257
+ const blockResults = await this.storageRepo.get({ blockIds });
1258
+
1259
+ // Check for stale revisions before allowing consensus
1260
+ if (pendRequest.rev !== undefined) {
1261
+ for (const blockId of blockIds) {
1262
+ const blockResult = blockResults[blockId];
1263
+ if (blockResult?.unavailable !== undefined) {
1264
+ // This member cannot establish the block's revision, so it cannot judge
1265
+ // staleness. Vote reject rather than approve on an answer it knows is a
1266
+ // guess — approving would let a stale pend reach consensus on the strength
1267
+ // of a member that could not check it. (Before StorageRepo caught
1268
+ // materialization faults per block, this read threw out of the promise
1269
+ // handler; rejecting keeps the fail-closed posture with a signed reason.)
1270
+ log('cluster-member:validation-block-unavailable', {
1271
+ messageHash: record.messageHash,
1272
+ blockId,
1273
+ reason: blockResult.unavailable
1274
+ });
1275
+ return { valid: false, reason: `block ${blockId} unavailable (${blockResult.unavailable}): cannot verify revision` };
1276
+ }
1277
+ const latest = blockResult?.state?.latest;
1278
+ if (latest !== undefined && latest.rev >= pendRequest.rev) {
1279
+ // Self is excluded so a redelivered pend for this same action stays
1280
+ // approvable — the same exclusion the pending-rival check below documents,
1281
+ // and the same rule storage applies (see {@link isOwnRevision}).
1282
+ if (isOwnRevision(latest, pendRequest.rev, pendRequest.actionId)) {
1283
+ continue;
1284
+ }
1285
+ log('cluster-member:validation-stale-revision', {
1286
+ messageHash: record.messageHash,
1287
+ blockId,
1288
+ requestedRev: pendRequest.rev,
1289
+ latestRev: latest.rev
1290
+ });
1291
+ // Deliberately prose-only: this reason is fed to computeSigningPayload, signed,
1292
+ // and carried as Signature.rejectReason, so adding a structured revision here
1293
+ // would change the signed byte layout and the Signature type every peer would
1294
+ // have to agree on the new format or verification breaks across versions. This
1295
+ // is NOT a StaleFailure producer, so StaleFailure.staleAt does not apply; the
1296
+ // coordinator's own local re-read (CoordinatorRepo.classifyStaleRejection)
1297
+ // supplies that number when it can confirm the revision itself.
1298
+ return { valid: false, reason: `stale revision: block ${blockId} at rev ${latest.rev}, requested rev ${pendRequest.rev}` };
1299
+ }
1300
+ }
1301
+ }
1302
+
1303
+ // Reject a pend whose blocks are held by a DIFFERENT unresolved pending action. This is
1304
+ // the durable reservation the in-memory table (`findConflict` / `activeTransactions`)
1305
+ // cannot provide: that table clears the moment the rival's PEND record reaches
1306
+ // consensus, but the rival's storage pending record written at pend-apply, removed at
1307
+ // commit or cancel — spans exactly the pend→commit window in which `latest.rev` has not
1308
+ // yet advanced. Storage's own pend would refuse this request at consensus-apply for the
1309
+ // same reason (`StorageRepo.pend`'s listPendingTransactions scan); voting reject here
1310
+ // moves that verdict into the phase where the cohort aggregates it, so the loser is
1311
+ // refused with a real answer instead of burning a consensus round it cannot win. A
1312
+ // member that has not yet applied the rival's pend has no record and simply abstains
1313
+ // from this reason; the coordinator returning the retained apply verdict
1314
+ // (getExecutedPendResult) catches that residual. Self is excluded so a redelivered pend
1315
+ // for this same action stays approvable. An unavailable block carries no `pendings` and
1316
+ // abstains (the rev branch above already fail-closes when a revision claim is at stake).
1317
+ // Reason stays plain prose: it is fed to computeSigningPayload and carried as
1318
+ // Signature.rejectReason, exactly like the stale-revision reason above.
1319
+ for (const blockId of blockIds) {
1320
+ const rivals = (blockResults[blockId]?.state?.pendings ?? []).filter(actionId => actionId !== pendRequest.actionId);
1321
+ if (rivals.length > 0) {
1322
+ log('cluster-member:validation-pending-conflict', {
1323
+ messageHash: record.messageHash,
1324
+ blockId,
1325
+ actionId: pendRequest.actionId,
1326
+ rivals
1327
+ });
1328
+ return { valid: false, reason: `pending conflict: block ${blockId} held by unresolved action(s) ${rivals.join(', ')}` };
1329
+ }
1330
+ }
1331
+
1332
+ // Re-check the transaction when a validator is configured. The unvalidatable-pend
1333
+ // policy and the throwing-validator catch live in the shared `checkPendValidation`,
1334
+ // which the storage tier runs too, so a member cannot vote approve on a shape its own
1335
+ // storage would refuse at apply. Its reasons are fed to computeSigningPayload and
1336
+ // carried as Signature.rejectReason, exactly like the stale-revision reason above, so
1337
+ // a fail-closed refusal here is signed evidence rather than a lost vote.
1338
+ const validator = this.validator;
1339
+ const validation = await checkPendValidation(
1340
+ pendRequest,
1341
+ validator && (({ transaction, operationsHash }) => validator.validate(transaction, operationsHash)),
1342
+ this.unvalidatablePendPolicy,
1343
+ event => event.kind === 'unvalidatable'
1344
+ // An operator can grep this line to see how much traffic goes unchecked.
1345
+ ? log('cluster-member:pend-unvalidatable', {
1346
+ messageHash: record.messageHash,
1347
+ actionId: pendRequest.actionId,
1348
+ policy: event.policy
1349
+ })
1350
+ : log('cluster-member:validator-fault', {
1351
+ messageHash: record.messageHash,
1352
+ error: event.error
1353
+ })
1354
+ );
1355
+ if (!validation.valid) {
1356
+ return { valid: false, reason: validation.reason };
1357
+ }
1358
+ }
1359
+ }
1360
+
1361
+ return { valid: true };
1362
+ }
1363
+
1364
+ /**
1365
+ * Promise-round check that a commit record's requested revision is not already committed HERE
1366
+ * under a different action. This is the member-side arm that keeps a DEAD rival's re-broadcast
1367
+ * commit from assembling consensus: after a race winner commits and members clear its record
1368
+ * from the reservation table, a loser's re-driven commit meets no conflict votes — without this
1369
+ * check every caught-up member would abstain (the content-digest check below abstains for
1370
+ * update-only transforms whose base moved) and the loser could reach commit-consensus for a
1371
+ * write no member will ever durably store (its apply is refused stale and tolerated as 'ahead'
1372
+ * divergence — see `applyConsensusOperation`).
1373
+ *
1374
+ * Same "must run on the promise round" rule as {@link validateCommitOperations}: the
1375
+ * commit-round vote is deliberately blind, so promise votes are the only ones that carry
1376
+ * "I checked this". The four-way rule, per committed block:
1377
+ * - no local `latest`, or `latest.rev < commit.rev` → abstain (approve). Preserves the
1378
+ * lagging-member tolerance (`coordinator-repo-commit-divergence.spec.ts`): a member behind
1379
+ * the commit cannot judge it.
1380
+ * - `latest.rev === commit.rev` with the SAME action abstain (approve). Idempotent
1381
+ * redelivery of an already-durable commit; rejecting would make the writer rebase and
1382
+ * re-append an action that already landed — a duplicate entry. (Storage's `alreadyDone`
1383
+ * partition returns success for this shape at apply.)
1384
+ * - `latest.rev === commit.rev` with a DIFFERENT action → reject: a rival took the revision.
1385
+ * - `latest.rev > commit.rev` → consult the {@link IRevisionActionReader} capability for who
1386
+ * holds `commit.rev`: a different action → reject; the same action → abstain (already
1387
+ * durable, history simply moved on); no record / capability absent / read fault → abstain.
1388
+ *
1389
+ * Never throws out of the vote path: any read fault is an abstain (mirroring the digest check's
1390
+ * preview-error arm), because a member that fails to vote at all is worse than one that
1391
+ * abstains. Reason stays plain prose it is fed to computeSigningPayload and carried as
1392
+ * Signature.rejectReason, exactly like the stale-revision pend reject.
1393
+ *
1394
+ * Residual (see the commit-tier handoff): a member that signed the winner's commit but has not
1395
+ * yet APPLIED it sits in a window where it holds neither the winner's record (reservation
1396
+ * dropped at commit-sign, `shouldPersist = false`) nor the winner's revision (storage still
1397
+ * behind) — it abstains here. A capability-less member, or one with history truncated below
1398
+ * `latest`, abstains at `latest.rev > commit.rev` too. A rival's commit can therefore still
1399
+ * pass the promise round if EVERY member is simultaneously in one of those states — on a fast
1400
+ * cohort that window is the COMMON case, not the corner. The backstop is downstream of
1401
+ * consensus: every member's apply then refuses the rival as stale, the coordinating node's own
1402
+ * member retains that refusal (`getExecutedCommitResult`), and `CoordinatorRepo.commit`
1403
+ * confirms the rival against local storage and answers the writer with a retryable conflict
1404
+ * instead of a fabricated success. `ConflictRaceLostError` conversion and classified
1405
+ * rejections close the re-drive route the same way.
1406
+ */
1407
+ private async validateCommitRevisions(record: ClusterRecord): Promise<{ valid: boolean; reason?: string }> {
1408
+ for (const operation of record.message.operations) {
1409
+ if (!('commit' in operation)) {
1410
+ continue;
1411
+ }
1412
+ const commit = operation.commit;
1413
+ let blockResults: Awaited<ReturnType<IRepo['get']>>;
1414
+ try {
1415
+ // The member's raw storage repo (no cluster recursion) — the same seam
1416
+ // validatePendOperations reads on every pend vote.
1417
+ blockResults = await this.storageRepo.get({ blockIds: commit.blockIds });
1418
+ } catch (err) {
1419
+ log('cluster-member:commit-staleness-read-error', {
1420
+ messageHash: record.messageHash,
1421
+ actionId: commit.actionId,
1422
+ error: err instanceof Error ? err.message : String(err)
1423
+ });
1424
+ continue; // a local read fault is an abstain, never an escape out of the vote path
1425
+ }
1426
+ for (const blockId of commit.blockIds) {
1427
+ const latest = blockResults[blockId]?.state?.latest;
1428
+ if (!latest || latest.rev < commit.rev) {
1429
+ continue; // behind (or block never seen): cannot judge — abstain
1430
+ }
1431
+ if (isOwnRevision(latest, commit.rev, commit.actionId)) {
1432
+ continue; // idempotent redelivery of an already-durable commit — MUST NOT reject
1433
+ }
1434
+ if (latest.rev === commit.rev) {
1435
+ log('cluster-member:validation-stale-commit', {
1436
+ messageHash: record.messageHash,
1437
+ blockId,
1438
+ actionId: commit.actionId,
1439
+ rev: commit.rev,
1440
+ committedBy: latest.actionId
1441
+ });
1442
+ return { valid: false, reason: `stale commit: block ${blockId} rev ${commit.rev} committed by a different action` };
1443
+ }
1444
+ // latest.rev > commit.rev: latest can no longer name who took commit.rev — ask the
1445
+ // revision index. Structural probe, same pattern as previewCommitDigest below: a repo
1446
+ // without the capability abstains.
1447
+ const reader = this.storageRepo as IRepo & Partial<IRevisionActionReader>;
1448
+ if (typeof reader.getRevisionAction !== 'function') {
1449
+ continue;
1450
+ }
1451
+ let takenBy: ActionId | undefined;
1452
+ try {
1453
+ takenBy = await reader.getRevisionAction(blockId, commit.rev);
1454
+ } catch (err) {
1455
+ log('cluster-member:commit-staleness-revision-read-error', {
1456
+ messageHash: record.messageHash,
1457
+ blockId,
1458
+ rev: commit.rev,
1459
+ error: err instanceof Error ? err.message : String(err)
1460
+ });
1461
+ continue; // read fault → abstain
1462
+ }
1463
+ if (takenBy !== undefined && takenBy !== commit.actionId) {
1464
+ log('cluster-member:validation-stale-commit', {
1465
+ messageHash: record.messageHash,
1466
+ blockId,
1467
+ actionId: commit.actionId,
1468
+ rev: commit.rev,
1469
+ committedBy: takenBy,
1470
+ latestRev: latest.rev
1471
+ });
1472
+ return { valid: false, reason: `stale commit: block ${blockId} rev ${commit.rev} committed by a different action` };
1473
+ }
1474
+ // takenBy === commit.actionId (already durable, history moved on) or undefined
1475
+ // (truncated history — unknown): abstain either way.
1476
+ }
1477
+ }
1478
+ return { valid: true };
1479
+ }
1480
+
1481
+ /**
1482
+ * Promise-round check of a commit record's declared content digests
1483
+ * (`CommitRequest.blockDigests`) against what this member's OWN pended copy of each transform
1484
+ * would materialize (`StorageRepo.previewCommitDigest`). This is what makes a promise approval on
1485
+ * a commit record MEAN something about content: before this hook, the promise round validated
1486
+ * nothing for commits (`validatePendOperations` only inspects pend operations).
1487
+ *
1488
+ * MUST run on the promise round, not the commit round: the commit-round vote is cast deliberately
1489
+ * blind — `getTransactionPhase` signs the commit whenever promise approvals reach super-majority,
1490
+ * regardless of this member's own promise vote — so promise approvals are the only votes that
1491
+ * carry "I checked this". Do not move it.
1492
+ *
1493
+ * Checkable/abstain rule, keyed on the member's OWN pended transform (the payload the client
1494
+ * authored, delivered at pend — a hostile declarer cannot force or dodge a check by mis-declaring
1495
+ * `baseRev`):
1496
+ * - transform carries an `insert` → base-independent, ALWAYS check (declared `baseRev` ignored);
1497
+ * - `updates` only → check iff this member's local base rev equals the declared `baseRev`
1498
+ * (StorageRepo.commit accepts any `latest.rev < request.rev`, so a lagging member applying an
1499
+ * update-only transform to an older base legitimately materializes different bytes);
1500
+ * - `delete` only / no base / unmaterializable base → materializes nothing to compare, abstain;
1501
+ * - no pending transform for the action (this member never saw the pend) → abstain.
1502
+ * "Abstain" = contribute no content attestation: approve exactly as before this check existed.
1503
+ *
1504
+ * Residual: a false digest survives only when the declarer lies AND enough of the cohort is
1505
+ * simultaneously unable to check (lagging on update-only blocks, missed pends) that no honest
1506
+ * checker is left any single caught-up honest member rejects. Strictly stronger than before,
1507
+ * when commit signatures bound no content at all. Verifying/persisting a durable content proof is
1508
+ * later work (persist-block-commit-proof).
1509
+ */
1510
+ private async validateCommitOperations(record: ClusterRecord): Promise<{ valid: boolean; reason?: string }> {
1511
+ // Capability probe: `storageRepo` is typed IRepo, and only a repo that owns the local
1512
+ // materialization can preview one. A repo without the capability abstains everywhere (also
1513
+ // keeps mock-repo harnesses and non-storage compositions on the legacy approve path).
1514
+ // NOTE: probing structurally means a decorating/caching repo later inserted at this seam
1515
+ // silently disables the whole check with no signal. `ICommitDigestPreviewer` exists so such a
1516
+ // decorator has a named contract to forward; if a non-forwarding wrapper is ever wired here,
1517
+ // promote this to a typed component field rather than widening the probe.
1518
+ const repo = this.storageRepo as IRepo & Partial<ICommitDigestPreviewer>;
1519
+ if (typeof repo.previewCommitDigest !== 'function') {
1520
+ return { valid: true };
1521
+ }
1522
+
1523
+ for (const operation of record.message.operations) {
1524
+ if (!('commit' in operation)) {
1525
+ continue;
1526
+ }
1527
+ const commit = operation.commit;
1528
+ // An upgraded member receiving a commit with no declarations abstains everywhere.
1529
+ if (!commit.blockDigests) {
1530
+ continue;
1531
+ }
1532
+ // One Set per commit operation: the surplus-entry filter below is a membership test per
1533
+ // declared id, and `blockIds` is a per-coordinator batch that can be wide.
1534
+ const committedIds = new Set<string>(commit.blockIds);
1535
+ // NOTE: previews run one block at a time, so a commit declaring N blocks adds N sequential
1536
+ // preview round-trips (each 1-3 block-storage reads plus a structuredClone of the base and
1537
+ // the transform) to this member's promise-round latency. Unmeasured, and sequencing buys the
1538
+ // short-circuit on the first mismatch. If wide commits ever show up as promise latency,
1539
+ // fan the previews out with Promise.all and reduce the results, rather than sampling a
1540
+ // subset of the declared ids — a skipped id is an unchecked id.
1541
+ for (const [blockId, declared] of Object.entries(commit.blockDigests)) {
1542
+ // Surplus (or hostile) entry for a block this commit does not even cover: ignore it —
1543
+ // never throw out of the vote path, and never reject on content nobody is committing.
1544
+ if (!committedIds.has(blockId)) {
1545
+ continue;
1546
+ }
1547
+ // `blockDigests` is untrusted wire data with no ingress schema behind it, so the entry
1548
+ // need not be the shape the type promises. A malformed entry is treated as an omitted
1549
+ // one (abstain) rather than a mismatch: rejecting on it would let a garbled request
1550
+ // look like forged content, and reading through it would throw a TypeError out of the
1551
+ // vote path — this member would then fail to vote at all instead of voting reject.
1552
+ if (typeof declared?.digest !== 'string') {
1553
+ continue;
1554
+ }
1555
+ let preview: CommitDigestPreview | undefined;
1556
+ try {
1557
+ preview = await repo.previewCommitDigest(blockId as BlockId, commit.actionId, commit.rev);
1558
+ } catch (err) {
1559
+ log('cluster-member:content-digest-preview-error', {
1560
+ messageHash: record.messageHash,
1561
+ blockId,
1562
+ error: err instanceof Error ? err.message : String(err)
1563
+ });
1564
+ continue; // a local preview fault is an abstain, never a content judgement
1565
+ }
1566
+ // No pend seen here, or the transform materializes nothing to compare (tombstone,
1567
+ // updates with no base, unmaterializable base) → abstain.
1568
+ if (preview === undefined || preview.digest === undefined) {
1569
+ continue;
1570
+ }
1571
+ // `typeof === 'number'` rather than `!== undefined` for the same untrusted-shape reason
1572
+ // as the digest guard above: a non-numeric declared baseRev can never equal a local
1573
+ // one, so it degrades to an abstain instead of comparing junk.
1574
+ const checkable = preview.baseIndependent
1575
+ || (typeof declared.baseRev === 'number' && preview.baseRev === declared.baseRev);
1576
+ if (!checkable) {
1577
+ continue;
1578
+ }
1579
+ if (preview.digest !== declared.digest) {
1580
+ log('cluster-member:content-digest-mismatch', {
1581
+ messageHash: record.messageHash,
1582
+ blockId,
1583
+ actionId: commit.actionId,
1584
+ rev: commit.rev,
1585
+ declaredDigest: declared.digest,
1586
+ declaredBaseRev: declared.baseRev,
1587
+ previewDigest: preview.digest,
1588
+ previewBaseRev: preview.baseRev,
1589
+ baseIndependent: preview.baseIndependent
1590
+ });
1591
+ // One vote per record: a single mismatching block rejects the whole record.
1592
+ return { valid: false, reason: CONTENT_DIGEST_MISMATCH };
1593
+ }
1594
+ }
1595
+ }
1596
+
1597
+ return { valid: true };
1598
+ }
1599
+
1600
+ private async handleCommitNeeded(record: ClusterRecord): Promise<ClusterRecord> {
1601
+ if (this.hasLocalCommit(record)) {
1602
+ return record;
1603
+ }
1604
+ const commitHash = await this.computeCommitHash(record);
1605
+ const sig = await this.signVote(commitHash, 'approve');
1606
+ const signature: Signature = {
1607
+ type: 'approve',
1608
+ signature: sig
1609
+ };
1610
+
1611
+ return {
1612
+ ...record,
1613
+ commits: {
1614
+ ...record.commits,
1615
+ [this.peerId.toString()]: signature
1616
+ }
1617
+ };
1618
+ }
1619
+
1620
+ /**
1621
+ * Executes operations after consensus is reached.
1622
+ *
1623
+ * @warning This method executes on ALL cluster peers, not just the coordinator.
1624
+ * Each peer independently applies the operations to its local storage.
1625
+ *
1626
+ * @pitfall **Check-then-act race** - The in-memory guard must be checked AND set
1627
+ * atomically (before any `await`) to prevent duplicate execution; JavaScript's
1628
+ * single-threaded nature makes that synchronous check-and-set atomic. The durable
1629
+ * marker, by contrast, is persisted only *after* apply succeeds — writing it eagerly
1630
+ * would leave a stuck marker on a caught fault or a crash mid-apply, silently dropping
1631
+ * the transaction on this member on redelivery.
1632
+ *
1633
+ * @pitfall **Independent node storage** - Each node has its own storage. After consensus,
1634
+ * each node applies operations locally. Nodes must fetch missing blocks from cluster
1635
+ * peers via `restoreCallback` if they don't have prior revisions.
1636
+ *
1637
+ * @see docs/internals.md "Check-Then-Act Race in Consensus" and "Independent Node Storage" pitfalls
1638
+ */
1639
+ private async handleConsensus(record: ClusterRecord): Promise<void> {
1640
+ // Check persistent store first for post-recovery dedup (in-memory map is cleared on restart).
1641
+ // wasTransactionExecutedAsync also checks the in-memory map as a fast path.
1642
+ if (await this.wasTransactionExecutedAsync(record.messageHash)) {
1643
+ log('cluster-member:consensus-already-executed', { messageHash: record.messageHash });
1644
+ return;
1645
+ }
1646
+ // Check-and-set ATOMICALLY to prevent race condition where multiple calls
1647
+ // pass the async check before any completes. Since JavaScript is single-threaded,
1648
+ // this synchronous check-and-set is atomic before any await.
1649
+ if (this.executedTransactions.has(record.messageHash)) {
1650
+ log('cluster-member:consensus-already-executed', { messageHash: record.messageHash });
1651
+ return;
1652
+ }
1653
+ // Set the in-memory guard IMMEDIATELY, before any async operations: its synchronous
1654
+ // check-and-set (line above's `has` + this `set`) is what prevents the concurrent
1655
+ // apply-window race where two handleConsensus calls for the same hash both pass the
1656
+ // async check. The durable marker is deliberately NOT written here — see below.
1657
+ const executedAt = Date.now();
1658
+ this.executedTransactions.set(record.messageHash, executedAt);
1659
+
1660
+ try {
1661
+ for (const operation of record.message.operations) {
1662
+ await this.applyConsensusOperation(record, operation);
1663
+ }
1664
+ } catch (err) {
1665
+ // A genuinely unexpected fault (e.g. storage I/O) roll back the in-memory
1666
+ // marker so a corrected retry can re-run, and propagate so the caller learns
1667
+ // the real cause. The durable marker was never written (it lands only after
1668
+ // apply succeeds, below), so there is nothing to roll back. Recoverable local
1669
+ // divergence is absorbed inside applyConsensusOperation and never reaches here.
1670
+ // A retained pend verdict rolls back with the marker: it belongs to an apply
1671
+ // that is now considered not-executed, and a re-run will retain a fresh one.
1672
+ this.executedTransactions.delete(record.messageHash);
1673
+ this.executedPendResults.delete(record.messageHash);
1674
+ this.executedCommitResults.delete(record.messageHash);
1675
+ throw err;
1676
+ }
1677
+
1678
+ // Persist the durable marker only now that apply has actually succeeded. Writing it
1679
+ // eagerly (before the loop) would leave a stuck marker on a caught fault OR a crash
1680
+ // mid-apply, and on redelivery handleConsensus short-circuits at the async
1681
+ // wasTransactionExecuted checksilently dropping the transaction on this member
1682
+ // forever. The durable marker exists only for post-restart dedup (the in-memory map
1683
+ // is empty after restart), and the narrow window between "apply succeeded" and
1684
+ // "durable write landed" is safe to re-run on restart: re-applying an
1685
+ // already-applied consensus transaction is idempotent (the "ahead" divergence path
1686
+ // in applyConsensusOperation tolerates it as a no-op), so it converges rather than
1687
+ // dropping. Fire-and-forget: a persist failure must not fail the apply that succeeded.
1688
+ this.stateStore?.markExecuted(record.messageHash, executedAt)
1689
+ .catch(err => log('cluster-member:persist-executed-error', { messageHash: record.messageHash, error: (err as Error).message }));
1690
+ }
1691
+
1692
+ /**
1693
+ * Applies one consensus-approved operation to local storage.
1694
+ *
1695
+ * By the time we reach here the cluster has *already* reached consensus, so the
1696
+ * operation is authoritative cluster-wide. A failure applying it to THIS member's
1697
+ * local store therefore does not mean the operation is invalid it means our
1698
+ * local state has diverged from the agreed history:
1699
+ *
1700
+ * - **ahead**: we already hold a newer revision, so a stale pend/commit is a
1701
+ * no-op for us (`StorageRepo.commit` returns `success:false` with `missing`);
1702
+ * - **behind**: we missed the prior `pend` cluster-transaction (cohort drift
1703
+ * between the independent pend and commit phases, or transient unreachability),
1704
+ * so we lack the pending action `StorageRepo.commit` *throws* "Pending
1705
+ * action not found";
1706
+ * - **behind (no base)**: we DID see the pend, but we never saw the revision that
1707
+ * created the block, so the transform has nothing to apply to
1708
+ * `StorageRepo.commit` returns `success:false` with a `missing-base-revision`
1709
+ * reason rather than recording a revision it could not materialize.
1710
+ *
1711
+ * For both **behind** cases we hold no usable revision of the committed blocks, so we
1712
+ * actively reconcile: pull the committed revision from a cohort peer that holds it
1713
+ * (`reconcileBlock`) and restore it locally. Lazy read-repair on a later read cannot
1714
+ * recover it on its own when cohort drift has left the block under-replicated (no
1715
+ * reachable peer the reader sees holds the newer rev), so reconciling here is what
1716
+ * keeps cross-cohort transactions converging. For the **ahead** case we already hold
1717
+ * the committed rev, so we tolerate the no-op without reconciling downward.
1718
+ *
1719
+ * Whatever happens, we must NOT reset the stream: throwing here would reset the
1720
+ * cluster stream the coordinator is awaiting and surface as a spurious
1721
+ * `StreamResetError`, sinking an otherwise-successful transaction. So divergence
1722
+ * and any reconciliation failure is tolerated (logged for observability). A
1723
+ * genuinely *invalid* pend can never get this far: it is rejected during the promise
1724
+ * phase (`validatePendOperations`, which validates pend ops only — commits carry no
1725
+ * promise-phase validation).
1726
+ *
1727
+ * The propagate-vs-tolerate split keys off the failure's *nature* via `CommitResult`,
1728
+ * not throw-vs-return: a missing pend (thrown "not found") or a stale/ahead commit
1729
+ * (`success:false` with `missing`) is divergence and tolerated, whereas a genuine
1730
+ * mid-commit `internalCommit` fault (`success:false` with a bare `reason`, no
1731
+ * `missing`) is propagated so {@link handleConsensus} rolls back the executed marker
1732
+ * and rethrows — exactly like an unexpected thrown fault.
1733
+ */
1734
+ private async applyConsensusOperation(record: ClusterRecord, operation: RepoMessage['operations'][number]): Promise<void> {
1735
+ const messageHash = record.messageHash;
1736
+ if ('get' in operation) {
1737
+ await this.storageRepo.get(operation.get);
1738
+ return;
1739
+ }
1740
+ if ('cancel' in operation) {
1741
+ await this.storageRepo.cancel(operation.cancel.actionRef);
1742
+ return;
1743
+ }
1744
+ if ('pend' in operation) {
1745
+ const result = await this.storageRepo.pend(operation.pend);
1746
+ // Retain the verdict either way so the coordinator can hand the writer storage's real
1747
+ // answer (see getExecutedPendResult). A refusal here is NOT local divergence the way a
1748
+ // commit refusal is: pend-consensus confers no durability — a refusal carrying `pending`
1749
+ // (a rival's unresolved action holds the blocks) or `missing` (the requested revision is
1750
+ // already committed) is the optimistic-concurrency verdict, and swallowing it acknowledged
1751
+ // writes that no member stored.
1752
+ this.executedPendResults.set(messageHash, result);
1753
+ if (!result.success) {
1754
+ log('cluster-member:consensus-pend-diverged', {
1755
+ messageHash,
1756
+ actionId: operation.pend.actionId,
1757
+ reason: result.reason,
1758
+ hasMissing: !!result.missing?.length,
1759
+ hasPending: !!result.pending?.length
1760
+ });
1761
+ }
1762
+ return;
1763
+ }
1764
+ if ('commit' in operation) {
1765
+ const commit = operation.commit;
1766
+ // Capture the consensus commit cert BEFORE applying to storage: StorageRepo.commit emits the
1767
+ // CollectionChangeEvent synchronously at the end of the call below, and the reactivity bridge
1768
+ // resolves the cert from that event — so it must already be retained when commit() returns.
1769
+ //
1770
+ // The commit-vote signed preimage is computed HERE, before the synchronous capture+commit
1771
+ // sequence below: it is the exact bytes each approving signer endorsed
1772
+ // (`utf8(commitHash + ":approve")`), reproduced from this already-validated `record` (its
1773
+ // commit signatures were verified against the same `computeCommitHash`). Reactivity sets a
1774
+ // notification's `digest` from it so a subscriber's threshold-verify over `digest` succeeds.
1775
+ // The one `await` happens before `captureCommitCert` runs, so the cert is still retained
1776
+ // synchronously before `commit()` emits its change event (do not move this past the commit).
1777
+ // Gated on the sink: with no reactivity wired the preimage has no consumer, so a sink-less
1778
+ // node pays neither the extra `sha256` nor the extra microtask — the true zero-cost default.
1779
+ if (this.onCommitCertificate) {
1780
+ const commitSignedPayload = clusterVoteSigningPayload(await this.computeCommitHash(record), 'approve');
1781
+ this.captureCommitCert(record, commit.actionId, commitSignedPayload);
1782
+ }
1783
+ // Project the consensus record into a durable BlockCommitProof and hand it down the commit
1784
+ // path StorageRepo persists it only where the local materialization matches the declared
1785
+ // digest (see persistProofIfContentMatches). A cheap projection (no hashing/signing).
1786
+ // `undefined` for a v1 / unversioned record: its hashes bind no peer set, so it is never
1787
+ // certifiable logged so an operator can see why a cohort retains no proofs. The cast is the
1788
+ // named ICommitProofPersister contract; a plain IRepo mock ignores the extra argument.
1789
+ const proof = buildBlockCommitProof(record);
1790
+ if (proof === undefined) {
1791
+ log('cluster-member:commit-proof-skipped', {
1792
+ messageHash,
1793
+ actionId: commit.actionId,
1794
+ membershipVersion: record.membershipVersion
1795
+ });
1796
+ }
1797
+ let result: CommitResult;
1798
+ try {
1799
+ result = await (this.storageRepo as IRepo & ICommitProofPersister).commit(commit, undefined, proof);
1800
+ } catch (err) {
1801
+ // `StorageRepo.commit` throws (rather than returning success:false) when
1802
+ // the pending action is missing — the canonical "behind" signal: this
1803
+ // member reached commit-consensus without the matching pend (cohort drift).
1804
+ if (isMissingPendingActionError(err)) {
1805
+ log('cluster-member:consensus-commit-diverged', {
1806
+ messageHash,
1807
+ actionId: commit.actionId,
1808
+ divergence: 'behind',
1809
+ reason: (err as Error).message
1810
+ });
1811
+ // We hold no revision of these blocks; pull the committed revision from a
1812
+ // cohort peer so the block is no longer under-replicated. Best-effort:
1813
+ // failures are logged inside, never thrown (a throw would reset the stream).
1814
+ await this.reconcileDivergentCommit(record, commit);
1815
+ return;
1816
+ }
1817
+ throw err;
1818
+ }
1819
+ // Retain the verdict either way (see getExecutedCommitResult): a success confirms local
1820
+ // durability, and an ahead-shaped refusal is the only evidence the coordinator has that
1821
+ // the tolerance below swallowed a rival's win at the requested revision. The
1822
+ // missing-pending throw path above retains nothing — no CommitResult exists there, and
1823
+ // the coordinator's fabricated-success fallback plus cohort reconcile is the right shape
1824
+ // for a member that is genuinely behind.
1825
+ this.executedCommitResults.set(messageHash, result);
1826
+ if (!result.success) {
1827
+ // success:false is a StaleFailure. `missing` ⇒ ahead/stale divergence
1828
+ // (we already hold ≥ this rev): tolerate, do NOT reconcile downward. A
1829
+ // missing-base reason behind divergence, reconcile (below). Any other bare
1830
+ // `reason` with no `missing` a genuine internalCommit fault: propagate so
1831
+ // handleConsensus rolls back the executed marker and rethrows.
1832
+ //
1833
+ // NOTE: this 'ahead' tolerance is what turns a rival's commit that somehow reaches
1834
+ // consensus into a reported success no member durably stored (consensus without
1835
+ // durability — the commit-tier acknowledgement hole). It must stay: a member
1836
+ // genuinely ahead of a redelivered/lagging commit is the common, correct case. The
1837
+ // guards live UPSTREAM: `validateCommitRevisions` rejects the rival at the promise
1838
+ // round, `CoordinatorRepo.commit` returns lost races as retryable conflicts instead
1839
+ // of re-driving them, and the verdict retained just above (getExecutedCommitResult)
1840
+ // lets the coordinating node convert its OWN member's rival-confirmed refusal into a
1841
+ // conflict answer — that last guard is what closes the signed-but-not-yet-applied
1842
+ // window, where two commits for one revision both assemble consensus because signing
1843
+ // drops each member's reservation before applying advances its storage. If
1844
+ // consensus-without-durability is ever observed again, look at those guards' abstain
1845
+ // residuals (non-coordinating members' verdicts are not threaded back; capability-less
1846
+ // or history-truncated storage abstains), not at this branch.
1847
+ if (result.missing?.length) {
1848
+ log('cluster-member:consensus-commit-diverged', {
1849
+ messageHash,
1850
+ actionId: commit.actionId,
1851
+ divergence: 'ahead',
1852
+ reason: result.reason,
1853
+ hasMissing: true
1854
+ });
1855
+ return;
1856
+ }
1857
+ // This member holds no materializable base for one of the blocks, so
1858
+ // `StorageRepo.commit` REFUSED rather than record a revision it could never serve.
1859
+ // Same "behind" divergence as a missing pend — and the same cure: pull the committed
1860
+ // revision from a cohort peer. Reconciling here (after commit released its per-block
1861
+ // latches) is what makes refusing safe; fetching inside the commit path would deadlock
1862
+ // against the latch `saveReplicatedBlock` needs to persist what it fetched.
1863
+ if (isMissingBaseRevisionFailure(result)) {
1864
+ log('cluster-member:consensus-commit-diverged', {
1865
+ messageHash,
1866
+ actionId: commit.actionId,
1867
+ divergence: 'behind',
1868
+ reason: result.reason
1869
+ });
1870
+ await this.reconcileDivergentCommit(record, commit);
1871
+ return;
1872
+ }
1873
+ throw new Error(`Consensus commit for action ${commit.actionId} failed: ${result.reason ?? 'unknown reason'}`);
1874
+ }
1875
+ return;
1876
+ }
1877
+ if ('invalidate' in operation) {
1878
+ await this.applyConsensusInvalidation(record, operation.invalidate);
1879
+ return;
1880
+ }
1881
+ }
1882
+
1883
+ /**
1884
+ * Applies a consensus-ordered invalidation on this member: dedup → certificate verification →
1885
+ * capture the invalidation's commit cert (for reactivity reuse) → delegate the compensating write +
1886
+ * durable log append to the injected {@link InvalidationApplySink}.
1887
+ *
1888
+ * Like every other branch of {@link applyConsensusOperation}, a failure here is tolerated rather
1889
+ * than thrown — a throw would reset the cluster stream. A forged/sub-threshold certificate is
1890
+ * rejected (and never reaches the sink); a sink fault is logged and its dedup marker rolled back
1891
+ * so a re-broadcast can retry. The durable, authoritative dedup is the invalidation log entry the
1892
+ * sink consults; the in-memory map is only a fast path.
1893
+ */
1894
+ private async applyConsensusInvalidation(record: ClusterRecord, request: InvalidateRequest): Promise<void> {
1895
+ const messageHash = record.messageHash;
1896
+ const dedupKey = `${request.invalidatedActionId}:${request.resolution.disputeId}`;
1897
+ if (this.appliedInvalidations.has(dedupKey)) {
1898
+ log('cluster-member:consensus-invalidate-duplicate', { messageHash, dedupKey });
1899
+ return;
1900
+ }
1901
+
1902
+ // Certificate verification BEFORE apply — never trust the originator's say-so. Verify the proof
1903
+ // against THIS request's own target: the votes are bound to the transaction the dispute resolved,
1904
+ // so a genuine proof carried in a request that points at a different (innocent) action/blocks fails
1905
+ // here (the network-facing replay boundary, #2). The proof is also bound to the legitimately-selected
1906
+ // arbitrator set (#1): membership + the challenger's set signature gate it (layer 1), and when a
1907
+ // recompute capability is wired we re-derive the eligible set from our topology and reject a forged
1908
+ // one (layer 2). When we cannot reconstruct the historical topology, we accept on layer 1 and LOG
1909
+ // that the invalidation was applied without full anchoring — the documented interim posture.
1910
+ const certified = await verifyInvalidationCertificate(
1911
+ request.resolution,
1912
+ { invalidatedActionId: request.invalidatedActionId, blockIds: request.blockIds },
1913
+ {
1914
+ recomputeArbitratorSet: this.recomputeArbitratorSet,
1915
+ onUnanchored: (info) => log('cluster-member:consensus-invalidate-unanchored', {
1916
+ messageHash,
1917
+ invalidatedActionId: request.invalidatedActionId,
1918
+ disputeId: info.disputeId,
1919
+ reason: info.reason,
1920
+ arbitratorSetSize: info.arbitratorSet.length
1921
+ })
1922
+ }
1923
+ );
1924
+ if (!certified) {
1925
+ log('cluster-member:consensus-invalidate-reject-certificate', {
1926
+ messageHash,
1927
+ invalidatedActionId: request.invalidatedActionId,
1928
+ disputeId: request.resolution.disputeId,
1929
+ outcome: request.resolution.outcome
1930
+ });
1931
+ return;
1932
+ }
1933
+
1934
+ // Capture the invalidation's own commit cert the threshold signature the cohort produced for
1935
+ // THIS consensus-ordered `invalidate` op (over `computeCommitHash(record)`), the reversal analogue
1936
+ // of the commit-cert capture in {@link applyConsensusOperation}. Reactivity reuses it bit-for-bit
1937
+ // as the invalidation notification's `sig` (never re-signed). Keyed on the deterministic
1938
+ // {@link invalidationActionId} the invalidation's change event also carries, so the bridge's
1939
+ // cert extractor resolves it. Gated on the sink — a node with no reactivity wired pays nothing.
1940
+ if (this.onCommitCertificate) {
1941
+ const invSignedPayload = clusterVoteSigningPayload(await this.computeCommitHash(record), 'approve');
1942
+ this.captureCommitCert(record, invalidationActionId(request.invalidatedActionId, request.resolution.disputeId), invSignedPayload);
1943
+ }
1944
+
1945
+ if (!this.onInvalidate) {
1946
+ log('cluster-member:consensus-invalidate-no-sink', { messageHash, dedupKey });
1947
+ return;
1948
+ }
1949
+
1950
+ this.appliedInvalidations.set(dedupKey, Date.now());
1951
+ try {
1952
+ await this.onInvalidate(request);
1953
+ log('cluster-member:consensus-invalidate-applied', {
1954
+ messageHash,
1955
+ invalidatedActionId: request.invalidatedActionId,
1956
+ disputeId: request.resolution.disputeId,
1957
+ blockCount: request.blockIds.length
1958
+ });
1959
+ } catch (err) {
1960
+ // Tolerate (don't reset the stream); roll back the marker so a re-broadcast retries.
1961
+ this.appliedInvalidations.delete(dedupKey);
1962
+ log('cluster-member:consensus-invalidate-sink-error', { messageHash, dedupKey, error: (err as Error).message });
1963
+ }
1964
+ }
1965
+
1966
+ /**
1967
+ * Record the consensus commit cert for `actionId` into the injected {@link CommitCertificateSink},
1968
+ * if one is wired. The cert is built from the agreed `record.commits` (the per-member `approve`
1969
+ * commit signatures), forwarded UNCHANGED for reactivity to reuse. `signedPayload` is the exact
1970
+ * commit-vote preimage those signatures were produced over (`utf8(commitHash + ":approve")`),
1971
+ * computed by the caller from the same `record` *before* the synchronous commit — reactivity sets a
1972
+ * notification's `digest` from it so a real threshold-verify over `digest` succeeds. Stays
1973
+ * **synchronous** (no `await` here) so the cert is retained before `StorageRepo.commit` emits its
1974
+ * change event. No-op (zero cost) when no sink is configured; a throwing sink is isolated + logged so
1975
+ * it can never break consensus.
1976
+ */
1977
+ private captureCommitCert(record: ClusterRecord, actionId: ActionId, signedPayload: Uint8Array): void {
1978
+ if (!this.onCommitCertificate) {
1979
+ return;
1980
+ }
1981
+ const minSigs = Math.ceil(Object.keys(record.peers).length * this.superMajorityThreshold);
1982
+ try {
1983
+ this.onCommitCertificate(actionId, buildCommitCert(record, minSigs, signedPayload));
1984
+ } catch (err) {
1985
+ log('cluster-member:commit-cert-sink-error', { actionId, error: (err as Error).message });
1986
+ }
1987
+ }
1988
+
1989
+ /**
1990
+ * After tolerating a "behind" commit divergence, pull the committed revision of
1991
+ * each block from a cohort peer that holds it and restore it locally. Best-effort:
1992
+ * a missing callback, an empty cohort, or a per-block failure/timeout is logged and
1993
+ * tolerated never thrown, since a throw out of consensus execution resets the
1994
+ * cluster stream.
1995
+ */
1996
+ private async reconcileDivergentCommit(record: ClusterRecord, commit: CommitRequest): Promise<void> {
1997
+ if (!this.reconcileBlock) {
1998
+ log('cluster-member:consensus-commit-reconcile-skip', { messageHash: record.messageHash, reason: 'no-callback' });
1999
+ return;
2000
+ }
2001
+ const cohortPeerIds = Object.keys(record.peers).filter(id => id !== this.peerId.toString());
2002
+ if (cohortPeerIds.length === 0) {
2003
+ log('cluster-member:consensus-commit-reconcile-skip', { messageHash: record.messageHash, reason: 'no-cohort-peers' });
2004
+ return;
2005
+ }
2006
+ const committed: ActionRev = { actionId: commit.actionId, rev: commit.rev };
2007
+ await Promise.all(
2008
+ commit.blockIds.map(blockId => this.reconcileOneBlock(record.messageHash, blockId, committed, cohortPeerIds))
2009
+ );
2010
+ }
2011
+
2012
+ /** Reconcile a single block, bounding the awaited callback and swallowing failures. */
2013
+ private async reconcileOneBlock(messageHash: string, blockId: BlockId, committed: ActionRev, cohortPeerIds: string[]): Promise<void> {
2014
+ try {
2015
+ await this.withReconcileTimeout(this.reconcileBlock!(blockId, committed, cohortPeerIds), blockId);
2016
+ // "attempted", not "reconciled": the callback returns void, and a quorum decline is a
2017
+ // normal, non-throwing outcome — so reaching here means the pass ran to completion, NOT
2018
+ // that anything was restored. `reconcile:restored` (reconcile-block.ts) is the line that
2019
+ // says the bytes actually landed; `reconcile:no-rev-quorum` / `reconcile:no-content-quorum`
2020
+ // say they did not.
2021
+ log('cluster-member:consensus-commit-reconcile-attempted', { messageHash, blockId, rev: committed.rev });
2022
+ } catch (err) {
2023
+ log('cluster-member:consensus-commit-reconcile-failed', {
2024
+ messageHash,
2025
+ blockId,
2026
+ rev: committed.rev,
2027
+ error: (err as Error).message
2028
+ });
2029
+ }
2030
+ }
2031
+
2032
+ /** Bound an awaited reconcile so a slow/unreachable cohort peer can't stall consensus. */
2033
+ private withReconcileTimeout<T>(promise: Promise<T>, blockId: BlockId): Promise<T> {
2034
+ let timer: NodeJS.Timeout | undefined;
2035
+ const timeout = new Promise<never>((_, reject) => {
2036
+ timer = setTimeout(
2037
+ () => reject(new Error(`reconcile for block ${blockId} timed out after ${ReconcileTimeoutMs}ms`)),
2038
+ ReconcileTimeoutMs
2039
+ );
2040
+ timer.unref();
2041
+ });
2042
+ return Promise.race([promise, timeout]).finally(() => {
2043
+ if (timer) clearTimeout(timer);
2044
+ });
2045
+ }
2046
+
2047
+ private async handleRejection(_record: ClusterRecord): Promise<void> {
2048
+ // Clean up any resources - will be cleared by shouldPersist = false in the main flow
2049
+ }
2050
+
2051
+ private setupTimeouts(record: ClusterRecord): { promiseTimeout?: NodeJS.Timeout; resolutionTimeout?: NodeJS.Timeout } {
2052
+ if (!record.message.expiration) {
2053
+ return {};
2054
+ }
2055
+
2056
+ return {
2057
+ promiseTimeout: setTimeout(
2058
+ () => this.handleExpiration(record.messageHash),
2059
+ record.message.expiration - Date.now()
2060
+ ).unref(),
2061
+ resolutionTimeout: setTimeout(
2062
+ () => this.resolveWithPeers(record.messageHash),
2063
+ record.message.expiration + 5000 - Date.now()
2064
+ ).unref()
2065
+ };
2066
+ }
2067
+
2068
+ /**
2069
+ * Scan this member's reservation table (`activeTransactions`) for a held transaction that
2070
+ * conflicts with `record` AND wins the deterministic race against it. Returns the winner's
2071
+ * identity — `{ blockedBy: messageHash }` — rather than a bare boolean, because that identity is
2072
+ * exactly what the resulting conflict vote must name (`Signature.conflictWith`); the old boolean
2073
+ * lost it. `undefined` means no blocking conflict. Side-effectful on the way through: stale
2074
+ * entries are swept, and a held transaction that LOSES the race to `record` is cleared.
2075
+ */
2076
+ private findConflict(record: ClusterRecord): { blockedBy: string } | undefined {
2077
+ const now = Date.now();
2078
+ const staleThresholdMs = 2000; // 2 seconds - allow more time for distributed consensus
2079
+
2080
+ const incomingBlockIds = this.getAffectedBlockIds(record.message.operations);
2081
+ log('cluster-member:findConflict-check', {
2082
+ messageHash: record.messageHash,
2083
+ activeCount: this.activeTransactions.size,
2084
+ incomingBlockIds
2085
+ });
2086
+
2087
+ for (const [existingHash, state] of Array.from(this.activeTransactions.entries())) {
2088
+ if (existingHash === record.messageHash) {
2089
+ continue;
2090
+ }
2091
+
2092
+ const existingBlockIds = this.getAffectedBlockIds(state.record.message.operations);
2093
+ log('cluster-member:findConflict-compare', {
2094
+ existing: existingHash,
2095
+ incoming: record.messageHash,
2096
+ existingBlockIds,
2097
+ incomingBlockIds
2098
+ });
2099
+
2100
+ // Clean up stale transactions that have been around too long
2101
+ if (now - state.lastUpdate > staleThresholdMs) {
2102
+ log('cluster-member:stale-cleanup', {
2103
+ messageHash: existingHash,
2104
+ age: now - state.lastUpdate
2105
+ });
2106
+ this.clearTransaction(existingHash);
2107
+ continue;
2108
+ }
2109
+
2110
+ if (this.operationsConflict(state.record.message.operations, record.message.operations)) {
2111
+ // Use race resolution to determine winner
2112
+ const resolution = this.resolveRace(state.record, record);
2113
+
2114
+ if (resolution === 'keep-existing') {
2115
+ log('cluster-member:race-keep-existing', {
2116
+ existing: existingHash,
2117
+ incoming: record.messageHash
2118
+ });
2119
+ return { blockedBy: existingHash }; // Reject incoming, naming the winner
2120
+ } else {
2121
+ // Accept incoming, abort existing
2122
+ log('cluster-member:race-accept-incoming', {
2123
+ existing: existingHash,
2124
+ incoming: record.messageHash
2125
+ });
2126
+ this.clearTransaction(existingHash);
2127
+ continue; // Check other conflicts
2128
+ }
2129
+ }
2130
+ }
2131
+
2132
+ return undefined; // No blocking conflicts
2133
+ }
2134
+
2135
+ /** Number of *approve* promise votes on a record — the count the commit rule uses. */
2136
+ private static approvalCount(record: ClusterRecord): number {
2137
+ return Object.values(record.promises).filter(s => s.type === 'approve').length;
2138
+ }
2139
+
2140
+ /**
2141
+ * Resolve a race between two conflicting transactions. Total and deterministic, so every honest
2142
+ * member computes the identical winner (the Theorem 1 Case-2 premise). Order:
2143
+ * 1. more *approve* promise signatures wins (progress monotonicity — see safety note below);
2144
+ * 2. equal approval counts → higher aged priority wins (fairness — see {@link recordPriority});
2145
+ * 3. still tied → higher message hash wins.
2146
+ *
2147
+ * The count is APPROVALS, not `promises` keys. `promises` is the vote map — a reject occupies a key
2148
+ * there exactly as an approve does — so counting keys would treat a rejection as progress, letting a
2149
+ * record that can never commit outrank (and therefore block, via {@link findConflict}) a fresh rival
2150
+ * for the whole staleness window. Approvals is also the count the invariant below actually needs:
2151
+ * the commit rule is `approvedPromises >= superMajority`, which never looks at rejections.
2152
+ *
2153
+ * Approval count is FIRST so this comparison never displaces a transaction that is further along.
2154
+ * That restores the pre-priority safety invariant: a member commits purely on promise supermajority
2155
+ * (`handleCommitNeeded` signs whenever `approvedPromises >= superMajority`; the commit path has NO
2156
+ * conflict re-check), so `resolveRace` is the ONLY arbiter among concurrently-pending conflicts.
2157
+ * With approvals-first, once transaction X holds a promise supermajority every conflicting rival Y has
2158
+ * strictly fewer approvals — Y can only match X's count by getting the intersecting quorum member to
2159
+ * approve it, but that member already holds X at supermajority and `resolveRace(X, Y)` returns
2160
+ * `keep-existing` on X's higher count, so it never does. By quorum intersection any Y-supermajority
2161
+ * overlaps X's in ≥1 honest member, and that member rejects Y. One winner (docs/correctness.md
2162
+ * Theorem 9). Priority-first would break this: it could displace an already-quorum-reached X for a
2163
+ * higher-priority Y with fewer approvals, letting BOTH commit (split brain) — the regression fixed by
2164
+ * ticket occ-priority-first-breaks-promise-monotonicity.
2165
+ *
2166
+ * Priority is now a tie-break that runs only at EQUAL approval counts, which is exactly the
2167
+ * concurrent-starvation case aging targets (two fresh rivals, 0 promises each, otherwise coin-flipping
2168
+ * on the hash). Priority still breaks those ties deterministically, so aging still solves the stated
2169
+ * fairness problem in its common case. It only orders two *concurrently-pending* conflicts; it does NOT
2170
+ * defer a fresh pend for an absent aged transaction (that residual — sequential sub-window starvation —
2171
+ * is the deferred feat-occ-priority-reservation).
2172
+ *
2173
+ * NOTE: residual-fairness tripwire. Under approvals-first an aged transaction can still lose to a fresh
2174
+ * rival that has *legitimately* gathered even one more approval — that is not the pure-coin-flip
2175
+ * starvation aging targets (equal counts, priority wins), it is the monotonicity behaviour we WANT (a
2176
+ * more-progressed rival is never displaced). If deeper fairness against a genuinely-more-progressed
2177
+ * rival is ever needed, it belongs to feat-occ-priority-reservation (reserve/defer at pend time), NOT
2178
+ * to this race tie-break.
2179
+ *
2180
+ * NOTE: Byzantine self-assert is a fairness DoS, not a safety hole. A coordinator can stamp
2181
+ * priority == MaxPriority on every transaction; recordPriority clamps to the cap so it cannot
2182
+ * exceed it, and priority never influences validity/operationsHash/stale-read checks — and now sits
2183
+ * below the approval count, so it can only break equal-count ties it might have ~50% won anyway,
2184
+ * degrading to at-worst-status-quo fairness (the same graceful-degradation class as spam under
2185
+ * honest-majority). Binding priority to provable age is out of scope (feat-occ-priority-reservation).
2186
+ *
2187
+ * NOTE: keep priority a self-contained additive message field + this one comparison key so it
2188
+ * composes with — does not block — a future HLC/crdt-sync redesign of this same path
2189
+ * (design-hot-log-tail-sharding-guidance).
2190
+ */
2191
+ private resolveRace(existing: ClusterRecord, incoming: ClusterRecord): 'keep-existing' | 'accept-incoming' {
2192
+ // 1. Transaction with more APPROVALS wins — never displace a more-progressed rival (safety, see
2193
+ // above). Counting `promises` keys instead would count reject votes as progress: a record holding
2194
+ // one rejection would outrank an untouched rival and reserve its blocks for the whole staleness
2195
+ // window, and the commit rule this ordering protects (`approvedPromises >= superMajority`) never
2196
+ // looks at rejections anyway.
2197
+ const existingCount = ClusterMember.approvalCount(existing);
2198
+ const incomingCount = ClusterMember.approvalCount(incoming);
2199
+ if (existingCount !== incomingCount) {
2200
+ return existingCount > incomingCount ? 'keep-existing' : 'accept-incoming';
2201
+ }
2202
+
2203
+ // 2. Equal approval counts → higher aged priority wins (fairness tie-break).
2204
+ const existingPriority = this.recordPriority(existing);
2205
+ const incomingPriority = this.recordPriority(incoming);
2206
+ if (existingPriority !== incomingPriority) {
2207
+ return existingPriority > incomingPriority ? 'keep-existing' : 'accept-incoming';
2208
+ }
2209
+
2210
+ // 3. Tie-breaker: higher message hash wins (deterministic).
2211
+ return existing.messageHash > incoming.messageHash ? 'keep-existing' : 'accept-incoming';
2212
+ }
2213
+
2214
+ /**
2215
+ * Aged advisory priority carried by a record's pend operation, clamped to [0, MaxPriority].
2216
+ * The multi-collection path carries it on `pend.validation.transaction.priority`; the single-collection
2217
+ * (`Collection.sync`) path carries it as top-level `pend.priority`; a record with neither — a
2218
+ * legacy/unversioned coordinator's transaction, or a non-pend operation — is priority 0
2219
+ * (backward compatible: such transactions simply never age). Both carriers live inside the signed
2220
+ * `message`, so priority is integrity-protected in transit; clamping here bounds a self-asserted
2221
+ * out-of-range value to the cap.
2222
+ *
2223
+ * NOTE: `message` is fixed for a transaction's whole lifecycle (promises/commits accrue in the
2224
+ * separate `promises`/`commits` maps, never in `message`), so a transaction keeps its rank through
2225
+ * the commit phase — there is no "priority drops to 0 at commit" asymmetry. resolveRace is only
2226
+ * consulted at the promise decision (findConflict), i.e. between two still-open conflicting
2227
+ * transactions, which is exactly the concurrent-contention case priority is meant to order.
2228
+ */
2229
+ private recordPriority(record: ClusterRecord): number {
2230
+ for (const op of record.message.operations) {
2231
+ if ('pend' in op) {
2232
+ // Every hop optional: `validation` arrives off the wire inside a signed message whose
2233
+ // hash binds its bytes, not its shape, so a malformed pair must yield priority 0 (what
2234
+ // clampPriority already does for a missing or Byzantine number) rather than throw out
2235
+ // of the vote path — the lost vote this fail-closed pass exists to prevent.
2236
+ return clampPriority(op.pend.validation?.transaction?.priority ?? op.pend.priority);
2237
+ }
2238
+ }
2239
+ return 0;
2240
+ }
2241
+
2242
+ private operationsConflict(ops1: RepoMessage['operations'], ops2: RepoMessage['operations']): boolean {
2243
+ // Check if one is a commit for the same action as a pend - these don't conflict
2244
+ const actionId1 = this.getActionId(ops1);
2245
+ const actionId2 = this.getActionId(ops2);
2246
+ if (actionId1 && actionId2 && actionId1 === actionId2) {
2247
+ // Same action - commit is resolving the pend, not conflicting
2248
+ return false;
2249
+ }
2250
+
2251
+ const blocks1 = new Set(this.getAffectedBlockIds(ops1));
2252
+ const blocks2 = new Set(this.getAffectedBlockIds(ops2));
2253
+
2254
+ for (const block of Array.from(blocks1)) {
2255
+ if (blocks2.has(block)) {
2256
+ log('cluster-member:conflict-detected', {
2257
+ blocks1: Array.from(blocks1),
2258
+ blocks2: Array.from(blocks2),
2259
+ conflictingBlock: block
2260
+ });
2261
+ return true;
2262
+ }
2263
+ }
2264
+
2265
+ return false;
2266
+ }
2267
+
2268
+ private getActionId(operations: RepoMessage['operations']): string | undefined {
2269
+ for (const operation of operations) {
2270
+ if ('pend' in operation) {
2271
+ return operation.pend.actionId;
2272
+ } else if ('commit' in operation) {
2273
+ return operation.commit.actionId;
2274
+ } else if ('cancel' in operation) {
2275
+ return operation.cancel.actionRef.actionId;
2276
+ }
2277
+ }
2278
+ return undefined;
2279
+ }
2280
+
2281
+ /**
2282
+ * Every block id the message's own operations name. Two consumers, deliberately sharing one
2283
+ * definition: conflict detection (which writes must serialize against each other) and the membership
2284
+ * admission gate's binding check (the set a legitimate `coordinatingBlockIds[0]` must come from —
2285
+ * {@link ClusterMember.deriveExpectedClusterView}). If the two ever disagreed, a coordinator could
2286
+ * name a block the record is not judged to touch.
2287
+ */
2288
+ private getAffectedBlockIds(operations: RepoMessage['operations']): string[] {
2289
+ const blockIds = new Set<string>();
2290
+
2291
+ for (const operation of operations) {
2292
+ if ('get' in operation) {
2293
+ operation.get.blockIds.forEach(id => blockIds.add(id));
2294
+ } else if ('pend' in operation) {
2295
+ // Use blockIdsForTransforms to correctly extract block IDs from Transforms structure
2296
+ blockIdsForTransforms(operation.pend.transforms).forEach(id => blockIds.add(id));
2297
+ } else if ('commit' in operation) {
2298
+ operation.commit.blockIds.forEach(id => blockIds.add(id));
2299
+ } else if ('cancel' in operation) {
2300
+ operation.cancel.actionRef.blockIds.forEach(id => blockIds.add(id));
2301
+ } else if ('invalidate' in operation) {
2302
+ // The invalidation writes compensating revisions to these blocks; surfacing them lets
2303
+ // conflict detection serialize a concurrent commit racing the invalidation on a block.
2304
+ operation.invalidate.blockIds.forEach(id => blockIds.add(id));
2305
+ }
2306
+ }
2307
+
2308
+ return Array.from(blockIds);
2309
+ }
2310
+
2311
+ private async propagateIfNeeded(record: ClusterRecord): Promise<void> {
2312
+ const promises = [];
2313
+ for (const peerId of Object.keys(record.peers)) {
2314
+ if (peerId === this.peerId.toString()) continue;
2315
+
2316
+ try {
2317
+ const client = ClusterClient.create(peerIdFromString(peerId), this.peerNetwork, this.protocolPrefix);
2318
+ promises.push(client.update(record));
2319
+ } catch (error) {
2320
+ log('ERROR: Failed to propagate to peer %s: %o', peerId, error);
2321
+ }
2322
+ }
2323
+ await Promise.allSettled(promises);
2324
+ }
2325
+
2326
+ private async handleExpiration(messageHash: string): Promise<void> {
2327
+ const state = this.activeTransactions.get(messageHash);
2328
+ if (!state) return;
2329
+
2330
+ if (!state.record.promises[this.peerId.toString()]) {
2331
+ const rejectReason = 'Transaction expired';
2332
+ const promiseHash = await this.computePromiseHash(state.record);
2333
+ const sig = await this.signVote(promiseHash, 'reject', rejectReason);
2334
+ const signature: Signature = {
2335
+ type: 'reject',
2336
+ signature: sig,
2337
+ rejectReason
2338
+ };
2339
+
2340
+ const updatedRecord = {
2341
+ ...state.record,
2342
+ promises: {
2343
+ ...state.record.promises,
2344
+ [this.peerId.toString()]: signature
2345
+ }
2346
+ };
2347
+
2348
+ this.activeTransactions.set(messageHash, {
2349
+ ...state,
2350
+ record: updatedRecord
2351
+ });
2352
+
2353
+ await this.propagateIfNeeded(updatedRecord);
2354
+ }
2355
+ }
2356
+
2357
+ private async resolveWithPeers(messageHash: string): Promise<void> {
2358
+ // This method is disabled - the coordinator handles all retry logic
2359
+ // Keeping the skeleton in case we need peer-initiated recovery in the future
2360
+ log('cluster-member:resolve-skipped', { messageHash, reason: 'coordinator-handles-retry' });
2361
+ }
2362
+
2363
+ private queueExpiredTransactions(): void {
2364
+ const now = Date.now();
2365
+ for (const [messageHash, state] of Array.from(this.activeTransactions.entries())) {
2366
+ if (state.record.message.expiration && state.record.message.expiration < now) {
2367
+ this.cleanupQueue.push(messageHash);
2368
+ }
2369
+ }
2370
+ // Also clean up old executed transaction records
2371
+ const expirationThreshold = now - ExecutedTransactionTtlMs;
2372
+ for (const [messageHash, executedAt] of Array.from(this.executedTransactions.entries())) {
2373
+ if (executedAt < expirationThreshold) {
2374
+ this.executedTransactions.delete(messageHash);
2375
+ this.executedPendResults.delete(messageHash);
2376
+ this.executedCommitResults.delete(messageHash);
2377
+ }
2378
+ }
2379
+ // Prune old applied-invalidation dedup markers on the same TTL.
2380
+ for (const [dedupKey, appliedAt] of Array.from(this.appliedInvalidations.entries())) {
2381
+ if (appliedAt < expirationThreshold) {
2382
+ this.appliedInvalidations.delete(dedupKey);
2383
+ }
2384
+ }
2385
+ this.stateStore?.pruneExecuted(expirationThreshold)
2386
+ .catch(err => log('cluster-member:prune-executed-error', { error: (err as Error).message }));
2387
+ }
2388
+
2389
+ private async processCleanupQueue(): Promise<void> {
2390
+ while (this.cleanupQueue.length > 0) {
2391
+ const messageHash = this.cleanupQueue.shift();
2392
+ if (!messageHash) continue;
2393
+
2394
+ const state = this.activeTransactions.get(messageHash);
2395
+ if (!state) continue;
2396
+
2397
+ // NOTE: an expired entry already in a terminal phase is deliberately left alone here —
2398
+ // `processUpdate` clears those on the update that made them terminal, so reaching this point
2399
+ // in one means that update never arrived. It is not stranded: `findConflict`'s 2 s staleness
2400
+ // sweep drops it on the next conflicting arrival. But on a member that then goes idle the
2401
+ // entry lingers until traffic returns. If member memory ever shows entries outliving their
2402
+ // expiration, delete unconditionally here instead of exempting the terminal phases.
2403
+ const { phase } = await this.getTransactionPhase(state.record);
2404
+ if (phase !== TransactionPhase.Consensus && phase !== TransactionPhase.Rejected) {
2405
+ this.activeTransactions.delete(messageHash);
2406
+ }
2407
+ }
2408
+ }
2409
+
2410
+ private hasLocalCommit(record: ClusterRecord): boolean {
2411
+ const ourId = this.peerId.toString();
2412
+ return Boolean(record.commits[ourId]);
2413
+ }
2414
+
2415
+ private clearTransaction(messageHash: string): void {
2416
+ const state = this.activeTransactions.get(messageHash);
2417
+ if (!state) {
2418
+ log('cluster-member:clear-miss', { messageHash });
2419
+ return;
2420
+ }
2421
+ if (state.promiseTimeout) {
2422
+ clearTimeout(state.promiseTimeout);
2423
+ }
2424
+ if (state.resolutionTimeout) {
2425
+ clearTimeout(state.resolutionTimeout);
2426
+ }
2427
+ this.activeTransactions.delete(messageHash);
2428
+ this.stateStore?.deleteParticipantState(messageHash)
2429
+ .catch(err => log('cluster-member:persist-delete-error', { messageHash, error: (err as Error).message }));
2430
+ log('cluster-member:clear-done', {
2431
+ messageHash,
2432
+ remaining: Array.from(this.activeTransactions.keys())
2433
+ });
2434
+ }
2435
+
2436
+ /** Fire-and-forget persist — errors are logged, never thrown. */
2437
+ private persistParticipantState(messageHash: string, record: ClusterRecord): void {
2438
+ if (!this.stateStore) return;
2439
+ this.stateStore.saveParticipantState(messageHash, {
2440
+ messageHash,
2441
+ record,
2442
+ lastUpdate: Date.now()
2443
+ }).catch(err => log('cluster-member:persist-error', { messageHash, error: (err as Error).message }));
2444
+ }
2445
+
2446
+ /**
2447
+ * Recover member transactions from persistent store after a restart.
2448
+ * Called during node startup, before accepting new requests.
2449
+ */
2450
+ async recoverTransactions(): Promise<void> {
2451
+ if (!this.stateStore) return;
2452
+ const now = Date.now();
2453
+
2454
+ // 1. Prune expired executed entries from persistent store
2455
+ await this.stateStore.pruneExecuted(now - ExecutedTransactionTtlMs);
2456
+ // Note: executed transactions are checked via wasTransactionExecutedAsync() at runtime,
2457
+ // which falls back to the persistent store when the in-memory map misses.
2458
+
2459
+ // 2. Restore active participant states
2460
+ const participantStates = await this.stateStore.getAllParticipantStates();
2461
+ for (const state of participantStates) {
2462
+ const { messageHash } = state;
2463
+ // Expired — clean up
2464
+ if (state.record.message.expiration && state.record.message.expiration < now) {
2465
+ log('cluster-member:recovery-expired', { messageHash });
2466
+ await this.stateStore.deleteParticipantState(messageHash);
2467
+ continue;
2468
+ }
2469
+ // Restore into activeTransactions with fresh timeouts
2470
+ log('cluster-member:recovery-restore', { messageHash });
2471
+ const timeouts = this.setupTimeouts(state.record);
2472
+ this.activeTransactions.set(messageHash, {
2473
+ record: state.record,
2474
+ lastUpdate: state.lastUpdate,
2475
+ promiseTimeout: timeouts.promiseTimeout,
2476
+ resolutionTimeout: timeouts.resolutionTimeout
2477
+ });
2478
+ }
2479
+
2480
+ log('cluster-member:recovery-complete', {
2481
+ restoredActive: this.activeTransactions.size,
2482
+ restoredExecuted: this.executedTransactions.size
2483
+ });
2484
+ }
2485
+
2486
+ /**
2487
+ * Checks if a transaction's operations were already executed during consensus.
2488
+ * Falls back to the persistent store when the in-memory map misses.
2489
+ */
2490
+ async wasTransactionExecutedAsync(messageHash: string): Promise<boolean> {
2491
+ if (this.executedTransactions.has(messageHash)) return true;
2492
+ if (!this.stateStore) return false;
2493
+ const persisted = await this.stateStore.wasExecuted(messageHash);
2494
+ if (persisted) {
2495
+ // Re-populate in-memory map for future synchronous checks
2496
+ this.executedTransactions.set(messageHash, Date.now());
2497
+ }
2498
+ return persisted;
2499
+ }
2500
+ }
2501
+