@optimystic/db-p2p 1.0.0-beta.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/dist/src/cluster/block-transfer-service.d.ts +73 -1
  2. package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
  3. package/dist/src/cluster/block-transfer-service.js +70 -3
  4. package/dist/src/cluster/block-transfer-service.js.map +1 -1
  5. package/dist/src/cluster/block-transfer.d.ts +16 -1
  6. package/dist/src/cluster/block-transfer.d.ts.map +1 -1
  7. package/dist/src/cluster/block-transfer.js +73 -72
  8. package/dist/src/cluster/block-transfer.js.map +1 -1
  9. package/dist/src/cluster/client.d.ts +4 -6
  10. package/dist/src/cluster/client.d.ts.map +1 -1
  11. package/dist/src/cluster/client.js +8 -13
  12. package/dist/src/cluster/client.js.map +1 -1
  13. package/dist/src/cluster/cluster-policy.d.ts +2 -1
  14. package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
  15. package/dist/src/cluster/cluster-policy.js.map +1 -1
  16. package/dist/src/cluster/cluster-repo.d.ts +172 -17
  17. package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
  18. package/dist/src/cluster/cluster-repo.js +553 -149
  19. package/dist/src/cluster/cluster-repo.js.map +1 -1
  20. package/dist/src/cluster/commit-proof.js +2 -2
  21. package/dist/src/cluster/race-resolution.d.ts +51 -7
  22. package/dist/src/cluster/race-resolution.d.ts.map +1 -1
  23. package/dist/src/cluster/race-resolution.js +71 -7
  24. package/dist/src/cluster/race-resolution.js.map +1 -1
  25. package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
  26. package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
  27. package/dist/src/cluster/rebalance-monitor.js +120 -26
  28. package/dist/src/cluster/rebalance-monitor.js.map +1 -1
  29. package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
  30. package/dist/src/cluster/spread-on-churn.js +34 -47
  31. package/dist/src/cluster/spread-on-churn.js.map +1 -1
  32. package/dist/src/cohort-topic/host.d.ts.map +1 -1
  33. package/dist/src/cohort-topic/host.js +8 -6
  34. package/dist/src/cohort-topic/host.js.map +1 -1
  35. package/dist/src/cohort-topic/membership-source.js +1 -1
  36. package/dist/src/cohort-topic/membership-source.js.map +1 -1
  37. package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
  38. package/dist/src/cohort-topic/peer-codec.js +5 -2
  39. package/dist/src/cohort-topic/peer-codec.js.map +1 -1
  40. package/dist/src/cohort-topic/stream-util.d.ts +24 -3
  41. package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
  42. package/dist/src/cohort-topic/stream-util.js +36 -3
  43. package/dist/src/cohort-topic/stream-util.js.map +1 -1
  44. package/dist/src/cohort-topic/topic-router.d.ts +9 -0
  45. package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
  46. package/dist/src/cohort-topic/topic-router.js +12 -2
  47. package/dist/src/cohort-topic/topic-router.js.map +1 -1
  48. package/dist/src/dispute/client.d.ts.map +1 -1
  49. package/dist/src/dispute/client.js +15 -3
  50. package/dist/src/dispute/client.js.map +1 -1
  51. package/dist/src/index.d.ts +5 -0
  52. package/dist/src/index.d.ts.map +1 -1
  53. package/dist/src/index.js +5 -0
  54. package/dist/src/index.js.map +1 -1
  55. package/dist/src/libp2p-key-network.d.ts +78 -17
  56. package/dist/src/libp2p-key-network.d.ts.map +1 -1
  57. package/dist/src/libp2p-key-network.js +241 -147
  58. package/dist/src/libp2p-key-network.js.map +1 -1
  59. package/dist/src/libp2p-node-base.d.ts +35 -1
  60. package/dist/src/libp2p-node-base.d.ts.map +1 -1
  61. package/dist/src/libp2p-node-base.js +151 -26
  62. package/dist/src/libp2p-node-base.js.map +1 -1
  63. package/dist/src/libp2p-node-rn.d.ts +2 -2
  64. package/dist/src/libp2p-node-rn.d.ts.map +1 -1
  65. package/dist/src/libp2p-node-rn.js.map +1 -1
  66. package/dist/src/libp2p-node.d.ts +2 -2
  67. package/dist/src/libp2p-node.d.ts.map +1 -1
  68. package/dist/src/libp2p-node.js.map +1 -1
  69. package/dist/src/logger.d.ts.map +1 -1
  70. package/dist/src/logger.js +19 -7
  71. package/dist/src/logger.js.map +1 -1
  72. package/dist/src/matchmaking/query-transport.d.ts +3 -4
  73. package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
  74. package/dist/src/matchmaking/query-transport.js +9 -9
  75. package/dist/src/matchmaking/query-transport.js.map +1 -1
  76. package/dist/src/network/network-manager-service.d.ts +10 -5
  77. package/dist/src/network/network-manager-service.d.ts.map +1 -1
  78. package/dist/src/network/network-manager-service.js +11 -2
  79. package/dist/src/network/network-manager-service.js.map +1 -1
  80. package/dist/src/network/relay-reservation.d.ts +170 -0
  81. package/dist/src/network/relay-reservation.d.ts.map +1 -0
  82. package/dist/src/network/relay-reservation.js +527 -0
  83. package/dist/src/network/relay-reservation.js.map +1 -0
  84. package/dist/src/pend-validation.d.ts +3 -2
  85. package/dist/src/pend-validation.d.ts.map +1 -1
  86. package/dist/src/pend-validation.js +3 -2
  87. package/dist/src/pend-validation.js.map +1 -1
  88. package/dist/src/protocol-client.d.ts +8 -1
  89. package/dist/src/protocol-client.d.ts.map +1 -1
  90. package/dist/src/protocol-client.js +11 -0
  91. package/dist/src/protocol-client.js.map +1 -1
  92. package/dist/src/reactivity/forwarder-host.d.ts +2 -1
  93. package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
  94. package/dist/src/reactivity/forwarder-host.js +2 -1
  95. package/dist/src/reactivity/forwarder-host.js.map +1 -1
  96. package/dist/src/reactivity/origination-manager.d.ts +3 -3
  97. package/dist/src/reactivity/origination-manager.js +3 -3
  98. package/dist/src/reactivity/recover-transport.d.ts +22 -14
  99. package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
  100. package/dist/src/reactivity/recover-transport.js +32 -19
  101. package/dist/src/reactivity/recover-transport.js.map +1 -1
  102. package/dist/src/reactivity/subscription-manager.d.ts +2 -2
  103. package/dist/src/reactivity/topic-bytes.d.ts +6 -7
  104. package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
  105. package/dist/src/reactivity/topic-bytes.js +8 -9
  106. package/dist/src/reactivity/topic-bytes.js.map +1 -1
  107. package/dist/src/repo/client.d.ts +5 -2
  108. package/dist/src/repo/client.d.ts.map +1 -1
  109. package/dist/src/repo/client.js +49 -40
  110. package/dist/src/repo/client.js.map +1 -1
  111. package/dist/src/repo/cluster-coordinator.d.ts +119 -5
  112. package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
  113. package/dist/src/repo/cluster-coordinator.js +181 -29
  114. package/dist/src/repo/cluster-coordinator.js.map +1 -1
  115. package/dist/src/repo/coordinator-repo.d.ts +259 -76
  116. package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
  117. package/dist/src/repo/coordinator-repo.js +716 -328
  118. package/dist/src/repo/coordinator-repo.js.map +1 -1
  119. package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
  120. package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
  121. package/dist/src/repo/i-under-replication-ledger.js +2 -0
  122. package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
  123. package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
  124. package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
  125. package/dist/src/repo/kv-under-replication-ledger.js +287 -0
  126. package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
  127. package/dist/src/repo/responsibility.d.ts +37 -0
  128. package/dist/src/repo/responsibility.d.ts.map +1 -0
  129. package/dist/src/repo/responsibility.js +37 -0
  130. package/dist/src/repo/responsibility.js.map +1 -0
  131. package/dist/src/repo/service.d.ts +44 -15
  132. package/dist/src/repo/service.d.ts.map +1 -1
  133. package/dist/src/repo/service.js +64 -27
  134. package/dist/src/repo/service.js.map +1 -1
  135. package/dist/src/repo/stuck-reservation.d.ts +127 -0
  136. package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
  137. package/dist/src/repo/stuck-reservation.js +179 -0
  138. package/dist/src/repo/stuck-reservation.js.map +1 -0
  139. package/dist/src/repo/under-replication-drain.d.ts +208 -0
  140. package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
  141. package/dist/src/repo/under-replication-drain.js +461 -0
  142. package/dist/src/repo/under-replication-drain.js.map +1 -0
  143. package/dist/src/rn.d.ts +5 -0
  144. package/dist/src/rn.d.ts.map +1 -1
  145. package/dist/src/rn.js +5 -0
  146. package/dist/src/rn.js.map +1 -1
  147. package/dist/src/storage/block-latch.d.ts +3 -2
  148. package/dist/src/storage/block-latch.d.ts.map +1 -1
  149. package/dist/src/storage/block-latch.js +7 -4
  150. package/dist/src/storage/block-latch.js.map +1 -1
  151. package/dist/src/storage/block-storage.d.ts +45 -3
  152. package/dist/src/storage/block-storage.d.ts.map +1 -1
  153. package/dist/src/storage/block-storage.js +156 -9
  154. package/dist/src/storage/block-storage.js.map +1 -1
  155. package/dist/src/storage/i-block-storage.d.ts +57 -5
  156. package/dist/src/storage/i-block-storage.d.ts.map +1 -1
  157. package/dist/src/storage/i-block-storage.js.map +1 -1
  158. package/dist/src/storage/pending-claim.d.ts +167 -0
  159. package/dist/src/storage/pending-claim.d.ts.map +1 -0
  160. package/dist/src/storage/pending-claim.js +156 -0
  161. package/dist/src/storage/pending-claim.js.map +1 -0
  162. package/dist/src/storage/raw-store-codec.d.ts +13 -0
  163. package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
  164. package/dist/src/storage/raw-store-codec.js +24 -3
  165. package/dist/src/storage/raw-store-codec.js.map +1 -1
  166. package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
  167. package/dist/src/storage/restoration-coordinator.js +2 -1
  168. package/dist/src/storage/restoration-coordinator.js.map +1 -1
  169. package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
  170. package/dist/src/storage/ring-shift-coordinator.js +3 -3
  171. package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
  172. package/dist/src/storage/storage-repo.d.ts +137 -5
  173. package/dist/src/storage/storage-repo.d.ts.map +1 -1
  174. package/dist/src/storage/storage-repo.js +273 -68
  175. package/dist/src/storage/storage-repo.js.map +1 -1
  176. package/dist/src/storage/struct.d.ts +58 -0
  177. package/dist/src/storage/struct.d.ts.map +1 -1
  178. package/dist/src/testing/mesh-harness.d.ts +92 -11
  179. package/dist/src/testing/mesh-harness.d.ts.map +1 -1
  180. package/dist/src/testing/mesh-harness.js +246 -103
  181. package/dist/src/testing/mesh-harness.js.map +1 -1
  182. package/package.json +2 -2
  183. package/readme.md +27 -3
  184. package/src/cluster/block-transfer-service.ts +132 -4
  185. package/src/cluster/block-transfer.ts +79 -84
  186. package/src/cluster/client.ts +115 -119
  187. package/src/cluster/cluster-policy.ts +2 -1
  188. package/src/cluster/cluster-repo.ts +3115 -2671
  189. package/src/cluster/commit-proof.ts +2 -2
  190. package/src/cluster/race-resolution.ts +76 -7
  191. package/src/cluster/rebalance-monitor.ts +677 -539
  192. package/src/cluster/spread-on-churn.ts +34 -54
  193. package/src/cohort-topic/host.ts +8 -6
  194. package/src/cohort-topic/membership-source.ts +1 -1
  195. package/src/cohort-topic/peer-codec.ts +34 -31
  196. package/src/cohort-topic/stream-util.ts +39 -4
  197. package/src/cohort-topic/topic-router.ts +12 -2
  198. package/src/dispute/client.ts +17 -7
  199. package/src/index.ts +64 -59
  200. package/src/libp2p-key-network.ts +1377 -1253
  201. package/src/libp2p-node-base.ts +186 -27
  202. package/src/libp2p-node-rn.ts +31 -30
  203. package/src/libp2p-node.ts +37 -36
  204. package/src/logger.ts +20 -7
  205. package/src/matchmaking/query-transport.ts +9 -9
  206. package/src/network/network-manager-service.ts +16 -5
  207. package/src/network/relay-reservation.ts +636 -0
  208. package/src/pend-validation.ts +3 -2
  209. package/src/protocol-client.ts +13 -1
  210. package/src/reactivity/forwarder-host.ts +2 -1
  211. package/src/reactivity/origination-manager.ts +3 -3
  212. package/src/reactivity/recover-transport.ts +39 -22
  213. package/src/reactivity/subscription-manager.ts +2 -2
  214. package/src/reactivity/topic-bytes.ts +35 -37
  215. package/src/repo/client.ts +172 -173
  216. package/src/repo/cluster-coordinator.ts +1296 -1113
  217. package/src/repo/coordinator-repo.ts +3087 -2687
  218. package/src/repo/i-under-replication-ledger.ts +89 -0
  219. package/src/repo/kv-under-replication-ledger.ts +302 -0
  220. package/src/repo/responsibility.ts +48 -0
  221. package/src/repo/service.ts +76 -39
  222. package/src/repo/stuck-reservation.ts +224 -0
  223. package/src/repo/under-replication-drain.ts +582 -0
  224. package/src/rn.ts +74 -69
  225. package/src/storage/block-latch.ts +7 -4
  226. package/src/storage/block-storage.ts +161 -10
  227. package/src/storage/i-block-storage.ts +60 -5
  228. package/src/storage/pending-claim.ts +200 -0
  229. package/src/storage/raw-store-codec.ts +26 -3
  230. package/src/storage/restoration-coordinator.ts +212 -211
  231. package/src/storage/ring-shift-coordinator.ts +271 -271
  232. package/src/storage/storage-repo.ts +311 -73
  233. package/src/storage/struct.ts +58 -0
  234. package/src/testing/mesh-harness.ts +329 -115
@@ -1,1113 +1,1296 @@
1
- import { peerIdFromString } from "@libp2p/peer-id";
2
- import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult, StaleFailure } from "@optimystic/db-core";
3
- import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, isConflictFailure, membershipDigest } from "@optimystic/db-core";
4
- import { Pending } from "@optimystic/db-core";
5
- import type { PeerId } from "@libp2p/interface";
6
- import { createLogger, verbose } from '../logger.js'
7
- import type { ClusterLogPeerOutcome } from './types.js'
8
- import type { FretService } from "p2p-fret";
9
- import type { IPeerReputation } from "../reputation/types.js";
10
- import { PenaltyReason } from "../reputation/types.js";
11
- import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
12
-
13
- const log = createLogger('cluster')
14
-
15
- /**
16
- * Pick each peer's OWN {@link ClusterRecord.applyOutcomes} entry out of the record that peer answered
17
- * with, and key it under the peer we actually asked.
18
- *
19
- * Taking only `response.applyOutcomes[peerId]` — rather than spreading the whole map — is what keeps
20
- * one member from reporting outcomes on other members' behalf: a peer that echoes back a record full
21
- * of entries contributes exactly one, its own. The field is unsigned advisory data (see its doc
22
- * comment for why that is safe), so this is a shaping rule, not a security boundary.
23
- *
24
- * Returns `undefined` when no peer reported anything, so the common case adds no empty object to the
25
- * record.
26
- */
27
- function collectApplyOutcomes(
28
- responses: ReadonlyArray<{ peerId: string; response?: ClusterRecord | null }>
29
- ): ClusterRecord['applyOutcomes'] {
30
- let collected: NonNullable<ClusterRecord['applyOutcomes']> | undefined;
31
- for (const { peerId, response } of responses) {
32
- const own = response?.applyOutcomes?.[peerId];
33
- if (own === undefined) continue;
34
- collected ??= {};
35
- collected[peerId] = own;
36
- }
37
- return collected;
38
- }
39
-
40
- /** Fold collected outcomes into a record in place, later report winning per peer. No-op for `undefined`. */
41
- function mergeApplyOutcomes(record: ClusterRecord, collected: ClusterRecord['applyOutcomes']): void {
42
- if (collected === undefined) return;
43
- record.applyOutcomes = { ...record.applyOutcomes, ...collected };
44
- }
45
-
46
- /**
47
- * Consensus refused a transaction: enough members voted reject that super-majority became
48
- * impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
49
- * "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
50
- * rejection reasons those are free-form text that is part of each member's signed vote payload
51
- * (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
52
- * `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
53
- * loss (confirmed against local storage) or a genuine validation fault.
54
- */
55
- export class ValidatorRejectionError extends Error {
56
- constructor(
57
- message: string,
58
- /** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
59
- readonly rejectReasons: Record<string, string>
60
- ) {
61
- super(message);
62
- this.name = 'ValidatorRejectionError';
63
- }
64
- }
65
-
66
- /**
67
- * The transaction lost a conflict race: one or more members answered with a signed `conflict`
68
- * vote (they hold a rival transaction that won the deterministic race on the same blocks) and
69
- * approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} nobody
70
- * judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
71
- * `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
72
- * `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
73
- * as a thrown error only from other paths. The commit conversion matters as much as the pend one:
74
- * at the moment this is thrown zero members approved and the members hold the winner nothing of
75
- * the loser landed yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
76
- * (it treats throws as transport faults), and that re-driven commit races into the window after
77
- * members apply the winner and clear its reservation, where it can assemble a consensus no member
78
- * will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
79
- * writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
80
- * and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
81
- * parsed out of prose.
82
- */
83
- export class ConflictRaceLostError extends Error {
84
- constructor(
85
- message: string,
86
- /** peerId → messageHash of the rival transaction that member holds as the race winner. */
87
- readonly conflicts: Record<string, string>
88
- ) {
89
- super(message);
90
- this.name = 'ConflictRaceLostError';
91
- }
92
- }
93
-
94
- /** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
95
- export type TimerCancel = () => void;
96
-
97
- /**
98
- * Production timer binding: a one-shot `setTimeout` whose handle is **unref'd** so a pending
99
- * commit-retry (or the deferred transaction cleanup) never keeps an otherwise-idle process alive.
100
- * The returned handle clears the timeout (idempotent). Mirrors the reactivity rotation
101
- * re-registration scheduler's `defaultSetTimer` (see reactivity/rotation-rereg-scheduler.ts).
102
- */
103
- function defaultSetTimer(fn: () => void, delayMs: number): TimerCancel {
104
- const handle = setTimeout(fn, delayMs);
105
- // An idle retry/cleanup timer must not pin a process (mirror rotation re-registration + push-state gossip).
106
- (handle as { unref?: () => void }).unref?.();
107
- return (): void => clearTimeout(handle);
108
- }
109
-
110
- /**
111
- * Optional injection seam for deterministic time. Production leaves both undefined and gets
112
- * `Date.now` + an unref'd `setTimeout`; tests inject a fake clock + timer queue so scheduled
113
- * commit-retries fire in virtual (not wall-clock) time.
114
- */
115
- export interface ClusterCoordinatorClock {
116
- /** Clock (Unix ms). Defaults to `Date.now`. */
117
- now?: () => number;
118
- /** Schedule a one-shot timer, returning a cancel handle. Defaults to an unref'd `setTimeout`. */
119
- setTimer?: (fn: () => void, delayMs: number) => TimerCancel;
120
- }
121
-
122
- /**
123
- * Manages the state of cluster transactions for a specific block ID
124
- */
125
- interface CommitRetryState {
126
- pendingPeers: Set<string>;
127
- attempt: number;
128
- intervalMs: number;
129
- cancel?: TimerCancel;
130
- }
131
-
132
- interface ClusterTransactionState {
133
- messageHash: string;
134
- record: ClusterRecord;
135
- pending: Pending<ClusterRecord>;
136
- lastUpdate: number;
137
- promiseTimeout?: NodeJS.Timeout;
138
- resolutionTimeout?: NodeJS.Timeout;
139
- retry?: CommitRetryState;
140
- }
141
-
142
- /** Manages distributed transactions across clusters */
143
- export class ClusterCoordinator {
144
- private transactions: Map<string, ClusterTransactionState> = new Map();
145
- private readonly retryInitialIntervalMs: number;
146
- private readonly retryBackoffFactor: number;
147
- private readonly retryMaxIntervalMs: number;
148
- private readonly retryMaxAttempts: number;
149
- private readonly commitBroadcastImmediateRetries: number;
150
- private readonly promiseImmediateRetries: number;
151
- /** Injected clock/timer seam; production defaults to `Date.now` + unref'd `setTimeout`. */
152
- private readonly now: () => number;
153
- private readonly setTimer: (fn: () => void, delayMs: number) => TimerCancel;
154
-
155
- constructor(
156
- private readonly keyNetwork: IKeyNetwork,
157
- /** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
158
- private readonly createClusterClient: (peerId: PeerId) => ICluster,
159
- private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
160
- private readonly localCluster?: {
161
- update: (record: ClusterRecord) => Promise<ClusterRecord>;
162
- peerId: PeerId;
163
- wasTransactionExecuted?: (messageHash: string) => boolean;
164
- /** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
165
- getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
166
- /** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
167
- getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
168
- },
169
- private readonly fretService?: FretService,
170
- private readonly reputation?: IPeerReputation,
171
- private readonly stateStore?: ITransactionStateStore,
172
- clock?: ClusterCoordinatorClock
173
- ) {
174
- this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
175
- this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
176
- this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
177
- this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
178
- this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
179
- this.promiseImmediateRetries = cfg.promiseImmediateRetries ?? 1;
180
- this.now = clock?.now ?? ((): number => Date.now());
181
- this.setTimer = clock?.setTimer ?? defaultSetTimer;
182
- }
183
-
184
- /**
185
- * Invoke one cluster member's `update`, retrying transient REMOTE failures up to
186
- * `immediateRetries` times before surfacing the error. The local cluster is invoked
187
- * exactly once — a local throw is a real fault (validation / merge / consensus), not a
188
- * transient transport blip. A remote call rides a libp2p stream that a circuit-relay
189
- * ("limited") connection can reset once a per-circuit cap or reservation lapses, which
190
- * surfaces as a StreamResetError; an immediate retry on the (usually still-warm)
191
- * connection recovers most of those without escalating the peer to a failure. Shared by
192
- * the promise-collection, commit-collection, and commit-broadcast phases so all three
193
- * react to a relayed reset the same way.
194
- */
195
- private async updateMember(peerIdStr: string, record: ClusterRecord, immediateRetries: number, phase: string): Promise<ClusterRecord> {
196
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
197
- if (isLocal) {
198
- return await this.localCluster!.update(record);
199
- }
200
- const maxAttempts = 1 + Math.max(0, immediateRetries);
201
- let lastError: unknown;
202
- for (let attempt = 1; attempt <= maxAttempts; attempt++) {
203
- try {
204
- return await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
205
- } catch (err) {
206
- lastError = err;
207
- if (attempt < maxAttempts) {
208
- log('cluster-tx:member-update-retry', {
209
- messageHash: record.messageHash,
210
- peerId: peerIdStr,
211
- phase,
212
- attempt,
213
- error: err instanceof Error ? err.message : String(err)
214
- });
215
- }
216
- }
217
- }
218
- throw lastError;
219
- }
220
-
221
- /**
222
- * Creates a base58btc string hash uniquely identifying a transaction. For a v2 record the caller
223
- * threads in the {@link membershipDigest} of the peer set so the responsible membership is bound into
224
- * the identity (two different peer sets two different hashes). Omitting `membershipDigestValue`
225
- * reproduces the legacy v1 hash byte-for-byte.
226
- *
227
- * NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
228
- * which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
229
- * `pend.priority` (single-collection) is automatically covered here and by the derived
230
- * promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
231
- * cannot strip or inflate it without invalidating the message hash the members verify. No separate
232
- * priority-hashing step is needed.
233
- */
234
- private async createMessageHash(message: RepoMessage, membershipDigestValue?: string): Promise<string> {
235
- return computeClusterMessageHash(message, membershipDigestValue);
236
- }
237
-
238
- /**
239
- * Gets all peers in the cluster for a specific block ID
240
- */
241
- private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
242
- const blockIdBytes = new TextEncoder().encode(blockId);
243
- try {
244
- const peers = await this.keyNetwork.findCluster(blockIdBytes);
245
- const peerIds = Object.keys(peers ?? {});
246
- log('cluster-tx:cluster-members', { blockId, peerIds });
247
- return peers;
248
- } catch (e) {
249
- log('WARN findCluster failed for %s: %o', blockId, e)
250
- return {} as ClusterPeers
251
- }
252
- }
253
-
254
- private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
255
- const peerCount = Object.keys(peers ?? {}).length;
256
- const record: ClusterRecord = {
257
- messageHash,
258
- peers,
259
- // v2: bind the responsible membership into the signed identity. messageHash was computed over
260
- // this same digest, so a different peer set would have produced a different messageHash.
261
- membershipVersion: CURRENT_MEMBERSHIP_VERSION,
262
- membershipDigest: membershipDigestValue,
263
- message,
264
- promises: {},
265
- commits: {},
266
- suggestedClusterSize: peerCount || undefined,
267
- minRequiredSize: this.cfg.allowClusterDownsize ? undefined : this.cfg.clusterSize
268
- };
269
-
270
- // Add network size hint if available
271
- if (this.fretService) {
272
- try {
273
- const estimate = this.fretService.getNetworkSizeEstimate();
274
- if (estimate.size_estimate > 0) {
275
- record.networkSizeHint = estimate.size_estimate;
276
- record.networkSizeConfidence = estimate.confidence;
277
- }
278
- } catch (err) {
279
- // Ignore errors getting size estimate
280
- }
281
- }
282
-
283
- return record;
284
- }
285
-
286
- /**
287
- * Initiates a 2-phase transaction for a specific block ID.
288
- * Returns the cluster record and whether the local cluster already executed the operations.
289
- */
290
- async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
291
- record: ClusterRecord;
292
- localExecuted: boolean;
293
- /**
294
- * Local storage's verdict for a pend operation this node's own cluster member applied during
295
- * consensus, when the member retained one. Meaningful only when `localExecuted` is true;
296
- * absent for non-pend messages, for a member that predates the retention, or after the
297
- * retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
298
- */
299
- localPendResult?: PendResult;
300
- /**
301
- * Local storage's verdict for a commit operation this node's own cluster member applied
302
- * during consensus, when the member retained one. Same availability contract as
303
- * `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
304
- * win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
305
- * success no member durably stored.
306
- */
307
- localCommitResult?: CommitResult;
308
- /**
309
- * Conflict-shaped pend refusals reported by OTHER cohort members on their consensus responses
310
- * (`ClusterRecord.applyOutcomes`), keyed by peer id. This is the arm `localPendResult` cannot
311
- * cover: the refusing member is frequently not the coordinating node, and its verdict used to
312
- * stay on that member while the writer was told the pend won. Unsigned advisory data — an
313
- * entry means "retry", never "this write was invalid". Absent when nobody reported one.
314
- *
315
- * Residual: a member that reaches consensus only via the scheduled commit-retry timer applies
316
- * after this method has already resolved, so its refusal arrives too late to appear here. The
317
- * member-side commit-promise guard (`validateCommitAgainstRefusedPend`) is the backstop for
318
- * that path.
319
- */
320
- cohortPendRefusals?: { [peerId: string]: StaleFailure };
321
- }> {
322
- // The coordinating block id is derived HERE, from the key this method is already handed, rather
323
- // than being set by each caller's message builder: a member's membership admission gate derives
324
- // its own cohort view from this field, and a builder that forgets it silently downgrades the gate
325
- // to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
326
- // admitted at pend, refused at commit). Doing it at the single choke point means a future message
327
- // builder cannot reintroduce the gap.
328
- //
329
- // Two constraints this shape exists to satisfy:
330
- // - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
331
- // N concurrent calls, one per block. In-place mutation would leak one block's id into another
332
- // block's transaction.
333
- // - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
334
- // not just its first block, so this must not overwrite it. Tested on `length`, not on the
335
- // field: an empty list carries no id for a member to derive from, so preserving one would be
336
- // the same silent downgrade to the fallback floor this choke point exists to prevent.
337
- const coordinated: RepoMessage = message.coordinatingBlockIds?.length
338
- ? message
339
- : { ...message, coordinatingBlockIds: [blockId] };
340
-
341
- // Get the cluster peers for this block
342
- const peers = await this.getClusterForBlock(blockId);
343
-
344
- // Bind the responsible membership into the transaction identity (v2): the digest is folded into
345
- // the messageHash below, so two different peer sets produce two different messageHashes rather
346
- // than one hash with a silent internal disagreement about who is responsible.
347
- const membershipDigestValue = await membershipDigest(peers);
348
-
349
- // Create a unique hash for this transaction (over message + membership digest). Hashing the
350
- // coordinating-block-bearing copy is what makes the field tamper-evident in transit — and it also
351
- // makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
352
- // identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
353
- const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
354
-
355
- // Create a cluster record for this transaction
356
- const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
357
- log('cluster-tx:start', {
358
- messageHash,
359
- blockId,
360
- peerCount: Object.keys(peers ?? {}).length,
361
- allowDownsize: this.cfg.allowClusterDownsize,
362
- configuredSize: this.cfg.clusterSize,
363
- suggestedSize: record.suggestedClusterSize,
364
- minRequiredSize: record.minRequiredSize
365
- });
366
-
367
- // Create a new pending transaction
368
- const transactionPromise = this.executeTransaction(peers, record);
369
- const pending = new Pending(transactionPromise);
370
-
371
- // Store the transaction state
372
- const state: ClusterTransactionState = {
373
- messageHash,
374
- record,
375
- pending,
376
- lastUpdate: this.now()
377
- };
378
- this.transactions.set(messageHash, state);
379
- this.persistCoordinatorState(messageHash, record, 'promising');
380
- log('cluster-tx:transaction-store', {
381
- messageHash,
382
- transactionKeys: Array.from(this.transactions.keys())
383
- });
384
-
385
- // Wait for the transaction to complete
386
- try {
387
- const result = await pending.result();
388
- // Check if the local cluster already executed the operations during consensus
389
- const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
390
- const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
391
- const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
392
- // Self is excluded: this node's own member verdict is already carried, more directly and
393
- // without the wire round trip, by `localPendResult` — and leaving it in both places would
394
- // make the coordinator's "prefer local" rule ambiguous.
395
- // Re-checked here rather than trusted: members are supposed to report only conflict-shaped
396
- // refusals, but the field arrives off the wire, so anything else (a success, a bare-reason
397
- // fault, a malformed entry) is dropped instead of being handed to a caller that would read
398
- // it as a retryable conflict.
399
- const selfId = this.localCluster?.peerId.toString();
400
- const cohortPendRefusals: { [peerId: string]: StaleFailure } = {};
401
- for (const [peerId, outcome] of Object.entries(result.applyOutcomes ?? {})) {
402
- const pend = outcome?.pend;
403
- if (peerId === selfId || pend === undefined || pend.success || !isConflictFailure(pend)) continue;
404
- cohortPendRefusals[peerId] = pend;
405
- }
406
- return {
407
- record: result,
408
- localExecuted,
409
- ...(localPendResult === undefined ? {} : { localPendResult }),
410
- ...(localCommitResult === undefined ? {} : { localCommitResult }),
411
- ...(Object.keys(cohortPendRefusals).length === 0 ? {} : { cohortPendRefusals })
412
- };
413
- } finally {
414
- const stored = this.transactions.get(messageHash);
415
- const retrySnapshot = stored?.retry ? {
416
- attempt: stored.retry.attempt,
417
- pending: Array.from(stored.retry.pendingPeers ?? [])
418
- } : undefined;
419
- log('cluster-tx:complete', {
420
- messageHash,
421
- finalPromises: stored ? Object.keys(stored.record.promises ?? {}) : undefined,
422
- finalCommits: stored ? Object.keys(stored.record.commits ?? {}) : undefined,
423
- retry: retrySnapshot
424
- });
425
- // Don't remove transaction immediately if retries are scheduled
426
- // Let the retry completion or abort handle cleanup
427
- if (!stored?.retry) {
428
- // Wait a bit before cleanup to allow any in-flight responses to arrive
429
- this.setTimer(() => {
430
- this.transactions.delete(messageHash);
431
- this.deleteCoordinatorState(messageHash);
432
- log('cluster-tx:transaction-remove', {
433
- messageHash,
434
- remaining: Array.from(this.transactions.keys())
435
- });
436
- }, 100);
437
- }
438
- }
439
- }
440
-
441
- /**
442
- * Executes the full transaction process
443
- */
444
- private async executeTransaction(peers: ClusterPeers, record: ClusterRecord): Promise<ClusterRecord> {
445
- const peerCount = Object.keys(peers).length;
446
-
447
- // Validate against minimum cluster size
448
- if (peerCount < this.cfg.minAbsoluteClusterSize) {
449
- const validated = await this.validateSmallCluster(peerCount, peers);
450
- if (!validated) {
451
- log('cluster-tx:reject-too-small', {
452
- peerCount,
453
- minRequired: this.cfg.minAbsoluteClusterSize
454
- });
455
- throw new Error(`Cluster size ${peerCount} below minimum ${this.cfg.minAbsoluteClusterSize} and not validated`);
456
- }
457
- log('cluster-tx:small-cluster-validated', { peerCount });
458
- }
459
-
460
- // Check configured cluster size
461
- if (!this.cfg.allowClusterDownsize && peerCount < this.cfg.clusterSize) {
462
- log('cluster-tx:reject-downsize', { peerCount, required: this.cfg.clusterSize });
463
- throw new Error(`Cluster size ${peerCount} below configured minimum ${this.cfg.clusterSize}`);
464
- }
465
-
466
- // Collect promises with super-majority requirement
467
- const promised = await this.collectPromises(peers, record);
468
- const superMajority = Math.ceil(peerCount * this.cfg.superMajorityThreshold);
469
-
470
- // Count approvals, rejections and conflict votes separately. A `conflict` vote is a member
471
- // saying "not now — I hold the race winner": it must count toward NEITHER approvals NOR
472
- // rejections, or a lost race would masquerade as a validator rejection (permanent) or as
473
- // silence (indistinguishable from an unreachable cohort) — both wrong.
474
- const promises = promised.record.promises;
475
- const approvalCount = Object.values(promises).filter(sig => sig.type === 'approve').length;
476
- const rejectionCount = Object.values(promises).filter(sig => sig.type === 'reject').length;
477
- const conflictCount = Object.values(promises).filter(sig => sig.type === 'conflict').length;
478
-
479
- // Check if rejections make super-majority impossible
480
- // If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
481
- const maxAllowedRejections = peerCount - superMajority;
482
- if (rejectionCount > maxAllowedRejections) {
483
- const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
484
- .flatMap(([peerId, sig]) => sig.type === 'reject' ? [[peerId, sig.rejectReason ?? 'unknown'] as const] : []));
485
- const rejectReasons = Object.entries(rejectReasonsByPeer)
486
- .map(([peerId, reason]) => `${peerId}: ${reason}`)
487
- .join('; ');
488
- log('cluster-tx:rejected-by-validators', {
489
- messageHash: record.messageHash,
490
- peerCount,
491
- rejections: rejectionCount,
492
- maxAllowed: maxAllowedRejections,
493
- reasons: rejectReasons
494
- });
495
- this.updateTransactionRecord(promised.record, 'rejected-by-validators');
496
- // Abandoning here without telling anyone leaves every member that voted holding this
497
- // transaction in its own reservation table, blocking its blocks until that member's
498
- // staleness sweep fires and each retry we throw back to the caller plants a fresh
499
- // reservation, so the block never frees. The merged record carries enough signed
500
- // rejections to *prove* the transaction is dead, so replaying it to the cohort makes
501
- // every member recompute `Rejected` and clear immediately. Proof-carrying, so a member
502
- // need not trust us: it verifies the signatures it is shown.
503
- this.broadcastAbandonment(promised.record, 'rejected-by-validators');
504
- throw new ValidatorRejectionError(
505
- `Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
506
- rejectReasonsByPeer);
507
- }
508
-
509
- // A conflict-answered shortfall is a LOST RACE, not a validator verdict and not silence.
510
- // Checked after the rejection threshold (a genuine validator rejection still wins) and
511
- // before the generic shortfall (which must stay reserved for the genuinely-silent cohort).
512
- if (conflictCount > 0 && approvalCount < superMajority) {
513
- const conflicts = Object.fromEntries(Object.entries(promises)
514
- .flatMap(([peerId, sig]) => sig.type === 'conflict' ? [[peerId, sig.conflictWith] as const] : []));
515
- log('cluster-tx:conflict-race-lost', {
516
- messageHash: record.messageHash,
517
- peerCount,
518
- approvals: approvalCount,
519
- rejections: rejectionCount,
520
- conflicts,
521
- superMajority
522
- });
523
- this.updateTransactionRecord(promised.record, 'conflict-race-lost');
524
- // Broadcast only when the merged record itself PROVES the transaction can no longer reach
525
- // super-majority (members re-derive ConflictSuperseded/Rejected from the signed votes and
526
- // clear their reservations immediately). Below that bar the record proves nothing and a
527
- // broadcast would be the unauthenticated "forget this" the shortfall NOTE below refuses.
528
- if (rejectionCount + conflictCount > maxAllowedRejections) {
529
- this.broadcastAbandonment(promised.record, 'conflict-race-lost');
530
- }
531
- throw new ConflictRaceLostError(
532
- `Conflict race lost: ${conflictCount}/${peerCount} member(s) hold a conflicting winner (${approvalCount}/${superMajority} approvals)`,
533
- conflicts);
534
- }
535
-
536
- if (peerCount > 1 && approvalCount < superMajority) {
537
- log('cluster-tx:supermajority-failed', {
538
- messageHash: record.messageHash,
539
- peerCount,
540
- approvals: approvalCount,
541
- rejections: rejectionCount,
542
- superMajority,
543
- threshold: this.cfg.superMajorityThreshold
544
- });
545
- this.updateTransactionRecord(promised.record, 'supermajority-failed');
546
- // NOTE: deliberately NOT broadcast, unlike the rejected-by-validators branch above. With
547
- // conflict-answered shortfalls peeled off above, we get here only because peers did not
548
- // answer at all, so the record carries no signed evidence that the transaction is dead — a
549
- // broadcast would be an unauthenticated "forget this" that any caller could use to clear a
550
- // live transaction out of a member's reservation table. Members that DID vote are freed by
551
- // their own staleness sweep instead.
552
- // NOTE: the message below is load-bearing wire text — the consuming repo
553
- // (sereus cadre-core control-write-retry) matches it verbatim to retry a genuinely-silent
554
- // cohort. Keep it byte-identical, and never fold conflict votes into its rejection count.
555
- throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
556
- }
557
-
558
- // Mark as disputed when minority rejections exist but super-majority approves
559
- if (rejectionCount > 0 && approvalCount >= superMajority) {
560
- const rejectingPeers: string[] = [];
561
- const rejectReasons: { [peerId: string]: string } = {};
562
- for (const [peerId, sig] of Object.entries(promises)) {
563
- if (sig.type === 'reject') {
564
- rejectingPeers.push(peerId);
565
- rejectReasons[peerId] = sig.rejectReason ?? 'unknown';
566
- }
567
- }
568
- promised.record.disputed = true;
569
- promised.record.disputeEvidence = { rejectingPeers, rejectReasons };
570
- log('cluster-tx:disputed', {
571
- messageHash: record.messageHash,
572
- rejectingPeers,
573
- rejectReasons,
574
- approvalCount,
575
- rejectionCount,
576
- peerCount
577
- });
578
- // [dispute-subsystem-dormant] Evidence is computed and persisted but initiateDispute() is
579
- // intentionally NOT called here. Dispute origination stays dormant pending arbitrator-set
580
- // anchoring — without it a forged synthetic cohort passes resolution.
581
- // Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
582
- // Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
583
- }
584
-
585
- this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
586
- return await this.commitTransaction(promised.record);
587
- }
588
-
589
- /**
590
- * The block's cohort peer ids as currently derivable. Empty when `findCluster` fails
591
- * (getClusterForBlock swallows the throw), so a caller branching on `length <= 1` is also taking
592
- * the degraded-routing branch; `CoordinatorRepo.commit` uses the ids to log whether a solo cohort
593
- * is genuinely just self or a routing failure.
594
- */
595
- async getClusterPeerIds(blockId: BlockId): Promise<string[]> {
596
- const peers = await this.getClusterForBlock(blockId);
597
- return Object.keys(peers ?? {});
598
- }
599
-
600
- /** {@link getClusterPeerIds}, counted. Derived from it rather than re-deriving the cohort, so the
601
- * size a caller branches on and the ids it logs can never come from two different rules. */
602
- async getClusterSize(blockId: BlockId): Promise<number> {
603
- return (await this.getClusterPeerIds(blockId)).length;
604
- }
605
-
606
- /**
607
- * Validate that a small cluster size is legitimate by querying remote peers
608
- * for their network size estimates. Returns true if estimates roughly agree.
609
- */
610
- private async validateSmallCluster(localSize: number, _peers: ClusterPeers): Promise<boolean> {
611
- // If we have FRET and it shows confident estimate
612
- if (this.fretService) {
613
- try {
614
- const estimate = this.fretService.getNetworkSizeEstimate();
615
- if (estimate.confidence > 0.5) {
616
- // Check if FRET estimate roughly matches observed cluster size
617
- const orderOfMagnitude = Math.floor(Math.log10(estimate.size_estimate + 1));
618
- const localOrderOfMagnitude = Math.floor(Math.log10(localSize + 1));
619
-
620
- // If within same order of magnitude, accept it
621
- if (Math.abs(orderOfMagnitude - localOrderOfMagnitude) <= 1) {
622
- log('cluster-tx:small-cluster-validated-by-fret', {
623
- localSize,
624
- fretEstimate: estimate.size_estimate,
625
- confidence: estimate.confidence,
626
- sources: estimate.sources
627
- });
628
- return true;
629
- }
630
- }
631
- } catch (err) {
632
- // Ignore errors
633
- }
634
- }
635
-
636
- // Fallback: with no confident network-size estimate, fail CLOSED by default.
637
- // An undersized cluster with no way to justify its size is unsafe (a lone/
638
- // near-lone node could rubber-stamp its own writes), so reject unless the
639
- // operator has explicitly opted in via allowUnvalidatedSmallCluster (e.g.
640
- // single-node / local dev knowingly running below the floor).
641
- const admit = this.cfg.allowUnvalidatedSmallCluster ?? false;
642
- log('cluster-tx:small-cluster-no-confident-estimate', {
643
- localSize,
644
- reason: 'no-confident-network-size-estimate',
645
- admit
646
- });
647
- return admit;
648
- }
649
-
650
- /**
651
- * Collects promises from all peers in the cluster
652
- */
653
- private async collectPromises(peers: ClusterPeers, record: ClusterRecord): Promise<{ record: ClusterRecord }> {
654
- const peerIds = Object.keys(peers);
655
- const summary: ClusterLogPeerOutcome[] = [];
656
- if (verbose) {
657
- const peerDetail = peerIds.map(id => ({
658
- id: id.substring(0, 12),
659
- addrs: peers[id]?.multiaddrs?.length ?? 0
660
- }));
661
- log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
662
- }
663
- // For each peer, create a client and request a promise. A remote promise rides
664
- // a libp2p stream that a relayed (limited) connection can reset transiently, so
665
- // each remote request gets `promiseImmediateRetries` in-line re-attempts before
666
- // it counts as a failurewithout this a single relayed reset drops the peer and
667
- // sinks super-majority (the commit broadcast already has the same guard).
668
- const promiseRequests = peerIds.map(peerIdStr => {
669
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
670
- log('cluster-tx:promise-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
671
- return new Pending(this.updateMember(peerIdStr, record, this.promiseImmediateRetries, 'promise'));
672
- });
673
-
674
- // Wait for all promises to complete
675
- const results = await Promise.all(promiseRequests.map((p, idx) => p.result().then(res => {
676
- const peerIdStr = peerIds[idx]!;
677
- log('cluster-tx:promise-response', {
678
- messageHash: record.messageHash,
679
- peerId: peerIdStr,
680
- success: true,
681
- returnedPromises: Object.keys(res.promises ?? {}),
682
- returnedCommits: Object.keys(res.commits ?? {})
683
- });
684
- summary.push({ peerId: peerIdStr, success: true });
685
- return res;
686
- }).catch(err => {
687
- const peerIdStr = peerIds[idx]!;
688
- log('cluster-tx:promise-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
689
- summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
690
- this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `promise:${record.messageHash}`);
691
- return null;
692
- })));
693
- const successes = summary.filter(entry => entry.success).map(entry => entry.peerId);
694
- const failures = summary.filter(entry => !entry.success);
695
- log('cluster-tx:promise-summary', {
696
- messageHash: record.messageHash,
697
- successes,
698
- failures
699
- });
700
-
701
- log('cluster-tx:promise-merge-begin', {
702
- messageHash: record.messageHash,
703
- initialPromises: Object.keys(record.promises ?? {}),
704
- transactionsKeys: Array.from(this.transactions.keys()),
705
- hasTransaction: this.transactions.has(record.messageHash)
706
- });
707
-
708
- // Merge all promises into the record
709
- for (const result of results.filter(Boolean) as ClusterRecord[]) {
710
- log('cluster-tx:promise-merge-input', {
711
- messageHash: record.messageHash,
712
- resultFrom: Object.keys(result.promises ?? {}),
713
- recordBefore: Object.keys(record.promises ?? {})
714
- });
715
- const resultPromises = Object.keys(result.promises ?? {});
716
- log('cluster-tx:promise-merge-result', {
717
- messageHash: record.messageHash,
718
- peerPromises: resultPromises
719
- });
720
- if (typeof record.suggestedClusterSize === 'number' && typeof result.suggestedClusterSize === 'number') {
721
- const expected = result.suggestedClusterSize;
722
- const actual = Object.keys(peers).length;
723
- const maxDiff = Math.ceil(Math.max(1, expected * this.cfg.clusterSizeTolerance));
724
- if (Math.abs(actual - expected) > maxDiff) {
725
- log('cluster-tx:size-variance', { expected, actual, tolerance: this.cfg.clusterSizeTolerance });
726
- }
727
- }
728
- record.promises = { ...record.promises, ...result.promises };
729
- log('cluster-tx:promise-merge-after', {
730
- messageHash: record.messageHash,
731
- mergedPromises: Object.keys(record.promises ?? {})
732
- });
733
- }
734
- log('cluster-tx:promise-merge', {
735
- messageHash: record.messageHash,
736
- mergedPromises: Object.keys(record.promises ?? {})
737
- });
738
- log('cluster-tx:promise-merge-end', {
739
- messageHash: record.messageHash,
740
- finalPromises: Object.keys(record.promises ?? {}),
741
- transactionsEntry: this.transactions.get(record.messageHash)
742
- });
743
- this.updateTransactionRecord(record, 'after-promises');
744
- return { record };
745
- }
746
-
747
- /**
748
- * Commits the transaction to all peers in the cluster
749
- */
750
- private async commitTransaction(record: ClusterRecord): Promise<ClusterRecord> {
751
- // For each peer, create a client and send the commit
752
- const peerIds = Object.keys(record.peers);
753
- const summary: ClusterLogPeerOutcome[] = [];
754
- if (verbose) {
755
- const peerDetail = peerIds.map(id => ({
756
- id: id.substring(0, 12),
757
- addrs: record.peers[id]?.multiaddrs?.length ?? 0
758
- }));
759
- log('cluster-tx:commit-peers', { messageHash: record.messageHash, peers: peerDetail });
760
- }
761
- // Send the record with promises to all peers
762
- // Each peer will add its own commit signature
763
- const commitPayload = {
764
- ...record
765
- };
766
- // No per-peer immediate retry here: a commit-collection failure is recovered
767
- // downstream by broadcastMergedRecord's in-line retry and the scheduled
768
- // commit-retry timer. (The promise phase has no such backstop, which is why
769
- // collectPromises gets the immediate retry instead.)
770
- const commitRequests = peerIds.map(peerIdStr => {
771
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
772
- log('cluster-tx:commit-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
773
- const promise = isLocal
774
- ? this.localCluster!.update(commitPayload)
775
- : this.createClusterClient(peerIdFromString(peerIdStr)).update(commitPayload);
776
- return new Pending(promise);
777
- });
778
-
779
- // Wait for all commits to complete
780
- const results = await Promise.all(commitRequests.map((p, idx) => p.result().then(res => {
781
- const peerIdStr = peerIds[idx]!;
782
- log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: true });
783
- summary.push({ peerId: peerIdStr, success: true });
784
- return res;
785
- }).catch(err => {
786
- const peerIdStr = peerIds[idx]!;
787
- log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
788
- summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
789
- this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `commit:${record.messageHash}`);
790
- return null;
791
- })));
792
- const commitSuccesses = summary.filter(entry => entry.success).map(entry => entry.peerId);
793
- const commitFailures = summary.filter(entry => !entry.success);
794
- log('cluster-tx:commit-summary', {
795
- messageHash: record.messageHash,
796
- successes: commitSuccesses,
797
- failures: commitFailures
798
- });
799
- log('cluster-tx:commit-merge-begin', {
800
- messageHash: record.messageHash,
801
- initialCommits: Object.keys(record.commits ?? {}),
802
- transactionsEntry: this.transactions.get(record.messageHash)
803
- });
804
-
805
- // Members that already held super-majority promises reach consensus during THIS round rather
806
- // than during the broadcast below, so their apply verdicts arrive on these responses. Collect
807
- // both; the broadcast's copy wins on overlap, being the later of the two.
808
- mergeApplyOutcomes(record, collectApplyOutcomes(results.map((response, idx) => ({ peerId: peerIds[idx]!, response }))));
809
-
810
- // Merge all commits into the record
811
- for (const result of results.filter(Boolean) as ClusterRecord[]) {
812
- log('cluster-tx:commit-merge-input', {
813
- messageHash: record.messageHash,
814
- resultFrom: Object.keys(result.commits ?? {}),
815
- recordBefore: Object.keys(record.commits ?? {})
816
- });
817
- log('cluster-tx:commit-merge-result', {
818
- messageHash: record.messageHash,
819
- peerCommits: Object.keys(result.commits ?? {})
820
- });
821
- record.commits = { ...record.commits, ...result.commits };
822
- log('cluster-tx:commit-merge-after', {
823
- messageHash: record.messageHash,
824
- mergedCommits: Object.keys(record.commits ?? {})
825
- });
826
- }
827
- log('cluster-tx:commit-merge', {
828
- messageHash: record.messageHash,
829
- mergedCommits: Object.keys(record.commits ?? {})
830
- });
831
- log('cluster-tx:commit-merge-end', {
832
- messageHash: record.messageHash,
833
- finalCommits: Object.keys(record.commits ?? {}),
834
- transactionsEntry: this.transactions.get(record.messageHash)
835
- });
836
- this.updateTransactionRecord(record, 'after-commit');
837
-
838
- // Check for simple majority (>50%) - this proves commitment
839
- const peerCount = Object.keys(record.peers).length;
840
- const simpleMajority = Math.floor(peerCount * this.cfg.simpleMajorityThreshold) + 1;
841
- const commitCount = Object.keys(record.commits).length;
842
-
843
- if (commitCount >= simpleMajority) {
844
- log('cluster-tx:commit-majority-reached', {
845
- messageHash: record.messageHash,
846
- commitCount,
847
- simpleMajority,
848
- peerCount,
849
- threshold: this.cfg.simpleMajorityThreshold
850
- });
851
- // Broadcast the merged record (with all commit signatures) to ALL peers
852
- // so each peer can independently reach consensus and execute the operations.
853
- // Without this, only the coordinator's local cluster executes — remote peers
854
- // never see enough commits to reach consensus on their own.
855
- const { failures: broadcastFailures, applyOutcomes } = await this.broadcastMergedRecord(record, peerIds);
856
- mergeApplyOutcomes(record, applyOutcomes);
857
- if (broadcastFailures.length > 0) {
858
- this.scheduleCommitRetry(record.messageHash, record, broadcastFailures);
859
- } else {
860
- this.clearRetry(record.messageHash);
861
- }
862
- } else {
863
- const missingPeers = commitFailures.map(entry => entry.peerId);
864
- if (missingPeers.length > 0) {
865
- this.scheduleCommitRetry(record.messageHash, record, missingPeers);
866
- } else {
867
- this.clearRetry(record.messageHash);
868
- }
869
- }
870
- return record;
871
- }
872
-
873
- /**
874
- * Broadcast the merged commit record to every peer, with `commitBroadcastImmediateRetries`
875
- * in-line re-attempts per peer before giving up. The libp2p connection used during
876
- * the prior commit phase is typically still warm, so a single immediate retry recovers
877
- * most transient stream errors without falling back to the scheduled retry timer.
878
- * Local cluster is invoked exactly once — local failures are fatal, not transient.
879
- */
880
- private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[]; applyOutcomes?: ClusterRecord['applyOutcomes'] }> {
881
- const results = await Promise.all(peerIds.map(async peerIdStr => {
882
- try {
883
- const response = await this.updateMember(peerIdStr, record, this.commitBroadcastImmediateRetries, 'commit-broadcast');
884
- return { peerId: peerIdStr, success: true as const, response };
885
- } catch (err) {
886
- log('cluster-tx:consensus-broadcast-error', {
887
- messageHash: record.messageHash,
888
- peerId: peerIdStr,
889
- error: err instanceof Error ? err.message : String(err)
890
- });
891
- return { peerId: peerIdStr, success: false as const, response: undefined };
892
- }
893
- }));
894
- const failures = results.filter(r => !r.success).map(r => r.peerId);
895
- // This broadcast is where members actually apply the operations, so their responses carry the
896
- // only report the coordinator ever gets of what each member's OWN storage said. Collecting it
897
- // here is what lets a pend refused by a non-coordinating member reach the writer as a conflict
898
- // instead of the fabricated success that used to fork the block.
899
- //
900
- // Each peer's entry is taken from that peer's OWN response and re-keyed under the peer we
901
- // asked, so a member cannot report an outcome on another member's behalf by echoing a record
902
- // full of entries. Unsigned and advisory either way — see ClusterRecord.applyOutcomes.
903
- const applyOutcomes = collectApplyOutcomes(results);
904
- return { failures, ...(applyOutcomes === undefined ? {} : { applyOutcomes }) };
905
- }
906
-
907
- /**
908
- * Fire-and-forget replay of an abandoned transaction's record to every peer in its cohort.
909
- *
910
- * Called only where the record itself proves the transaction is dead (enough signed rejections that
911
- * super-majority is unreachable). Each member re-derives `TransactionPhase.Rejected` from the votes
912
- * it verifies and drops the entry from its own reservation table, freeing the blocks immediately
913
- * instead of after its 2 s staleness window. No new message type and no wire-format change — this is
914
- * the same `update()` every other phase uses.
915
- *
916
- * Never awaited into the caller's throw and never rethrows: an abandonment must not turn into a
917
- * *different* failure, and the staleness sweep remains the backstop if delivery fails.
918
- */
919
- private broadcastAbandonment(record: ClusterRecord, reason: string): void {
920
- const peerIds = Object.keys(record.peers);
921
- log('cluster-tx:abandon-broadcast', { messageHash: record.messageHash, reason, peerIds });
922
- void Promise.all(peerIds.map(async peerIdStr => {
923
- try {
924
- await this.updateMember(peerIdStr, record, 0, 'abandon-broadcast');
925
- } catch (err) {
926
- log('cluster-tx:abandon-broadcast-error', {
927
- messageHash: record.messageHash,
928
- peerId: peerIdStr,
929
- error: err instanceof Error ? err.message : String(err)
930
- });
931
- }
932
- }));
933
- }
934
-
935
- private updateTransactionRecord(record: ClusterRecord, stage: string): void {
936
- const state = this.transactions.get(record.messageHash);
937
- if (!state) {
938
- log('cluster-tx:transaction-update-miss', { messageHash: record.messageHash, stage });
939
- return;
940
- }
941
- state.record = { ...record };
942
- state.lastUpdate = this.now();
943
- log('cluster-tx:transaction-update', {
944
- messageHash: record.messageHash,
945
- stage,
946
- promises: Object.keys(record.promises ?? {}),
947
- commits: Object.keys(record.commits ?? {})
948
- });
949
- }
950
-
951
- private scheduleCommitRetry(messageHash: string, _record: ClusterRecord, missingPeers: string[]): void {
952
- const state = this.transactions.get(messageHash);
953
- if (!state) {
954
- return;
955
- }
956
- const existing = state.retry;
957
- const nextAttempt = (existing?.attempt ?? 0) + 1;
958
- if (nextAttempt > this.retryMaxAttempts) {
959
- log('cluster-tx:retry-abort', { messageHash, missingPeers });
960
- return;
961
- }
962
- if (missingPeers.length === 0) {
963
- this.clearRetry(messageHash);
964
- return;
965
- }
966
- const pendingPeers = new Set(missingPeers);
967
- const baseInterval = existing ? Math.min(existing.intervalMs * this.retryBackoffFactor, this.retryMaxIntervalMs) : this.retryInitialIntervalMs;
968
- existing?.cancel?.();
969
- const cancel = this.setTimer(() => {
970
- void this.retryCommits(messageHash);
971
- }, baseInterval);
972
- state.retry = {
973
- pendingPeers,
974
- attempt: nextAttempt,
975
- intervalMs: baseInterval,
976
- cancel
977
- };
978
- this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
979
- pendingPeers: Array.from(pendingPeers),
980
- attempt: nextAttempt,
981
- intervalMs: baseInterval
982
- });
983
- log('cluster-tx:retry-scheduled', { messageHash, attempt: nextAttempt, missingPeers, delayMs: baseInterval });
984
- }
985
-
986
- private async retryCommits(messageHash: string): Promise<void> {
987
- const state = this.transactions.get(messageHash);
988
- if (!state?.retry) {
989
- return;
990
- }
991
- const { pendingPeers, attempt } = state.retry;
992
- if (pendingPeers.size === 0) {
993
- this.clearRetry(messageHash);
994
- return;
995
- }
996
- const peerIds = Array.from(pendingPeers);
997
- const record = state.record;
998
- log('cluster-tx:retry-start', { messageHash, attempt, peerIds });
999
- const results = await Promise.all(peerIds.map(async peerIdStr => {
1000
- const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
1001
- const payload: ClusterRecord = {
1002
- ...record,
1003
- commits: record.commits
1004
- };
1005
- try {
1006
- const res = isLocal
1007
- ? await this.localCluster!.update(payload)
1008
- : await this.createClusterClient(peerIdFromString(peerIdStr)).update(payload);
1009
- state.record.commits = { ...state.record.commits, ...res.commits };
1010
- return { peerId: peerIdStr, success: true as const };
1011
- } catch (err) {
1012
- return {
1013
- peerId: peerIdStr,
1014
- success: false as const,
1015
- error: err instanceof Error ? err.message : String(err)
1016
- };
1017
- }
1018
- }));
1019
- const successes = results.filter(r => r.success).map(r => r.peerId);
1020
- const failures = results.filter(r => !r.success);
1021
- for (const peerId of successes) {
1022
- pendingPeers.delete(peerId);
1023
- }
1024
- log('cluster-tx:retry-complete', { messageHash, attempt, successes, failures });
1025
- if (pendingPeers.size === 0) {
1026
- log('cluster-tx:retry-finished', { messageHash });
1027
- this.clearRetry(messageHash);
1028
- return;
1029
- }
1030
- if (!this.transactions.has(messageHash)) {
1031
- return;
1032
- }
1033
- this.scheduleCommitRetry(messageHash, state.record, Array.from(pendingPeers));
1034
- }
1035
-
1036
- private clearRetry(messageHash: string): void {
1037
- const state = this.transactions.get(messageHash);
1038
- if (!state?.retry) {
1039
- return;
1040
- }
1041
- state.retry.cancel?.();
1042
- state.retry = undefined;
1043
- // Clean up the transaction after retry is complete
1044
- this.setTimer(() => {
1045
- this.transactions.delete(messageHash);
1046
- this.deleteCoordinatorState(messageHash);
1047
- log('cluster-tx:transaction-remove', {
1048
- messageHash,
1049
- remaining: Array.from(this.transactions.keys())
1050
- });
1051
- }, 100);
1052
- }
1053
-
1054
- /** Fire-and-forget persist errors are logged, never thrown. */
1055
- private persistCoordinatorState(
1056
- messageHash: string,
1057
- record: ClusterRecord,
1058
- phase: 'promising' | 'committing' | 'broadcasting',
1059
- retryState?: { pendingPeers: string[]; attempt: number; intervalMs: number }
1060
- ): void {
1061
- if (!this.stateStore) return;
1062
- this.stateStore.saveCoordinatorState(messageHash, {
1063
- messageHash,
1064
- record,
1065
- lastUpdate: this.now(),
1066
- phase,
1067
- retryState
1068
- }).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
1069
- }
1070
-
1071
- /** Fire-and-forget delete — errors are logged, never thrown. */
1072
- private deleteCoordinatorState(messageHash: string): void {
1073
- if (!this.stateStore) return;
1074
- this.stateStore.deleteCoordinatorState(messageHash)
1075
- .catch(err => log('cluster-tx:persist-delete-error', { messageHash, error: (err as Error).message }));
1076
- }
1077
-
1078
- /**
1079
- * Recover coordinator transactions from persistent store after a restart.
1080
- * Called during node startup, before accepting new requests.
1081
- */
1082
- async recoverTransactions(): Promise<void> {
1083
- if (!this.stateStore) return;
1084
- const states = await this.stateStore.getAllCoordinatorStates();
1085
- for (const state of states) {
1086
- const { messageHash } = state;
1087
- // Expired clean up
1088
- if (state.record.message.expiration && state.record.message.expiration < this.now()) {
1089
- log('cluster-tx:recovery-expired', { messageHash });
1090
- await this.stateStore.deleteCoordinatorState(messageHash);
1091
- continue;
1092
- }
1093
- // Broadcasting phase with retry state resume retries
1094
- if (state.phase === 'broadcasting' && state.retryState) {
1095
- log('cluster-tx:recovery-resume-broadcast', { messageHash, attempt: state.retryState.attempt });
1096
- const pending = new Pending(Promise.resolve(state.record));
1097
- const txState: ClusterTransactionState = {
1098
- messageHash,
1099
- record: state.record,
1100
- pending,
1101
- lastUpdate: state.lastUpdate
1102
- };
1103
- this.transactions.set(messageHash, txState);
1104
- // Schedule retry from where we left off
1105
- this.scheduleCommitRetry(messageHash, state.record, state.retryState.pendingPeers);
1106
- continue;
1107
- }
1108
- // Promising or committing — cannot resume (caller context is gone)
1109
- log('cluster-tx:recovery-stale', { messageHash, phase: state.phase });
1110
- await this.stateStore.deleteCoordinatorState(messageHash);
1111
- }
1112
- }
1113
- }
1
+ import { peerIdFromString } from "@libp2p/peer-id";
2
+ import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult, StaleFailure } from "@optimystic/db-core";
3
+ import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, isConflictFailure, membershipDigest, routingKeyForBlock } from "@optimystic/db-core";
4
+ import { Pending } from "@optimystic/db-core";
5
+ import type { PeerId } from "@libp2p/interface";
6
+ import { createLogger, verbose } from '../logger.js'
7
+ import type { ClusterLogPeerOutcome } from './types.js'
8
+ import type { FretService } from "p2p-fret";
9
+ import type { IPeerReputation } from "../reputation/types.js";
10
+ import { PenaltyReason } from "../reputation/types.js";
11
+ import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
12
+ import { ResponsibilityRefusalError } from "./responsibility.js";
13
+
14
+ const log = createLogger('cluster')
15
+
16
+ /**
17
+ * Pick each peer's OWN {@link ClusterRecord.applyOutcomes} entry out of the record that peer answered
18
+ * with, and key it under the peer we actually asked.
19
+ *
20
+ * Taking only `response.applyOutcomes[peerId]` rather than spreading the whole map is what keeps
21
+ * one member from reporting outcomes on other members' behalf: a peer that echoes back a record full
22
+ * of entries contributes exactly one, its own. The field is unsigned advisory data (see its doc
23
+ * comment for why that is safe), so this is a shaping rule, not a security boundary.
24
+ *
25
+ * Returns `undefined` when no peer reported anything, so the common case adds no empty object to the
26
+ * record.
27
+ */
28
+ function collectApplyOutcomes(
29
+ responses: ReadonlyArray<{ peerId: string; response?: ClusterRecord | null }>
30
+ ): ClusterRecord['applyOutcomes'] {
31
+ let collected: NonNullable<ClusterRecord['applyOutcomes']> | undefined;
32
+ for (const { peerId, response } of responses) {
33
+ const own = response?.applyOutcomes?.[peerId];
34
+ if (own === undefined) continue;
35
+ collected ??= {};
36
+ collected[peerId] = own;
37
+ }
38
+ return collected;
39
+ }
40
+
41
+ /** Fold collected outcomes into a record in place, later report winning per peer. No-op for `undefined`. */
42
+ function mergeApplyOutcomes(record: ClusterRecord, collected: ClusterRecord['applyOutcomes']): void {
43
+ if (collected === undefined) return;
44
+ record.applyOutcomes = { ...record.applyOutcomes, ...collected };
45
+ }
46
+
47
+ /**
48
+ * Consensus refused a transaction: enough members voted reject that super-majority became
49
+ * impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
50
+ * "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
51
+ * rejection reasons — those are free-form text that is part of each member's signed vote payload
52
+ * (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
53
+ * `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
54
+ * loss (confirmed against local storage) or a genuine validation fault.
55
+ */
56
+ export class ValidatorRejectionError extends Error {
57
+ constructor(
58
+ message: string,
59
+ /** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
60
+ readonly rejectReasons: Record<string, string>
61
+ ) {
62
+ super(message);
63
+ this.name = 'ValidatorRejectionError';
64
+ }
65
+ }
66
+
67
+ /**
68
+ * The transaction lost a conflict race: one or more members answered with a signed `conflict`
69
+ * vote (they hold a rival transaction that won the deterministic race on the same blocks) and
70
+ * approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} nobody
71
+ * judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
72
+ * `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
73
+ * `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
74
+ * as a thrown error only from other paths. The commit conversion matters as much as the pend one:
75
+ * at the moment this is thrown zero members approved and the members hold the winner — nothing of
76
+ * the loser landed yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
77
+ * (it treats throws as transport faults), and that re-driven commit races into the window after
78
+ * members apply the winner and clear its reservation, where it can assemble a consensus no member
79
+ * will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
80
+ * writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
81
+ * and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
82
+ * parsed out of prose.
83
+ */
84
+ export class ConflictRaceLostError extends Error {
85
+ constructor(
86
+ message: string,
87
+ /** peerId messageHash of the rival transaction that member holds as the race winner. */
88
+ readonly conflicts: Record<string, string>
89
+ ) {
90
+ super(message);
91
+ this.name = 'ConflictRaceLostError';
92
+ }
93
+ }
94
+
95
+ /**
96
+ * The transaction's pend could not proceed because one or more members answered with a signed `held`
97
+ * vote: the requested blocks are reserved by a different unresolved action in that member's durable
98
+ * storage. Sibling of {@link ConflictRaceLostError} and retryable for the same reason nobody judged
99
+ * this write invalid; it queued behind a reservation that disappears when the holder commits or
100
+ * cancels.
101
+ *
102
+ * The two are separate because they name different things. A conflict vote names the winning rival's
103
+ * `messageHash`, which the member holds whole; a held vote can only name the rival's **action id**,
104
+ * because it fires in the window where the rival has left the member's in-memory table but not yet its
105
+ * storage. `CoordinatorRepo.pend` converts this into a `StaleFailure` with `conflict: true` so the
106
+ * normal retry machinery (`isConflictFailure`) absorbs it, exactly as it does a lost race.
107
+ *
108
+ * Only a PEND record can produce it: `held` votes come from `ClusterMember.validatePendOperations`,
109
+ * which inspects pend operations only, so `CoordinatorRepo.commit` never meets one.
110
+ */
111
+ export class BlocksHeldError extends Error {
112
+ constructor(
113
+ message: string,
114
+ /** peerId → actionId of the unresolved action that member's storage says holds the blocks. */
115
+ readonly heldBy: Record<string, string>
116
+ ) {
117
+ super(message);
118
+ this.name = 'BlocksHeldError';
119
+ }
120
+ }
121
+
122
+ /** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
123
+ export type TimerCancel = () => void;
124
+
125
+ /**
126
+ * Production timer binding: a one-shot `setTimeout` whose handle is **unref'd** so a pending
127
+ * commit-retry (or the deferred transaction cleanup) never keeps an otherwise-idle process alive.
128
+ * The returned handle clears the timeout (idempotent). Mirrors the reactivity rotation
129
+ * re-registration scheduler's `defaultSetTimer` (see reactivity/rotation-rereg-scheduler.ts).
130
+ */
131
+ function defaultSetTimer(fn: () => void, delayMs: number): TimerCancel {
132
+ const handle = setTimeout(fn, delayMs);
133
+ // An idle retry/cleanup timer must not pin a process (mirror rotation re-registration + push-state gossip).
134
+ (handle as { unref?: () => void }).unref?.();
135
+ return (): void => clearTimeout(handle);
136
+ }
137
+
138
+ /**
139
+ * Optional injection seam for deterministic time. Production leaves both undefined and gets
140
+ * `Date.now` + an unref'd `setTimeout`; tests inject a fake clock + timer queue so scheduled
141
+ * commit-retries fire in virtual (not wall-clock) time.
142
+ */
143
+ export interface ClusterCoordinatorClock {
144
+ /** Clock (Unix ms). Defaults to `Date.now`. */
145
+ now?: () => number;
146
+ /** Schedule a one-shot timer, returning a cancel handle. Defaults to an unref'd `setTimeout`. */
147
+ setTimer?: (fn: () => void, delayMs: number) => TimerCancel;
148
+ }
149
+
150
+ /**
151
+ * Manages the state of cluster transactions for a specific block ID
152
+ */
153
+ interface CommitRetryState {
154
+ pendingPeers: Set<string>;
155
+ attempt: number;
156
+ intervalMs: number;
157
+ cancel?: TimerCancel;
158
+ }
159
+
160
+ interface ClusterTransactionState {
161
+ messageHash: string;
162
+ record: ClusterRecord;
163
+ pending: Pending<ClusterRecord>;
164
+ lastUpdate: number;
165
+ promiseTimeout?: NodeJS.Timeout;
166
+ resolutionTimeout?: NodeJS.Timeout;
167
+ retry?: CommitRetryState;
168
+ }
169
+
170
+ /** Manages distributed transactions across clusters */
171
+ /**
172
+ * What a cohort lookup established about a block's cohort. `resolved: false` covers BOTH a lookup
173
+ * that threw and one that answered with nobody: neither names a destination for a write, and the
174
+ * durability class both produce is the same (`unrouted`). `reason` is for logs only — never branch
175
+ * on it.
176
+ */
177
+ export type CohortResolution =
178
+ | { readonly resolved: true; readonly peerIds: readonly string[] }
179
+ | { readonly resolved: false; readonly reason: string };
180
+
181
+ export class ClusterCoordinator {
182
+ private transactions: Map<string, ClusterTransactionState> = new Map();
183
+ private readonly retryInitialIntervalMs: number;
184
+ private readonly retryBackoffFactor: number;
185
+ private readonly retryMaxIntervalMs: number;
186
+ private readonly retryMaxAttempts: number;
187
+ private readonly commitBroadcastImmediateRetries: number;
188
+ private readonly promiseImmediateRetries: number;
189
+ /** Injected clock/timer seam; production defaults to `Date.now` + unref'd `setTimeout`. */
190
+ private readonly now: () => number;
191
+ private readonly setTimer: (fn: () => void, delayMs: number) => TimerCancel;
192
+
193
+ constructor(
194
+ private readonly keyNetwork: IKeyNetwork,
195
+ /** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
196
+ private readonly createClusterClient: (peerId: PeerId) => ICluster,
197
+ private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
198
+ private readonly localCluster?: {
199
+ update: (record: ClusterRecord) => Promise<ClusterRecord>;
200
+ peerId: PeerId;
201
+ wasTransactionExecuted?: (messageHash: string) => boolean;
202
+ /** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
203
+ getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
204
+ /** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
205
+ getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
206
+ },
207
+ private readonly fretService?: FretService,
208
+ private readonly reputation?: IPeerReputation,
209
+ private readonly stateStore?: ITransactionStateStore,
210
+ clock?: ClusterCoordinatorClock
211
+ ) {
212
+ this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
213
+ this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
214
+ this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
215
+ this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
216
+ this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
217
+ this.promiseImmediateRetries = cfg.promiseImmediateRetries ?? 1;
218
+ this.now = clock?.now ?? ((): number => Date.now());
219
+ this.setTimer = clock?.setTimer ?? defaultSetTimer;
220
+ }
221
+
222
+ /**
223
+ * Invoke one cluster member's `update`, retrying transient REMOTE failures up to
224
+ * `immediateRetries` times before surfacing the error. The local cluster is invoked
225
+ * exactly once a local throw is a real fault (validation / merge / consensus), not a
226
+ * transient transport blip. A remote call rides a libp2p stream that a circuit-relay
227
+ * ("limited") connection can reset once a per-circuit cap or reservation lapses, which
228
+ * surfaces as a StreamResetError; an immediate retry on the (usually still-warm)
229
+ * connection recovers most of those without escalating the peer to a failure. Shared by
230
+ * the promise-collection, commit-collection, and commit-broadcast phases so all three
231
+ * react to a relayed reset the same way.
232
+ */
233
+ private async updateMember(peerIdStr: string, record: ClusterRecord, immediateRetries: number, phase: string): Promise<ClusterRecord> {
234
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
235
+ if (isLocal) {
236
+ return await this.localCluster!.update(record);
237
+ }
238
+ const maxAttempts = 1 + Math.max(0, immediateRetries);
239
+ let lastError: unknown;
240
+ for (let attempt = 1; attempt <= maxAttempts; attempt++) {
241
+ try {
242
+ return await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
243
+ } catch (err) {
244
+ lastError = err;
245
+ if (attempt < maxAttempts) {
246
+ log('cluster-tx:member-update-retry', {
247
+ messageHash: record.messageHash,
248
+ peerId: peerIdStr,
249
+ phase,
250
+ attempt,
251
+ error: err instanceof Error ? err.message : String(err)
252
+ });
253
+ }
254
+ }
255
+ }
256
+ throw lastError;
257
+ }
258
+
259
+ /**
260
+ * Creates a base58btc string hash uniquely identifying a transaction. For a v2 record the caller
261
+ * threads in the {@link membershipDigest} of the peer set so the responsible membership is bound into
262
+ * the identity (two different peer sets ⇒ two different hashes). Omitting `membershipDigestValue`
263
+ * reproduces the legacy v1 hash byte-for-byte.
264
+ *
265
+ * NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
266
+ * which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
267
+ * `pend.priority` (single-collection) is automatically covered here and by the derived
268
+ * promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
269
+ * cannot strip or inflate it without invalidating the message hash the members verify. No separate
270
+ * priority-hashing step is needed.
271
+ */
272
+ private async createMessageHash(message: RepoMessage, membershipDigestValue?: string): Promise<string> {
273
+ return computeClusterMessageHash(message, membershipDigestValue);
274
+ }
275
+
276
+ /**
277
+ * The ONE cohort lookup every accessor on this class derives from: the raw peer map when the key
278
+ * network answered, otherwise the reason it did not. A thrown `findCluster` is logged here and
279
+ * nowhere else. Callers that need the map (`executeClusterTransaction`, which builds the record's
280
+ * `peers`) go through {@link getClusterForBlock}; callers that need to know whether the cohort
281
+ * RESOLVED go through {@link resolveCohort}.
282
+ */
283
+ private async lookupCluster(blockId: BlockId): Promise<{ peers: ClusterPeers } | { reason: string }> {
284
+ try {
285
+ const peers = await this.keyNetwork.findCluster(routingKeyForBlock(blockId));
286
+ const peerIds = Object.keys(peers ?? {});
287
+ log('cluster-tx:cluster-members', { blockId, peerIds });
288
+ return { peers: peers ?? {} };
289
+ } catch (e) {
290
+ log('WARN findCluster failed for %s: %o', blockId, e)
291
+ return { reason: `findCluster threw: ${(e as Error)?.message ?? String(e)}` };
292
+ }
293
+ }
294
+
295
+ /**
296
+ * Gets all peers in the cluster for a specific block ID. Empty when the lookup failed the
297
+ * consensus path treats "no cohort" and "lookup failed" alike (there is nobody to run consensus
298
+ * with either way); a caller that must tell them apart uses {@link resolveCohort}.
299
+ */
300
+ private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
301
+ const outcome = await this.lookupCluster(blockId);
302
+ return 'peers' in outcome ? outcome.peers : {};
303
+ }
304
+
305
+ /**
306
+ * Whether the block's cohort could be established, and who it is. The primitive behind
307
+ * {@link getClusterPeerIds} and {@link getClusterSize}: a lookup that threw and a lookup that named
308
+ * nobody used to reach every caller as the same empty list, and `CoordinatorRepo`'s solo
309
+ * short-circuit then acknowledged a write it had no idea where to send exactly as it acknowledged a
310
+ * write to a genuine cohort of one (GitHub #19). Both shapes are still `resolved: false` here —
311
+ * neither names a destination but they are distinguishable from a resolved cohort, which is what
312
+ * the write's durability class needs (`unrouted` vs `local`).
313
+ */
314
+ async resolveCohort(blockId: BlockId): Promise<CohortResolution> {
315
+ const outcome = await this.lookupCluster(blockId);
316
+ if ('reason' in outcome) return { resolved: false, reason: outcome.reason };
317
+ const peerIds = Object.keys(outcome.peers);
318
+ if (peerIds.length === 0) return { resolved: false, reason: 'findCluster named nobody' };
319
+ return { resolved: true, peerIds };
320
+ }
321
+
322
+ /**
323
+ * A node never runs a cluster transaction for a cohort it is not in. Behind members reconcile from the
324
+ * coordinator's own proof-carrying copy (its member applies before the merged record fans out), and a
325
+ * coordinator outside `record.peers` is not a reconcile target so a cohort with no holder would stay
326
+ * behind and the commit durability gate would refuse, having first put this node's vote and storage
327
+ * where the cohort does not look. The invariant is held here, at the one place a record's `peers` is
328
+ * chosen, rather than left to the routing convention.
329
+ *
330
+ * Fires only on a RESOLVED cohort (at least one peer) that excludes the wired local member. An empty
331
+ * cohort is a failed lookup, not a cohort this node is outside of, so it is left to `executeTransaction`'s
332
+ * size checks; `CoordinatorRepo`'s solo short-circuit keeps unresolved and single-peer cohorts away from
333
+ * this method altogether in any case. After its responsibility check, what remains is a multi-member
334
+ * cohort that changed inside the responsibility cache's staleness window. With no local member wired the guard does not apply: that
335
+ * bypass exists for wiring without an identity (direct constructors, some tests), never for production.
336
+ */
337
+ private assertLocalMemberInCohort(blockId: BlockId, peers: ClusterPeers): void {
338
+ if (!this.localCluster) return;
339
+ const peerIds = Object.keys(peers);
340
+ const selfId = this.localCluster.peerId.toString();
341
+ if (peerIds.length === 0 || peerIds.includes(selfId)) return;
342
+ log('cluster-tx:not-in-cohort', { blockId, selfId, peerIds });
343
+ throw new ResponsibilityRefusalError('not-responsible', [blockId],
344
+ `refusing to coordinate a cluster transaction for a cohort this node is not in: ${peerIds.join(', ')}`);
345
+ }
346
+
347
+ private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
348
+ const peerCount = Object.keys(peers ?? {}).length;
349
+ const record: ClusterRecord = {
350
+ messageHash,
351
+ peers,
352
+ // v2: bind the responsible membership into the signed identity. messageHash was computed over
353
+ // this same digest, so a different peer set would have produced a different messageHash.
354
+ membershipVersion: CURRENT_MEMBERSHIP_VERSION,
355
+ membershipDigest: membershipDigestValue,
356
+ message,
357
+ promises: {},
358
+ commits: {},
359
+ suggestedClusterSize: peerCount || undefined,
360
+ minRequiredSize: this.cfg.allowClusterDownsize ? undefined : this.cfg.clusterSize
361
+ };
362
+
363
+ // Add network size hint if available
364
+ if (this.fretService) {
365
+ try {
366
+ const estimate = this.fretService.getNetworkSizeEstimate();
367
+ if (estimate.size_estimate > 0) {
368
+ record.networkSizeHint = estimate.size_estimate;
369
+ record.networkSizeConfidence = estimate.confidence;
370
+ }
371
+ } catch (err) {
372
+ // Ignore errors getting size estimate
373
+ }
374
+ }
375
+
376
+ return record;
377
+ }
378
+
379
+ /**
380
+ * Initiates a 2-phase transaction for a specific block ID.
381
+ * Returns the cluster record and whether the local cluster already executed the operations.
382
+ */
383
+ async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
384
+ record: ClusterRecord;
385
+ localExecuted: boolean;
386
+ /**
387
+ * Local storage's verdict for a pend operation this node's own cluster member applied during
388
+ * consensus, when the member retained one. Meaningful only when `localExecuted` is true;
389
+ * absent for non-pend messages, for a member that predates the retention, or after the
390
+ * retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
391
+ */
392
+ localPendResult?: PendResult;
393
+ /**
394
+ * Local storage's verdict for a commit operation this node's own cluster member applied
395
+ * during consensus, when the member retained one. Same availability contract as
396
+ * `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
397
+ * win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
398
+ * success no member durably stored.
399
+ */
400
+ localCommitResult?: CommitResult;
401
+ /**
402
+ * Conflict-shaped pend refusals reported by OTHER cohort members on their consensus responses
403
+ * (`ClusterRecord.applyOutcomes`), keyed by peer id. This is the arm `localPendResult` cannot
404
+ * cover: the refusing member is frequently not the coordinating node, and its verdict used to
405
+ * stay on that member while the writer was told the pend won. Unsigned advisory data — an
406
+ * entry means "retry", never "this write was invalid". Absent when nobody reported one.
407
+ *
408
+ * Residual: a member that reaches consensus only via the scheduled commit-retry timer applies
409
+ * after this method has already resolved, so its refusal arrives too late to appear here. The
410
+ * member-side commit-promise guard (`validateCommitAgainstRefusedPend`) is the backstop for
411
+ * that path.
412
+ */
413
+ cohortPendRefusals?: { [peerId: string]: StaleFailure };
414
+ /**
415
+ * What OTHER cohort members reported about durably holding a commit after applying it at
416
+ * consensus (`ClusterRecord.applyOutcomes[peer].commit`), keyed by peer id — successes AND
417
+ * refusals, because `CoordinatorRepo.commit`'s durability gate counts the successes against
418
+ * the cohort the commit ran on and acknowledges only a majority. Each member's verdict is
419
+ * measured after its own reconcile, so a member that pulled the revision from a cohort peer
420
+ * reports success. Self is excluded for the same reason as `cohortPendRefusals` (its verdict
421
+ * travels as `localCommitResult`). Unsigned advisory data: a false success is one holder the
422
+ * member's signed approve vote already admitted to the majority; a false refusal is retry
423
+ * pressure. Absent when nobody reported one (a pend message, or pre-upgrade members).
424
+ *
425
+ * Same residual as `cohortPendRefusals`: a member reached only by the scheduled commit-retry
426
+ * timer applies after this method has resolved, and its report arrives too late to count —
427
+ * the gate then refuses honestly and the writer re-drives.
428
+ */
429
+ cohortCommitOutcomes?: { [peerId: string]: CommitResult };
430
+ }> {
431
+ // The coordinating block id is derived HERE, from the key this method is already handed, rather
432
+ // than being set by each caller's message builder: a member's membership admission gate derives
433
+ // its own cohort view from this field, and a builder that forgets it silently downgrades the gate
434
+ // to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
435
+ // admitted at pend, refused at commit). Doing it at the single choke point means a future message
436
+ // builder cannot reintroduce the gap.
437
+ //
438
+ // Two constraints this shape exists to satisfy:
439
+ // - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
440
+ // N concurrent calls, one per block. In-place mutation would leak one block's id into another
441
+ // block's transaction.
442
+ // - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
443
+ // not just its first block, so this must not overwrite it. Tested on `length`, not on the
444
+ // field: an empty list carries no id for a member to derive from, so preserving one would be
445
+ // the same silent downgrade to the fallback floor this choke point exists to prevent.
446
+ const coordinated: RepoMessage = message.coordinatingBlockIds?.length
447
+ ? message
448
+ : { ...message, coordinatingBlockIds: [blockId] };
449
+
450
+ // Get the cluster peers for this block
451
+ const peers = await this.getClusterForBlock(blockId);
452
+ this.assertLocalMemberInCohort(blockId, peers);
453
+
454
+ // Bind the responsible membership into the transaction identity (v2): the digest is folded into
455
+ // the messageHash below, so two different peer sets produce two different messageHashes rather
456
+ // than one hash with a silent internal disagreement about who is responsible.
457
+ const membershipDigestValue = await membershipDigest(peers);
458
+
459
+ // Create a unique hash for this transaction (over message + membership digest). Hashing the
460
+ // coordinating-block-bearing copy is what makes the field tamper-evident in transit — and it also
461
+ // makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
462
+ // identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
463
+ const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
464
+
465
+ // Create a cluster record for this transaction
466
+ const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
467
+ log('cluster-tx:start', {
468
+ messageHash,
469
+ blockId,
470
+ peerCount: Object.keys(peers ?? {}).length,
471
+ allowDownsize: this.cfg.allowClusterDownsize,
472
+ configuredSize: this.cfg.clusterSize,
473
+ suggestedSize: record.suggestedClusterSize,
474
+ minRequiredSize: record.minRequiredSize
475
+ });
476
+
477
+ // Create a new pending transaction
478
+ const transactionPromise = this.executeTransaction(peers, record);
479
+ const pending = new Pending(transactionPromise);
480
+
481
+ // Store the transaction state
482
+ const state: ClusterTransactionState = {
483
+ messageHash,
484
+ record,
485
+ pending,
486
+ lastUpdate: this.now()
487
+ };
488
+ this.transactions.set(messageHash, state);
489
+ this.persistCoordinatorState(messageHash, record, 'promising');
490
+ log('cluster-tx:transaction-store', {
491
+ messageHash,
492
+ transactionKeys: Array.from(this.transactions.keys())
493
+ });
494
+
495
+ // Wait for the transaction to complete
496
+ try {
497
+ const result = await pending.result();
498
+ // Check if the local cluster already executed the operations during consensus
499
+ const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
500
+ const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
501
+ const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
502
+ // Self is excluded: this node's own member verdict is already carried, more directly and
503
+ // without the wire round trip, by `localPendResult` — and leaving it in both places would
504
+ // make the coordinator's "prefer local" rule ambiguous.
505
+ // Re-checked here rather than trusted: members are supposed to report only conflict-shaped
506
+ // refusals, but the field arrives off the wire, so anything else (a success, a bare-reason
507
+ // fault, a malformed entry) is dropped instead of being handed to a caller that would read
508
+ // it as a retryable conflict.
509
+ const selfId = this.localCluster?.peerId.toString();
510
+ const cohortPendRefusals: { [peerId: string]: StaleFailure } = {};
511
+ // The commit arm is re-checked the same way, to the shape the gate reads: a plain
512
+ // `success: true`, or an object whose `success` is `false`. Anything else off the wire is
513
+ // dropped rather than counted as a holder.
514
+ const cohortCommitOutcomes: { [peerId: string]: CommitResult } = {};
515
+ for (const [peerId, outcome] of Object.entries(result.applyOutcomes ?? {})) {
516
+ if (peerId === selfId) continue;
517
+ const pend = outcome?.pend;
518
+ if (pend !== undefined && !pend.success && isConflictFailure(pend)) {
519
+ cohortPendRefusals[peerId] = pend;
520
+ }
521
+ const commit = outcome?.commit;
522
+ if (commit !== null && typeof commit === 'object' && (commit.success === true || commit.success === false)) {
523
+ cohortCommitOutcomes[peerId] = commit;
524
+ }
525
+ }
526
+ return {
527
+ record: result,
528
+ localExecuted,
529
+ ...(localPendResult === undefined ? {} : { localPendResult }),
530
+ ...(localCommitResult === undefined ? {} : { localCommitResult }),
531
+ ...(Object.keys(cohortPendRefusals).length === 0 ? {} : { cohortPendRefusals }),
532
+ ...(Object.keys(cohortCommitOutcomes).length === 0 ? {} : { cohortCommitOutcomes })
533
+ };
534
+ } finally {
535
+ const stored = this.transactions.get(messageHash);
536
+ const retrySnapshot = stored?.retry ? {
537
+ attempt: stored.retry.attempt,
538
+ pending: Array.from(stored.retry.pendingPeers ?? [])
539
+ } : undefined;
540
+ log('cluster-tx:complete', {
541
+ messageHash,
542
+ finalPromises: stored ? Object.keys(stored.record.promises ?? {}) : undefined,
543
+ finalCommits: stored ? Object.keys(stored.record.commits ?? {}) : undefined,
544
+ retry: retrySnapshot
545
+ });
546
+ // Don't remove transaction immediately if retries are scheduled
547
+ // Let the retry completion or abort handle cleanup
548
+ if (!stored?.retry) {
549
+ // Wait a bit before cleanup to allow any in-flight responses to arrive
550
+ this.setTimer(() => {
551
+ this.transactions.delete(messageHash);
552
+ this.deleteCoordinatorState(messageHash);
553
+ log('cluster-tx:transaction-remove', {
554
+ messageHash,
555
+ remaining: Array.from(this.transactions.keys())
556
+ });
557
+ }, 100);
558
+ }
559
+ }
560
+ }
561
+
562
+ /**
563
+ * Executes the full transaction process
564
+ */
565
+ private async executeTransaction(peers: ClusterPeers, record: ClusterRecord): Promise<ClusterRecord> {
566
+ const peerCount = Object.keys(peers).length;
567
+
568
+ // Validate against minimum cluster size
569
+ if (peerCount < this.cfg.minAbsoluteClusterSize) {
570
+ const validated = await this.validateSmallCluster(peerCount, peers);
571
+ if (!validated) {
572
+ log('cluster-tx:reject-too-small', {
573
+ peerCount,
574
+ minRequired: this.cfg.minAbsoluteClusterSize
575
+ });
576
+ throw new Error(`Cluster size ${peerCount} below minimum ${this.cfg.minAbsoluteClusterSize} and not validated`);
577
+ }
578
+ log('cluster-tx:small-cluster-validated', { peerCount });
579
+ }
580
+
581
+ // Check configured cluster size
582
+ if (!this.cfg.allowClusterDownsize && peerCount < this.cfg.clusterSize) {
583
+ log('cluster-tx:reject-downsize', { peerCount, required: this.cfg.clusterSize });
584
+ throw new Error(`Cluster size ${peerCount} below configured minimum ${this.cfg.clusterSize}`);
585
+ }
586
+
587
+ // Collect promises with super-majority requirement
588
+ const promised = await this.collectPromises(peers, record);
589
+ const superMajority = Math.ceil(peerCount * this.cfg.superMajorityThreshold);
590
+
591
+ // Count approvals, rejections and the two RETRYABLE refusals separately. A `conflict` vote is a
592
+ // member saying "not now — I hold the race winner"; a `held` vote is a member saying "not now
593
+ // a different unresolved action holds these blocks in my storage". Neither may count toward
594
+ // approvals OR rejections, or a transient refusal would masquerade as a validator rejection
595
+ // (permanent) or as silence (indistinguishable from an unreachable cohort) both wrong.
596
+ const promises = promised.record.promises;
597
+ const approvalCount = Object.values(promises).filter(sig => sig.type === 'approve').length;
598
+ const rejectionCount = Object.values(promises).filter(sig => sig.type === 'reject').length;
599
+ const conflictCount = Object.values(promises).filter(sig => sig.type === 'conflict').length;
600
+ const heldCount = Object.values(promises).filter(sig => sig.type === 'held').length;
601
+
602
+ // Check if rejections make super-majority impossible
603
+ // If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
604
+ const maxAllowedRejections = peerCount - superMajority;
605
+ // Whether the merged record itself PROVES super-majority unreachable — the same sum a member
606
+ // re-derives as `ConflictSuperseded`/`Rejected` from the signed votes, which is what makes an
607
+ // abandonment broadcast proof-carrying rather than an unauthenticated "forget this".
608
+ const refusalsProveUnreachable = rejectionCount + conflictCount + heldCount > maxAllowedRejections;
609
+ if (rejectionCount > maxAllowedRejections) {
610
+ const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
611
+ .flatMap(([peerId, sig]) => sig.type === 'reject' ? [[peerId, sig.rejectReason ?? 'unknown'] as const] : []));
612
+ const rejectReasons = Object.entries(rejectReasonsByPeer)
613
+ .map(([peerId, reason]) => `${peerId}: ${reason}`)
614
+ .join('; ');
615
+ log('cluster-tx:rejected-by-validators', {
616
+ messageHash: record.messageHash,
617
+ peerCount,
618
+ rejections: rejectionCount,
619
+ maxAllowed: maxAllowedRejections,
620
+ reasons: rejectReasons
621
+ });
622
+ this.updateTransactionRecord(promised.record, 'rejected-by-validators');
623
+ // Abandoning here without telling anyone leaves every member that voted holding this
624
+ // transaction in its own reservation table, blocking its blocks until that member's
625
+ // staleness sweep fires — and each retry we throw back to the caller plants a fresh
626
+ // reservation, so the block never frees. The merged record carries enough signed
627
+ // rejections to *prove* the transaction is dead, so replaying it to the cohort makes
628
+ // every member recompute `Rejected` and clear immediately. Proof-carrying, so a member
629
+ // need not trust us: it verifies the signatures it is shown.
630
+ this.broadcastAbandonment(promised.record, 'rejected-by-validators');
631
+ throw new ValidatorRejectionError(
632
+ `Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
633
+ rejectReasonsByPeer);
634
+ }
635
+
636
+ // A conflict-answered shortfall is a LOST RACE, not a validator verdict and not silence.
637
+ // Checked after the rejection threshold (a genuine validator rejection still wins) and
638
+ // before the generic shortfall (which must stay reserved for the genuinely-silent cohort).
639
+ if (conflictCount > 0 && approvalCount < superMajority) {
640
+ const conflicts = Object.fromEntries(Object.entries(promises)
641
+ .flatMap(([peerId, sig]) => sig.type === 'conflict' ? [[peerId, sig.conflictWith] as const] : []));
642
+ log('cluster-tx:conflict-race-lost', {
643
+ messageHash: record.messageHash,
644
+ peerCount,
645
+ approvals: approvalCount,
646
+ rejections: rejectionCount,
647
+ conflicts,
648
+ superMajority
649
+ });
650
+ this.updateTransactionRecord(promised.record, 'conflict-race-lost');
651
+ // Broadcast only when the merged record itself PROVES the transaction can no longer reach
652
+ // super-majority (members re-derive ConflictSuperseded/Rejected from the signed votes and
653
+ // clear their reservations immediately). Below that bar the record proves nothing and a
654
+ // broadcast would be the unauthenticated "forget this" the shortfall NOTE below refuses.
655
+ if (refusalsProveUnreachable) {
656
+ this.broadcastAbandonment(promised.record, 'conflict-race-lost');
657
+ }
658
+ throw new ConflictRaceLostError(
659
+ `Conflict race lost: ${conflictCount}/${peerCount} member(s) hold a conflicting winner (${approvalCount}/${superMajority} approvals)`,
660
+ conflicts);
661
+ }
662
+
663
+ // A `held`-answered shortfall is the OTHER retryable refusal: the pend queued behind a rival's
664
+ // unresolved reservation. Checked after the conflict branch so a lost race still wins when both
665
+ // answer a conflict vote names the winning transaction's messageHash, which is strictly more
666
+ // actionable than an action idand, like it, before the generic shortfall, which must stay
667
+ // reserved for the genuinely-silent cohort.
668
+ if (heldCount > 0 && approvalCount < superMajority) {
669
+ const heldBy = Object.fromEntries(Object.entries(promises)
670
+ .flatMap(([peerId, sig]) => sig.type === 'held' ? [[peerId, sig.heldBy] as const] : []));
671
+ log('cluster-tx:pend-blocks-held', {
672
+ messageHash: record.messageHash,
673
+ peerCount,
674
+ approvals: approvalCount,
675
+ rejections: rejectionCount,
676
+ heldBy,
677
+ superMajority
678
+ });
679
+ this.updateTransactionRecord(promised.record, 'pend-blocks-held');
680
+ if (refusalsProveUnreachable) {
681
+ this.broadcastAbandonment(promised.record, 'pend-blocks-held');
682
+ }
683
+ throw new BlocksHeldError(
684
+ `Pend blocks held: ${heldCount}/${peerCount} member(s) hold an unresolved rival action (${approvalCount}/${superMajority} approvals)`,
685
+ heldBy);
686
+ }
687
+
688
+ if (peerCount > 1 && approvalCount < superMajority) {
689
+ log('cluster-tx:supermajority-failed', {
690
+ messageHash: record.messageHash,
691
+ peerCount,
692
+ approvals: approvalCount,
693
+ rejections: rejectionCount,
694
+ superMajority,
695
+ threshold: this.cfg.superMajorityThreshold
696
+ });
697
+ this.updateTransactionRecord(promised.record, 'supermajority-failed');
698
+ // NOTE: deliberately NOT broadcast, unlike the rejected-by-validators branch above. With
699
+ // conflict-answered shortfalls peeled off above, we get here only because peers did not
700
+ // answer at all, so the record carries no signed evidence that the transaction is dead — a
701
+ // broadcast would be an unauthenticated "forget this" that any caller could use to clear a
702
+ // live transaction out of a member's reservation table. Members that DID vote are freed by
703
+ // their own staleness sweep instead.
704
+ // NOTE: the message below is load-bearing wire text — the consuming repo
705
+ // (sereus cadre-core control-write-retry) matches it verbatim to retry a genuinely-silent
706
+ // cohort. Keep it byte-identical, and never fold `conflict` or `held` votes into its
707
+ // rejection count.
708
+ throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
709
+ }
710
+
711
+ // Mark as disputed when minority rejections exist but super-majority approves
712
+ if (rejectionCount > 0 && approvalCount >= superMajority) {
713
+ const rejectingPeers: string[] = [];
714
+ const rejectReasons: { [peerId: string]: string } = {};
715
+ for (const [peerId, sig] of Object.entries(promises)) {
716
+ if (sig.type === 'reject') {
717
+ rejectingPeers.push(peerId);
718
+ rejectReasons[peerId] = sig.rejectReason ?? 'unknown';
719
+ }
720
+ }
721
+ promised.record.disputed = true;
722
+ promised.record.disputeEvidence = { rejectingPeers, rejectReasons };
723
+ log('cluster-tx:disputed', {
724
+ messageHash: record.messageHash,
725
+ rejectingPeers,
726
+ rejectReasons,
727
+ approvalCount,
728
+ rejectionCount,
729
+ peerCount
730
+ });
731
+ // [dispute-subsystem-dormant] Evidence is computed and persisted but initiateDispute() is
732
+ // intentionally NOT called here. Dispute origination stays dormant pending arbitrator-set
733
+ // anchoring — without it a forged synthetic cohort passes resolution.
734
+ // Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
735
+ // Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
736
+ }
737
+
738
+ this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
739
+ return await this.commitTransaction(promised.record);
740
+ }
741
+
742
+ /**
743
+ * The block's cohort peer ids as currently derivable. Empty when the cohort did not resolve
744
+ * ({@link resolveCohort}: `findCluster` threw, or named nobody), so a caller branching on
745
+ * `length <= 1` is also taking the degraded-routing branch. Derived from `resolveCohort` rather
746
+ * than re-deriving the cohort, so there is exactly one lookup rule.
747
+ */
748
+ async getClusterPeerIds(blockId: BlockId): Promise<string[]> {
749
+ const cohort = await this.resolveCohort(blockId);
750
+ return cohort.resolved ? [...cohort.peerIds] : [];
751
+ }
752
+
753
+ /** {@link getClusterPeerIds}, counted. Derived from it rather than re-deriving the cohort, so the
754
+ * size a caller branches on and the ids it logs can never come from two different rules. */
755
+ async getClusterSize(blockId: BlockId): Promise<number> {
756
+ return (await this.getClusterPeerIds(blockId)).length;
757
+ }
758
+
759
+ /**
760
+ * Validate that a small cluster size is legitimate by querying remote peers
761
+ * for their network size estimates. Returns true if estimates roughly agree.
762
+ */
763
+ private async validateSmallCluster(localSize: number, _peers: ClusterPeers): Promise<boolean> {
764
+ // If we have FRET and it shows confident estimate
765
+ if (this.fretService) {
766
+ try {
767
+ const estimate = this.fretService.getNetworkSizeEstimate();
768
+ if (estimate.confidence > 0.5) {
769
+ // Check if FRET estimate roughly matches observed cluster size
770
+ const orderOfMagnitude = Math.floor(Math.log10(estimate.size_estimate + 1));
771
+ const localOrderOfMagnitude = Math.floor(Math.log10(localSize + 1));
772
+
773
+ // If within same order of magnitude, accept it
774
+ if (Math.abs(orderOfMagnitude - localOrderOfMagnitude) <= 1) {
775
+ log('cluster-tx:small-cluster-validated-by-fret', {
776
+ localSize,
777
+ fretEstimate: estimate.size_estimate,
778
+ confidence: estimate.confidence,
779
+ sources: estimate.sources
780
+ });
781
+ return true;
782
+ }
783
+ }
784
+ } catch (err) {
785
+ // Ignore errors
786
+ }
787
+ }
788
+
789
+ // Fallback: with no confident network-size estimate, fail CLOSED by default.
790
+ // An undersized cluster with no way to justify its size is unsafe (a lone/
791
+ // near-lone node could rubber-stamp its own writes), so reject unless the
792
+ // operator has explicitly opted in via allowUnvalidatedSmallCluster (e.g.
793
+ // single-node / local dev knowingly running below the floor).
794
+ const admit = this.cfg.allowUnvalidatedSmallCluster ?? false;
795
+ log('cluster-tx:small-cluster-no-confident-estimate', {
796
+ localSize,
797
+ reason: 'no-confident-network-size-estimate',
798
+ admit
799
+ });
800
+ return admit;
801
+ }
802
+
803
+ /**
804
+ * Collects promises from all peers in the cluster
805
+ */
806
+ private async collectPromises(peers: ClusterPeers, record: ClusterRecord): Promise<{ record: ClusterRecord }> {
807
+ const peerIds = Object.keys(peers);
808
+ const summary: ClusterLogPeerOutcome[] = [];
809
+ if (verbose) {
810
+ const peerDetail = peerIds.map(id => ({
811
+ id: id.substring(0, 12),
812
+ addrs: peers[id]?.multiaddrs?.length ?? 0
813
+ }));
814
+ log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
815
+ }
816
+ // For each peer, create a client and request a promise. A remote promise rides
817
+ // a libp2p stream that a relayed (limited) connection can reset transiently, so
818
+ // each remote request gets `promiseImmediateRetries` in-line re-attempts before
819
+ // it counts as a failure — without this a single relayed reset drops the peer and
820
+ // sinks super-majority (the commit broadcast already has the same guard).
821
+ const promiseRequests = peerIds.map(peerIdStr => {
822
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
823
+ log('cluster-tx:promise-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
824
+ return new Pending(this.updateMember(peerIdStr, record, this.promiseImmediateRetries, 'promise'));
825
+ });
826
+
827
+ // Wait for all promises to complete
828
+ const results = await Promise.all(promiseRequests.map((p, idx) => p.result().then(res => {
829
+ const peerIdStr = peerIds[idx]!;
830
+ log('cluster-tx:promise-response', {
831
+ messageHash: record.messageHash,
832
+ peerId: peerIdStr,
833
+ success: true,
834
+ returnedPromises: Object.keys(res.promises ?? {}),
835
+ returnedCommits: Object.keys(res.commits ?? {})
836
+ });
837
+ summary.push({ peerId: peerIdStr, success: true });
838
+ return res;
839
+ }).catch(err => {
840
+ const peerIdStr = peerIds[idx]!;
841
+ log('cluster-tx:promise-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
842
+ summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
843
+ this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `promise:${record.messageHash}`);
844
+ return null;
845
+ })));
846
+ const successes = summary.filter(entry => entry.success).map(entry => entry.peerId);
847
+ const failures = summary.filter(entry => !entry.success);
848
+ log('cluster-tx:promise-summary', {
849
+ messageHash: record.messageHash,
850
+ successes,
851
+ failures
852
+ });
853
+
854
+ log('cluster-tx:promise-merge-begin', {
855
+ messageHash: record.messageHash,
856
+ initialPromises: Object.keys(record.promises ?? {}),
857
+ transactionsKeys: Array.from(this.transactions.keys()),
858
+ hasTransaction: this.transactions.has(record.messageHash)
859
+ });
860
+
861
+ // Merge all promises into the record
862
+ for (const result of results.filter(Boolean) as ClusterRecord[]) {
863
+ log('cluster-tx:promise-merge-input', {
864
+ messageHash: record.messageHash,
865
+ resultFrom: Object.keys(result.promises ?? {}),
866
+ recordBefore: Object.keys(record.promises ?? {})
867
+ });
868
+ const resultPromises = Object.keys(result.promises ?? {});
869
+ log('cluster-tx:promise-merge-result', {
870
+ messageHash: record.messageHash,
871
+ peerPromises: resultPromises
872
+ });
873
+ if (typeof record.suggestedClusterSize === 'number' && typeof result.suggestedClusterSize === 'number') {
874
+ const expected = result.suggestedClusterSize;
875
+ const actual = Object.keys(peers).length;
876
+ const maxDiff = Math.ceil(Math.max(1, expected * this.cfg.clusterSizeTolerance));
877
+ if (Math.abs(actual - expected) > maxDiff) {
878
+ log('cluster-tx:size-variance', { expected, actual, tolerance: this.cfg.clusterSizeTolerance });
879
+ }
880
+ }
881
+ record.promises = { ...record.promises, ...result.promises };
882
+ log('cluster-tx:promise-merge-after', {
883
+ messageHash: record.messageHash,
884
+ mergedPromises: Object.keys(record.promises ?? {})
885
+ });
886
+ }
887
+ log('cluster-tx:promise-merge', {
888
+ messageHash: record.messageHash,
889
+ mergedPromises: Object.keys(record.promises ?? {})
890
+ });
891
+ log('cluster-tx:promise-merge-end', {
892
+ messageHash: record.messageHash,
893
+ finalPromises: Object.keys(record.promises ?? {}),
894
+ transactionsEntry: this.transactions.get(record.messageHash)
895
+ });
896
+ this.updateTransactionRecord(record, 'after-promises');
897
+ return { record };
898
+ }
899
+
900
+ /**
901
+ * Commits the transaction to all peers in the cluster
902
+ */
903
+ private async commitTransaction(record: ClusterRecord): Promise<ClusterRecord> {
904
+ // For each peer, create a client and send the commit
905
+ const peerIds = Object.keys(record.peers);
906
+ const summary: ClusterLogPeerOutcome[] = [];
907
+ if (verbose) {
908
+ const peerDetail = peerIds.map(id => ({
909
+ id: id.substring(0, 12),
910
+ addrs: record.peers[id]?.multiaddrs?.length ?? 0
911
+ }));
912
+ log('cluster-tx:commit-peers', { messageHash: record.messageHash, peers: peerDetail });
913
+ }
914
+ // Send the record with promises to all peers
915
+ // Each peer will add its own commit signature
916
+ const commitPayload = {
917
+ ...record
918
+ };
919
+ // No per-peer immediate retry here: a commit-collection failure is recovered
920
+ // downstream by broadcastMergedRecord's in-line retry and the scheduled
921
+ // commit-retry timer. (The promise phase has no such backstop, which is why
922
+ // collectPromises gets the immediate retry instead.)
923
+ const commitRequests = peerIds.map(peerIdStr => {
924
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
925
+ log('cluster-tx:commit-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
926
+ const promise = isLocal
927
+ ? this.localCluster!.update(commitPayload)
928
+ : this.createClusterClient(peerIdFromString(peerIdStr)).update(commitPayload);
929
+ return new Pending(promise);
930
+ });
931
+
932
+ // Wait for all commits to complete
933
+ const results = await Promise.all(commitRequests.map((p, idx) => p.result().then(res => {
934
+ const peerIdStr = peerIds[idx]!;
935
+ log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: true });
936
+ summary.push({ peerId: peerIdStr, success: true });
937
+ return res;
938
+ }).catch(err => {
939
+ const peerIdStr = peerIds[idx]!;
940
+ log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
941
+ summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
942
+ this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `commit:${record.messageHash}`);
943
+ return null;
944
+ })));
945
+ const commitSuccesses = summary.filter(entry => entry.success).map(entry => entry.peerId);
946
+ const commitFailures = summary.filter(entry => !entry.success);
947
+ log('cluster-tx:commit-summary', {
948
+ messageHash: record.messageHash,
949
+ successes: commitSuccesses,
950
+ failures: commitFailures
951
+ });
952
+ log('cluster-tx:commit-merge-begin', {
953
+ messageHash: record.messageHash,
954
+ initialCommits: Object.keys(record.commits ?? {}),
955
+ transactionsEntry: this.transactions.get(record.messageHash)
956
+ });
957
+
958
+ // A member can reach consensus during THIS round rather than during the broadcast below (a
959
+ // record that already carries commits — a retried delivery), so its apply verdicts arrive on
960
+ // these responses. Collect both; the broadcast's copy wins on overlap, being the later of the two.
961
+ mergeApplyOutcomes(record, collectApplyOutcomes(results.map((response, idx) => ({ peerId: peerIds[idx]!, response }))));
962
+
963
+ // Merge all commits into the record
964
+ for (const result of results.filter(Boolean) as ClusterRecord[]) {
965
+ log('cluster-tx:commit-merge-input', {
966
+ messageHash: record.messageHash,
967
+ resultFrom: Object.keys(result.commits ?? {}),
968
+ recordBefore: Object.keys(record.commits ?? {})
969
+ });
970
+ log('cluster-tx:commit-merge-result', {
971
+ messageHash: record.messageHash,
972
+ peerCommits: Object.keys(result.commits ?? {})
973
+ });
974
+ record.commits = { ...record.commits, ...result.commits };
975
+ log('cluster-tx:commit-merge-after', {
976
+ messageHash: record.messageHash,
977
+ mergedCommits: Object.keys(record.commits ?? {})
978
+ });
979
+ }
980
+ log('cluster-tx:commit-merge', {
981
+ messageHash: record.messageHash,
982
+ mergedCommits: Object.keys(record.commits ?? {})
983
+ });
984
+ log('cluster-tx:commit-merge-end', {
985
+ messageHash: record.messageHash,
986
+ finalCommits: Object.keys(record.commits ?? {}),
987
+ transactionsEntry: this.transactions.get(record.messageHash)
988
+ });
989
+ this.updateTransactionRecord(record, 'after-commit');
990
+
991
+ // Check for simple majority (>50%) - this proves commitment
992
+ const peerCount = Object.keys(record.peers).length;
993
+ const simpleMajority = Math.floor(peerCount * this.cfg.simpleMajorityThreshold) + 1;
994
+ const commitCount = Object.keys(record.commits).length;
995
+
996
+ if (commitCount >= simpleMajority) {
997
+ log('cluster-tx:commit-majority-reached', {
998
+ messageHash: record.messageHash,
999
+ commitCount,
1000
+ simpleMajority,
1001
+ peerCount,
1002
+ threshold: this.cfg.simpleMajorityThreshold
1003
+ });
1004
+ // Broadcast the merged record (with all commit signatures) to ALL peers
1005
+ // so each peer can independently reach consensus and execute the operations.
1006
+ // Without this, only the coordinator's local cluster executes — remote peers
1007
+ // never see enough commits to reach consensus on their own.
1008
+ const { failures: broadcastFailures, applyOutcomes } = await this.broadcastMergedRecord(record, peerIds);
1009
+ mergeApplyOutcomes(record, applyOutcomes);
1010
+ if (broadcastFailures.length > 0) {
1011
+ this.scheduleCommitRetry(record.messageHash, record, broadcastFailures);
1012
+ } else {
1013
+ this.clearRetry(record.messageHash);
1014
+ }
1015
+ } else {
1016
+ const missingPeers = commitFailures.map(entry => entry.peerId);
1017
+ if (missingPeers.length > 0) {
1018
+ this.scheduleCommitRetry(record.messageHash, record, missingPeers);
1019
+ } else {
1020
+ this.clearRetry(record.messageHash);
1021
+ }
1022
+ }
1023
+ return record;
1024
+ }
1025
+
1026
+ /**
1027
+ * Broadcast the merged commit record to every peer, with `commitBroadcastImmediateRetries`
1028
+ * in-line re-attempts per peer before giving up. The libp2p connection used during
1029
+ * the prior commit phase is typically still warm, so a single immediate retry recovers
1030
+ * most transient stream errors without falling back to the scheduled retry timer.
1031
+ * Local cluster is invoked exactly once — local failures are fatal, not transient.
1032
+ *
1033
+ * **Delivery order is load-bearing: this node's own member first, awaited, then the remote
1034
+ * members in parallel.** This broadcast is where members apply the commit, and a member that is
1035
+ * behind (it never saw the pend, or holds no base for the block) reconciles the committed
1036
+ * revision from `record.peers` DURING its apply. The coordinator's own member is the one peer
1037
+ * guaranteed to hold the revision by then — provided it has actually applied, which a single
1038
+ * `Promise.all` over every peer did not guarantee: the remote members' reconciles raced the
1039
+ * local apply and found no holder. Its copy also carries the cohort's commit proof
1040
+ * (`buildBlockCommitProof`), which `createReconcileBlock` accepts from a single holder, so a
1041
+ * whole cohort of behind members can heal from it. The cost is one in-process apply before the
1042
+ * network fan-out; no extra round trip. The commit round in `commitTransaction` may stay
1043
+ * parallel: on the first pass the record it carries has no commit signatures yet, so no member
1044
+ * can reach consensus (and apply) there. The scheduled retry (`retryCommits`) does re-send a
1045
+ * record that already carries them, in parallel — but by then this node's member applied in the
1046
+ * first broadcast unless it was itself among the failed deliveries, which is the retry residual
1047
+ * documented on `executeClusterTransaction`. A coordinator outside `record.peers` is not a
1048
+ * reconcile target and gains nothing from this ordering; the durability gate in
1049
+ * `CoordinatorRepo.commit` is what makes that shape refuse rather than acknowledge.
1050
+ *
1051
+ * NOTE: when the coordinating member is ITSELF behind (it never saw the pend), its reconcile
1052
+ * runs here before any remote member has applied, finds no holder, and reports not-durable; the
1053
+ * remote members then apply and may carry the majority on their own. Fine while the coordinator
1054
+ * ordinarily saw the pend; if coordinators are routinely picked after the pend phase, deliver
1055
+ * local-first only when the local member holds the pend, or reconcile it once more afterwards.
1056
+ */
1057
+ private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[]; applyOutcomes?: ClusterRecord['applyOutcomes'] }> {
1058
+ const deliver = async (peerIdStr: string) => {
1059
+ try {
1060
+ const response = await this.updateMember(peerIdStr, record, this.commitBroadcastImmediateRetries, 'commit-broadcast');
1061
+ return { peerId: peerIdStr, success: true as const, response };
1062
+ } catch (err) {
1063
+ log('cluster-tx:consensus-broadcast-error', {
1064
+ messageHash: record.messageHash,
1065
+ peerId: peerIdStr,
1066
+ error: err instanceof Error ? err.message : String(err)
1067
+ });
1068
+ return { peerId: peerIdStr, success: false as const, response: undefined };
1069
+ }
1070
+ };
1071
+ const selfId = this.localCluster?.peerId.toString();
1072
+ const localFirst = peerIds.filter(id => id === selfId);
1073
+ const remote = peerIds.filter(id => id !== selfId);
1074
+ const localResults = await Promise.all(localFirst.map(deliver));
1075
+ const remoteResults = await Promise.all(remote.map(deliver));
1076
+ const results = [...localResults, ...remoteResults];
1077
+ const failures = results.filter(r => !r.success).map(r => r.peerId);
1078
+ // This broadcast is where members actually apply the operations, so their responses carry the
1079
+ // only report the coordinator ever gets of what each member's OWN storage said. Collecting it
1080
+ // here is what lets a pend refused by a non-coordinating member reach the writer as a conflict
1081
+ // instead of the fabricated success that used to fork the block.
1082
+ //
1083
+ // Each peer's entry is taken from that peer's OWN response and re-keyed under the peer we
1084
+ // asked, so a member cannot report an outcome on another member's behalf by echoing a record
1085
+ // full of entries. Unsigned and advisory either way — see ClusterRecord.applyOutcomes.
1086
+ const applyOutcomes = collectApplyOutcomes(results);
1087
+ return { failures, ...(applyOutcomes === undefined ? {} : { applyOutcomes }) };
1088
+ }
1089
+
1090
+ /**
1091
+ * Fire-and-forget replay of an abandoned transaction's record to every peer in its cohort.
1092
+ *
1093
+ * Called only where the record itself proves the transaction is dead (enough signed rejections that
1094
+ * super-majority is unreachable). Each member re-derives `TransactionPhase.Rejected` from the votes
1095
+ * it verifies and drops the entry from its own reservation table, freeing the blocks immediately
1096
+ * instead of after its 2 s staleness window. No new message type and no wire-format change — this is
1097
+ * the same `update()` every other phase uses.
1098
+ *
1099
+ * Never awaited into the caller's throw and never rethrows: an abandonment must not turn into a
1100
+ * *different* failure, and the staleness sweep remains the backstop if delivery fails.
1101
+ */
1102
+ private broadcastAbandonment(record: ClusterRecord, reason: string): void {
1103
+ const peerIds = Object.keys(record.peers);
1104
+ log('cluster-tx:abandon-broadcast', { messageHash: record.messageHash, reason, peerIds });
1105
+ void Promise.all(peerIds.map(async peerIdStr => {
1106
+ try {
1107
+ await this.updateMember(peerIdStr, record, 0, 'abandon-broadcast');
1108
+ } catch (err) {
1109
+ log('cluster-tx:abandon-broadcast-error', {
1110
+ messageHash: record.messageHash,
1111
+ peerId: peerIdStr,
1112
+ error: err instanceof Error ? err.message : String(err)
1113
+ });
1114
+ }
1115
+ }));
1116
+ }
1117
+
1118
+ private updateTransactionRecord(record: ClusterRecord, stage: string): void {
1119
+ const state = this.transactions.get(record.messageHash);
1120
+ if (!state) {
1121
+ log('cluster-tx:transaction-update-miss', { messageHash: record.messageHash, stage });
1122
+ return;
1123
+ }
1124
+ state.record = { ...record };
1125
+ state.lastUpdate = this.now();
1126
+ log('cluster-tx:transaction-update', {
1127
+ messageHash: record.messageHash,
1128
+ stage,
1129
+ promises: Object.keys(record.promises ?? {}),
1130
+ commits: Object.keys(record.commits ?? {})
1131
+ });
1132
+ }
1133
+
1134
+ private scheduleCommitRetry(messageHash: string, _record: ClusterRecord, missingPeers: string[]): void {
1135
+ const state = this.transactions.get(messageHash);
1136
+ if (!state) {
1137
+ return;
1138
+ }
1139
+ const existing = state.retry;
1140
+ const nextAttempt = (existing?.attempt ?? 0) + 1;
1141
+ if (nextAttempt > this.retryMaxAttempts) {
1142
+ log('cluster-tx:retry-abort', { messageHash, missingPeers });
1143
+ return;
1144
+ }
1145
+ if (missingPeers.length === 0) {
1146
+ this.clearRetry(messageHash);
1147
+ return;
1148
+ }
1149
+ const pendingPeers = new Set(missingPeers);
1150
+ const baseInterval = existing ? Math.min(existing.intervalMs * this.retryBackoffFactor, this.retryMaxIntervalMs) : this.retryInitialIntervalMs;
1151
+ existing?.cancel?.();
1152
+ const cancel = this.setTimer(() => {
1153
+ void this.retryCommits(messageHash);
1154
+ }, baseInterval);
1155
+ state.retry = {
1156
+ pendingPeers,
1157
+ attempt: nextAttempt,
1158
+ intervalMs: baseInterval,
1159
+ cancel
1160
+ };
1161
+ this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
1162
+ pendingPeers: Array.from(pendingPeers),
1163
+ attempt: nextAttempt,
1164
+ intervalMs: baseInterval
1165
+ });
1166
+ log('cluster-tx:retry-scheduled', { messageHash, attempt: nextAttempt, missingPeers, delayMs: baseInterval });
1167
+ }
1168
+
1169
+ private async retryCommits(messageHash: string): Promise<void> {
1170
+ const state = this.transactions.get(messageHash);
1171
+ if (!state?.retry) {
1172
+ return;
1173
+ }
1174
+ const { pendingPeers, attempt } = state.retry;
1175
+ if (pendingPeers.size === 0) {
1176
+ this.clearRetry(messageHash);
1177
+ return;
1178
+ }
1179
+ const peerIds = Array.from(pendingPeers);
1180
+ const record = state.record;
1181
+ log('cluster-tx:retry-start', { messageHash, attempt, peerIds });
1182
+ const results = await Promise.all(peerIds.map(async peerIdStr => {
1183
+ const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
1184
+ const payload: ClusterRecord = {
1185
+ ...record,
1186
+ commits: record.commits
1187
+ };
1188
+ try {
1189
+ const res = isLocal
1190
+ ? await this.localCluster!.update(payload)
1191
+ : await this.createClusterClient(peerIdFromString(peerIdStr)).update(payload);
1192
+ state.record.commits = { ...state.record.commits, ...res.commits };
1193
+ return { peerId: peerIdStr, success: true as const };
1194
+ } catch (err) {
1195
+ return {
1196
+ peerId: peerIdStr,
1197
+ success: false as const,
1198
+ error: err instanceof Error ? err.message : String(err)
1199
+ };
1200
+ }
1201
+ }));
1202
+ const successes = results.filter(r => r.success).map(r => r.peerId);
1203
+ const failures = results.filter(r => !r.success);
1204
+ for (const peerId of successes) {
1205
+ pendingPeers.delete(peerId);
1206
+ }
1207
+ log('cluster-tx:retry-complete', { messageHash, attempt, successes, failures });
1208
+ if (pendingPeers.size === 0) {
1209
+ log('cluster-tx:retry-finished', { messageHash });
1210
+ this.clearRetry(messageHash);
1211
+ return;
1212
+ }
1213
+ if (!this.transactions.has(messageHash)) {
1214
+ return;
1215
+ }
1216
+ this.scheduleCommitRetry(messageHash, state.record, Array.from(pendingPeers));
1217
+ }
1218
+
1219
+ private clearRetry(messageHash: string): void {
1220
+ const state = this.transactions.get(messageHash);
1221
+ if (!state?.retry) {
1222
+ return;
1223
+ }
1224
+ state.retry.cancel?.();
1225
+ state.retry = undefined;
1226
+ // Clean up the transaction after retry is complete
1227
+ this.setTimer(() => {
1228
+ this.transactions.delete(messageHash);
1229
+ this.deleteCoordinatorState(messageHash);
1230
+ log('cluster-tx:transaction-remove', {
1231
+ messageHash,
1232
+ remaining: Array.from(this.transactions.keys())
1233
+ });
1234
+ }, 100);
1235
+ }
1236
+
1237
+ /** Fire-and-forget persist — errors are logged, never thrown. */
1238
+ private persistCoordinatorState(
1239
+ messageHash: string,
1240
+ record: ClusterRecord,
1241
+ phase: 'promising' | 'committing' | 'broadcasting',
1242
+ retryState?: { pendingPeers: string[]; attempt: number; intervalMs: number }
1243
+ ): void {
1244
+ if (!this.stateStore) return;
1245
+ this.stateStore.saveCoordinatorState(messageHash, {
1246
+ messageHash,
1247
+ record,
1248
+ lastUpdate: this.now(),
1249
+ phase,
1250
+ retryState
1251
+ }).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
1252
+ }
1253
+
1254
+ /** Fire-and-forget delete — errors are logged, never thrown. */
1255
+ private deleteCoordinatorState(messageHash: string): void {
1256
+ if (!this.stateStore) return;
1257
+ this.stateStore.deleteCoordinatorState(messageHash)
1258
+ .catch(err => log('cluster-tx:persist-delete-error', { messageHash, error: (err as Error).message }));
1259
+ }
1260
+
1261
+ /**
1262
+ * Recover coordinator transactions from persistent store after a restart.
1263
+ * Called during node startup, before accepting new requests.
1264
+ */
1265
+ async recoverTransactions(): Promise<void> {
1266
+ if (!this.stateStore) return;
1267
+ const states = await this.stateStore.getAllCoordinatorStates();
1268
+ for (const state of states) {
1269
+ const { messageHash } = state;
1270
+ // Expired — clean up
1271
+ if (state.record.message.expiration && state.record.message.expiration < this.now()) {
1272
+ log('cluster-tx:recovery-expired', { messageHash });
1273
+ await this.stateStore.deleteCoordinatorState(messageHash);
1274
+ continue;
1275
+ }
1276
+ // Broadcasting phase with retry state — resume retries
1277
+ if (state.phase === 'broadcasting' && state.retryState) {
1278
+ log('cluster-tx:recovery-resume-broadcast', { messageHash, attempt: state.retryState.attempt });
1279
+ const pending = new Pending(Promise.resolve(state.record));
1280
+ const txState: ClusterTransactionState = {
1281
+ messageHash,
1282
+ record: state.record,
1283
+ pending,
1284
+ lastUpdate: state.lastUpdate
1285
+ };
1286
+ this.transactions.set(messageHash, txState);
1287
+ // Schedule retry from where we left off
1288
+ this.scheduleCommitRetry(messageHash, state.record, state.retryState.pendingPeers);
1289
+ continue;
1290
+ }
1291
+ // Promising or committing — cannot resume (caller context is gone)
1292
+ log('cluster-tx:recovery-stale', { messageHash, phase: state.phase });
1293
+ await this.stateStore.deleteCoordinatorState(messageHash);
1294
+ }
1295
+ }
1296
+ }