@optimystic/db-p2p 1.0.0-beta.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/block-transfer-service.d.ts +73 -1
- package/dist/src/cluster/block-transfer-service.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer-service.js +70 -3
- package/dist/src/cluster/block-transfer-service.js.map +1 -1
- package/dist/src/cluster/block-transfer.d.ts +16 -1
- package/dist/src/cluster/block-transfer.d.ts.map +1 -1
- package/dist/src/cluster/block-transfer.js +73 -72
- package/dist/src/cluster/block-transfer.js.map +1 -1
- package/dist/src/cluster/client.d.ts +4 -6
- package/dist/src/cluster/client.d.ts.map +1 -1
- package/dist/src/cluster/client.js +8 -13
- package/dist/src/cluster/client.js.map +1 -1
- package/dist/src/cluster/cluster-policy.d.ts +2 -1
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +172 -17
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +553 -149
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/commit-proof.js +2 -2
- package/dist/src/cluster/race-resolution.d.ts +51 -7
- package/dist/src/cluster/race-resolution.d.ts.map +1 -1
- package/dist/src/cluster/race-resolution.js +71 -7
- package/dist/src/cluster/race-resolution.js.map +1 -1
- package/dist/src/cluster/rebalance-monitor.d.ts +67 -5
- package/dist/src/cluster/rebalance-monitor.d.ts.map +1 -1
- package/dist/src/cluster/rebalance-monitor.js +120 -26
- package/dist/src/cluster/rebalance-monitor.js.map +1 -1
- package/dist/src/cluster/spread-on-churn.d.ts.map +1 -1
- package/dist/src/cluster/spread-on-churn.js +34 -47
- package/dist/src/cluster/spread-on-churn.js.map +1 -1
- package/dist/src/cohort-topic/host.d.ts.map +1 -1
- package/dist/src/cohort-topic/host.js +8 -6
- package/dist/src/cohort-topic/host.js.map +1 -1
- package/dist/src/cohort-topic/membership-source.js +1 -1
- package/dist/src/cohort-topic/membership-source.js.map +1 -1
- package/dist/src/cohort-topic/peer-codec.d.ts.map +1 -1
- package/dist/src/cohort-topic/peer-codec.js +5 -2
- package/dist/src/cohort-topic/peer-codec.js.map +1 -1
- package/dist/src/cohort-topic/stream-util.d.ts +24 -3
- package/dist/src/cohort-topic/stream-util.d.ts.map +1 -1
- package/dist/src/cohort-topic/stream-util.js +36 -3
- package/dist/src/cohort-topic/stream-util.js.map +1 -1
- package/dist/src/cohort-topic/topic-router.d.ts +9 -0
- package/dist/src/cohort-topic/topic-router.d.ts.map +1 -1
- package/dist/src/cohort-topic/topic-router.js +12 -2
- package/dist/src/cohort-topic/topic-router.js.map +1 -1
- package/dist/src/dispute/client.d.ts.map +1 -1
- package/dist/src/dispute/client.js +15 -3
- package/dist/src/dispute/client.js.map +1 -1
- package/dist/src/index.d.ts +5 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +5 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +78 -17
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +241 -147
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +35 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +151 -26
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/libp2p-node-rn.d.ts +2 -2
- package/dist/src/libp2p-node-rn.d.ts.map +1 -1
- package/dist/src/libp2p-node-rn.js.map +1 -1
- package/dist/src/libp2p-node.d.ts +2 -2
- package/dist/src/libp2p-node.d.ts.map +1 -1
- package/dist/src/libp2p-node.js.map +1 -1
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/logger.js +19 -7
- package/dist/src/logger.js.map +1 -1
- package/dist/src/matchmaking/query-transport.d.ts +3 -4
- package/dist/src/matchmaking/query-transport.d.ts.map +1 -1
- package/dist/src/matchmaking/query-transport.js +9 -9
- package/dist/src/matchmaking/query-transport.js.map +1 -1
- package/dist/src/network/network-manager-service.d.ts +10 -5
- package/dist/src/network/network-manager-service.d.ts.map +1 -1
- package/dist/src/network/network-manager-service.js +11 -2
- package/dist/src/network/network-manager-service.js.map +1 -1
- package/dist/src/network/relay-reservation.d.ts +170 -0
- package/dist/src/network/relay-reservation.d.ts.map +1 -0
- package/dist/src/network/relay-reservation.js +527 -0
- package/dist/src/network/relay-reservation.js.map +1 -0
- package/dist/src/pend-validation.d.ts +3 -2
- package/dist/src/pend-validation.d.ts.map +1 -1
- package/dist/src/pend-validation.js +3 -2
- package/dist/src/pend-validation.js.map +1 -1
- package/dist/src/protocol-client.d.ts +8 -1
- package/dist/src/protocol-client.d.ts.map +1 -1
- package/dist/src/protocol-client.js +11 -0
- package/dist/src/protocol-client.js.map +1 -1
- package/dist/src/reactivity/forwarder-host.d.ts +2 -1
- package/dist/src/reactivity/forwarder-host.d.ts.map +1 -1
- package/dist/src/reactivity/forwarder-host.js +2 -1
- package/dist/src/reactivity/forwarder-host.js.map +1 -1
- package/dist/src/reactivity/origination-manager.d.ts +3 -3
- package/dist/src/reactivity/origination-manager.js +3 -3
- package/dist/src/reactivity/recover-transport.d.ts +22 -14
- package/dist/src/reactivity/recover-transport.d.ts.map +1 -1
- package/dist/src/reactivity/recover-transport.js +32 -19
- package/dist/src/reactivity/recover-transport.js.map +1 -1
- package/dist/src/reactivity/subscription-manager.d.ts +2 -2
- package/dist/src/reactivity/topic-bytes.d.ts +6 -7
- package/dist/src/reactivity/topic-bytes.d.ts.map +1 -1
- package/dist/src/reactivity/topic-bytes.js +8 -9
- package/dist/src/reactivity/topic-bytes.js.map +1 -1
- package/dist/src/repo/client.d.ts +5 -2
- package/dist/src/repo/client.d.ts.map +1 -1
- package/dist/src/repo/client.js +49 -40
- package/dist/src/repo/client.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +119 -5
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +181 -29
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +259 -76
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +716 -328
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/repo/i-under-replication-ledger.d.ts +87 -0
- package/dist/src/repo/i-under-replication-ledger.d.ts.map +1 -0
- package/dist/src/repo/i-under-replication-ledger.js +2 -0
- package/dist/src/repo/i-under-replication-ledger.js.map +1 -0
- package/dist/src/repo/kv-under-replication-ledger.d.ts +91 -0
- package/dist/src/repo/kv-under-replication-ledger.d.ts.map +1 -0
- package/dist/src/repo/kv-under-replication-ledger.js +287 -0
- package/dist/src/repo/kv-under-replication-ledger.js.map +1 -0
- package/dist/src/repo/responsibility.d.ts +37 -0
- package/dist/src/repo/responsibility.d.ts.map +1 -0
- package/dist/src/repo/responsibility.js +37 -0
- package/dist/src/repo/responsibility.js.map +1 -0
- package/dist/src/repo/service.d.ts +44 -15
- package/dist/src/repo/service.d.ts.map +1 -1
- package/dist/src/repo/service.js +64 -27
- package/dist/src/repo/service.js.map +1 -1
- package/dist/src/repo/stuck-reservation.d.ts +127 -0
- package/dist/src/repo/stuck-reservation.d.ts.map +1 -0
- package/dist/src/repo/stuck-reservation.js +179 -0
- package/dist/src/repo/stuck-reservation.js.map +1 -0
- package/dist/src/repo/under-replication-drain.d.ts +208 -0
- package/dist/src/repo/under-replication-drain.d.ts.map +1 -0
- package/dist/src/repo/under-replication-drain.js +461 -0
- package/dist/src/repo/under-replication-drain.js.map +1 -0
- package/dist/src/rn.d.ts +5 -0
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +5 -0
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/block-latch.d.ts +3 -2
- package/dist/src/storage/block-latch.d.ts.map +1 -1
- package/dist/src/storage/block-latch.js +7 -4
- package/dist/src/storage/block-latch.js.map +1 -1
- package/dist/src/storage/block-storage.d.ts +45 -3
- package/dist/src/storage/block-storage.d.ts.map +1 -1
- package/dist/src/storage/block-storage.js +156 -9
- package/dist/src/storage/block-storage.js.map +1 -1
- package/dist/src/storage/i-block-storage.d.ts +57 -5
- package/dist/src/storage/i-block-storage.d.ts.map +1 -1
- package/dist/src/storage/i-block-storage.js.map +1 -1
- package/dist/src/storage/pending-claim.d.ts +167 -0
- package/dist/src/storage/pending-claim.d.ts.map +1 -0
- package/dist/src/storage/pending-claim.js +156 -0
- package/dist/src/storage/pending-claim.js.map +1 -0
- package/dist/src/storage/raw-store-codec.d.ts +13 -0
- package/dist/src/storage/raw-store-codec.d.ts.map +1 -1
- package/dist/src/storage/raw-store-codec.js +24 -3
- package/dist/src/storage/raw-store-codec.js.map +1 -1
- package/dist/src/storage/restoration-coordinator.d.ts.map +1 -1
- package/dist/src/storage/restoration-coordinator.js +2 -1
- package/dist/src/storage/restoration-coordinator.js.map +1 -1
- package/dist/src/storage/ring-shift-coordinator.d.ts.map +1 -1
- package/dist/src/storage/ring-shift-coordinator.js +3 -3
- package/dist/src/storage/ring-shift-coordinator.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts +137 -5
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +273 -68
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/storage/struct.d.ts +58 -0
- package/dist/src/storage/struct.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +92 -11
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +246 -103
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/package.json +2 -2
- package/readme.md +27 -3
- package/src/cluster/block-transfer-service.ts +132 -4
- package/src/cluster/block-transfer.ts +79 -84
- package/src/cluster/client.ts +115 -119
- package/src/cluster/cluster-policy.ts +2 -1
- package/src/cluster/cluster-repo.ts +3115 -2671
- package/src/cluster/commit-proof.ts +2 -2
- package/src/cluster/race-resolution.ts +76 -7
- package/src/cluster/rebalance-monitor.ts +677 -539
- package/src/cluster/spread-on-churn.ts +34 -54
- package/src/cohort-topic/host.ts +8 -6
- package/src/cohort-topic/membership-source.ts +1 -1
- package/src/cohort-topic/peer-codec.ts +34 -31
- package/src/cohort-topic/stream-util.ts +39 -4
- package/src/cohort-topic/topic-router.ts +12 -2
- package/src/dispute/client.ts +17 -7
- package/src/index.ts +64 -59
- package/src/libp2p-key-network.ts +1377 -1253
- package/src/libp2p-node-base.ts +186 -27
- package/src/libp2p-node-rn.ts +31 -30
- package/src/libp2p-node.ts +37 -36
- package/src/logger.ts +20 -7
- package/src/matchmaking/query-transport.ts +9 -9
- package/src/network/network-manager-service.ts +16 -5
- package/src/network/relay-reservation.ts +636 -0
- package/src/pend-validation.ts +3 -2
- package/src/protocol-client.ts +13 -1
- package/src/reactivity/forwarder-host.ts +2 -1
- package/src/reactivity/origination-manager.ts +3 -3
- package/src/reactivity/recover-transport.ts +39 -22
- package/src/reactivity/subscription-manager.ts +2 -2
- package/src/reactivity/topic-bytes.ts +35 -37
- package/src/repo/client.ts +172 -173
- package/src/repo/cluster-coordinator.ts +1296 -1113
- package/src/repo/coordinator-repo.ts +3087 -2687
- package/src/repo/i-under-replication-ledger.ts +89 -0
- package/src/repo/kv-under-replication-ledger.ts +302 -0
- package/src/repo/responsibility.ts +48 -0
- package/src/repo/service.ts +76 -39
- package/src/repo/stuck-reservation.ts +224 -0
- package/src/repo/under-replication-drain.ts +582 -0
- package/src/rn.ts +74 -69
- package/src/storage/block-latch.ts +7 -4
- package/src/storage/block-storage.ts +161 -10
- package/src/storage/i-block-storage.ts +60 -5
- package/src/storage/pending-claim.ts +200 -0
- package/src/storage/raw-store-codec.ts +26 -3
- package/src/storage/restoration-coordinator.ts +212 -211
- package/src/storage/ring-shift-coordinator.ts +271 -271
- package/src/storage/storage-repo.ts +311 -73
- package/src/storage/struct.ts +58 -0
- package/src/testing/mesh-harness.ts +329 -115
|
@@ -1,1113 +1,1296 @@
|
|
|
1
|
-
import { peerIdFromString } from "@libp2p/peer-id";
|
|
2
|
-
import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult, StaleFailure } from "@optimystic/db-core";
|
|
3
|
-
import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, isConflictFailure, membershipDigest } from "@optimystic/db-core";
|
|
4
|
-
import { Pending } from "@optimystic/db-core";
|
|
5
|
-
import type { PeerId } from "@libp2p/interface";
|
|
6
|
-
import { createLogger, verbose } from '../logger.js'
|
|
7
|
-
import type { ClusterLogPeerOutcome } from './types.js'
|
|
8
|
-
import type { FretService } from "p2p-fret";
|
|
9
|
-
import type { IPeerReputation } from "../reputation/types.js";
|
|
10
|
-
import { PenaltyReason } from "../reputation/types.js";
|
|
11
|
-
import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
collected
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
* `
|
|
53
|
-
*
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
* `
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
* the
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
*
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
*/
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
/**
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
export
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
*
|
|
224
|
-
*
|
|
225
|
-
*
|
|
226
|
-
*
|
|
227
|
-
*
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
* promise
|
|
231
|
-
*
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
//
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
//
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
//
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
//
|
|
547
|
-
//
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
peerCount
|
|
577
|
-
}
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
const
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
}
|
|
635
|
-
|
|
636
|
-
//
|
|
637
|
-
//
|
|
638
|
-
//
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
//
|
|
664
|
-
//
|
|
665
|
-
//
|
|
666
|
-
//
|
|
667
|
-
//
|
|
668
|
-
|
|
669
|
-
const
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
}
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
}
|
|
715
|
-
const
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
}
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
}
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
}
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
}
|
|
1
|
+
import { peerIdFromString } from "@libp2p/peer-id";
|
|
2
|
+
import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult, StaleFailure } from "@optimystic/db-core";
|
|
3
|
+
import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, isConflictFailure, membershipDigest, routingKeyForBlock } from "@optimystic/db-core";
|
|
4
|
+
import { Pending } from "@optimystic/db-core";
|
|
5
|
+
import type { PeerId } from "@libp2p/interface";
|
|
6
|
+
import { createLogger, verbose } from '../logger.js'
|
|
7
|
+
import type { ClusterLogPeerOutcome } from './types.js'
|
|
8
|
+
import type { FretService } from "p2p-fret";
|
|
9
|
+
import type { IPeerReputation } from "../reputation/types.js";
|
|
10
|
+
import { PenaltyReason } from "../reputation/types.js";
|
|
11
|
+
import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
|
|
12
|
+
import { ResponsibilityRefusalError } from "./responsibility.js";
|
|
13
|
+
|
|
14
|
+
const log = createLogger('cluster')
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Pick each peer's OWN {@link ClusterRecord.applyOutcomes} entry out of the record that peer answered
|
|
18
|
+
* with, and key it under the peer we actually asked.
|
|
19
|
+
*
|
|
20
|
+
* Taking only `response.applyOutcomes[peerId]` — rather than spreading the whole map — is what keeps
|
|
21
|
+
* one member from reporting outcomes on other members' behalf: a peer that echoes back a record full
|
|
22
|
+
* of entries contributes exactly one, its own. The field is unsigned advisory data (see its doc
|
|
23
|
+
* comment for why that is safe), so this is a shaping rule, not a security boundary.
|
|
24
|
+
*
|
|
25
|
+
* Returns `undefined` when no peer reported anything, so the common case adds no empty object to the
|
|
26
|
+
* record.
|
|
27
|
+
*/
|
|
28
|
+
function collectApplyOutcomes(
|
|
29
|
+
responses: ReadonlyArray<{ peerId: string; response?: ClusterRecord | null }>
|
|
30
|
+
): ClusterRecord['applyOutcomes'] {
|
|
31
|
+
let collected: NonNullable<ClusterRecord['applyOutcomes']> | undefined;
|
|
32
|
+
for (const { peerId, response } of responses) {
|
|
33
|
+
const own = response?.applyOutcomes?.[peerId];
|
|
34
|
+
if (own === undefined) continue;
|
|
35
|
+
collected ??= {};
|
|
36
|
+
collected[peerId] = own;
|
|
37
|
+
}
|
|
38
|
+
return collected;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Fold collected outcomes into a record in place, later report winning per peer. No-op for `undefined`. */
|
|
42
|
+
function mergeApplyOutcomes(record: ClusterRecord, collected: ClusterRecord['applyOutcomes']): void {
|
|
43
|
+
if (collected === undefined) return;
|
|
44
|
+
record.applyOutcomes = { ...record.applyOutcomes, ...collected };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Consensus refused a transaction: enough members voted reject that super-majority became
|
|
49
|
+
* impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
|
|
50
|
+
* "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
|
|
51
|
+
* rejection reasons — those are free-form text that is part of each member's signed vote payload
|
|
52
|
+
* (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
|
|
53
|
+
* `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
|
|
54
|
+
* loss (confirmed against local storage) or a genuine validation fault.
|
|
55
|
+
*/
|
|
56
|
+
export class ValidatorRejectionError extends Error {
|
|
57
|
+
constructor(
|
|
58
|
+
message: string,
|
|
59
|
+
/** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
|
|
60
|
+
readonly rejectReasons: Record<string, string>
|
|
61
|
+
) {
|
|
62
|
+
super(message);
|
|
63
|
+
this.name = 'ValidatorRejectionError';
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* The transaction lost a conflict race: one or more members answered with a signed `conflict`
|
|
69
|
+
* vote (they hold a rival transaction that won the deterministic race on the same blocks) and
|
|
70
|
+
* approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} — nobody
|
|
71
|
+
* judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
|
|
72
|
+
* `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
|
|
73
|
+
* `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
|
|
74
|
+
* as a thrown error only from other paths. The commit conversion matters as much as the pend one:
|
|
75
|
+
* at the moment this is thrown zero members approved and the members hold the winner — nothing of
|
|
76
|
+
* the loser landed — yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
|
|
77
|
+
* (it treats throws as transport faults), and that re-driven commit races into the window after
|
|
78
|
+
* members apply the winner and clear its reservation, where it can assemble a consensus no member
|
|
79
|
+
* will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
|
|
80
|
+
* writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
|
|
81
|
+
* and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
|
|
82
|
+
* parsed out of prose.
|
|
83
|
+
*/
|
|
84
|
+
export class ConflictRaceLostError extends Error {
|
|
85
|
+
constructor(
|
|
86
|
+
message: string,
|
|
87
|
+
/** peerId → messageHash of the rival transaction that member holds as the race winner. */
|
|
88
|
+
readonly conflicts: Record<string, string>
|
|
89
|
+
) {
|
|
90
|
+
super(message);
|
|
91
|
+
this.name = 'ConflictRaceLostError';
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* The transaction's pend could not proceed because one or more members answered with a signed `held`
|
|
97
|
+
* vote: the requested blocks are reserved by a different unresolved action in that member's durable
|
|
98
|
+
* storage. Sibling of {@link ConflictRaceLostError} and retryable for the same reason — nobody judged
|
|
99
|
+
* this write invalid; it queued behind a reservation that disappears when the holder commits or
|
|
100
|
+
* cancels.
|
|
101
|
+
*
|
|
102
|
+
* The two are separate because they name different things. A conflict vote names the winning rival's
|
|
103
|
+
* `messageHash`, which the member holds whole; a held vote can only name the rival's **action id**,
|
|
104
|
+
* because it fires in the window where the rival has left the member's in-memory table but not yet its
|
|
105
|
+
* storage. `CoordinatorRepo.pend` converts this into a `StaleFailure` with `conflict: true` so the
|
|
106
|
+
* normal retry machinery (`isConflictFailure`) absorbs it, exactly as it does a lost race.
|
|
107
|
+
*
|
|
108
|
+
* Only a PEND record can produce it: `held` votes come from `ClusterMember.validatePendOperations`,
|
|
109
|
+
* which inspects pend operations only, so `CoordinatorRepo.commit` never meets one.
|
|
110
|
+
*/
|
|
111
|
+
export class BlocksHeldError extends Error {
|
|
112
|
+
constructor(
|
|
113
|
+
message: string,
|
|
114
|
+
/** peerId → actionId of the unresolved action that member's storage says holds the blocks. */
|
|
115
|
+
readonly heldBy: Record<string, string>
|
|
116
|
+
) {
|
|
117
|
+
super(message);
|
|
118
|
+
this.name = 'BlocksHeldError';
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
|
|
123
|
+
export type TimerCancel = () => void;
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Production timer binding: a one-shot `setTimeout` whose handle is **unref'd** so a pending
|
|
127
|
+
* commit-retry (or the deferred transaction cleanup) never keeps an otherwise-idle process alive.
|
|
128
|
+
* The returned handle clears the timeout (idempotent). Mirrors the reactivity rotation
|
|
129
|
+
* re-registration scheduler's `defaultSetTimer` (see reactivity/rotation-rereg-scheduler.ts).
|
|
130
|
+
*/
|
|
131
|
+
function defaultSetTimer(fn: () => void, delayMs: number): TimerCancel {
|
|
132
|
+
const handle = setTimeout(fn, delayMs);
|
|
133
|
+
// An idle retry/cleanup timer must not pin a process (mirror rotation re-registration + push-state gossip).
|
|
134
|
+
(handle as { unref?: () => void }).unref?.();
|
|
135
|
+
return (): void => clearTimeout(handle);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Optional injection seam for deterministic time. Production leaves both undefined and gets
|
|
140
|
+
* `Date.now` + an unref'd `setTimeout`; tests inject a fake clock + timer queue so scheduled
|
|
141
|
+
* commit-retries fire in virtual (not wall-clock) time.
|
|
142
|
+
*/
|
|
143
|
+
export interface ClusterCoordinatorClock {
|
|
144
|
+
/** Clock (Unix ms). Defaults to `Date.now`. */
|
|
145
|
+
now?: () => number;
|
|
146
|
+
/** Schedule a one-shot timer, returning a cancel handle. Defaults to an unref'd `setTimeout`. */
|
|
147
|
+
setTimer?: (fn: () => void, delayMs: number) => TimerCancel;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Manages the state of cluster transactions for a specific block ID
|
|
152
|
+
*/
|
|
153
|
+
interface CommitRetryState {
|
|
154
|
+
pendingPeers: Set<string>;
|
|
155
|
+
attempt: number;
|
|
156
|
+
intervalMs: number;
|
|
157
|
+
cancel?: TimerCancel;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
interface ClusterTransactionState {
|
|
161
|
+
messageHash: string;
|
|
162
|
+
record: ClusterRecord;
|
|
163
|
+
pending: Pending<ClusterRecord>;
|
|
164
|
+
lastUpdate: number;
|
|
165
|
+
promiseTimeout?: NodeJS.Timeout;
|
|
166
|
+
resolutionTimeout?: NodeJS.Timeout;
|
|
167
|
+
retry?: CommitRetryState;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Manages distributed transactions across clusters */
|
|
171
|
+
/**
|
|
172
|
+
* What a cohort lookup established about a block's cohort. `resolved: false` covers BOTH a lookup
|
|
173
|
+
* that threw and one that answered with nobody: neither names a destination for a write, and the
|
|
174
|
+
* durability class both produce is the same (`unrouted`). `reason` is for logs only — never branch
|
|
175
|
+
* on it.
|
|
176
|
+
*/
|
|
177
|
+
export type CohortResolution =
|
|
178
|
+
| { readonly resolved: true; readonly peerIds: readonly string[] }
|
|
179
|
+
| { readonly resolved: false; readonly reason: string };
|
|
180
|
+
|
|
181
|
+
export class ClusterCoordinator {
|
|
182
|
+
private transactions: Map<string, ClusterTransactionState> = new Map();
|
|
183
|
+
private readonly retryInitialIntervalMs: number;
|
|
184
|
+
private readonly retryBackoffFactor: number;
|
|
185
|
+
private readonly retryMaxIntervalMs: number;
|
|
186
|
+
private readonly retryMaxAttempts: number;
|
|
187
|
+
private readonly commitBroadcastImmediateRetries: number;
|
|
188
|
+
private readonly promiseImmediateRetries: number;
|
|
189
|
+
/** Injected clock/timer seam; production defaults to `Date.now` + unref'd `setTimeout`. */
|
|
190
|
+
private readonly now: () => number;
|
|
191
|
+
private readonly setTimer: (fn: () => void, delayMs: number) => TimerCancel;
|
|
192
|
+
|
|
193
|
+
constructor(
|
|
194
|
+
private readonly keyNetwork: IKeyNetwork,
|
|
195
|
+
/** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
|
|
196
|
+
private readonly createClusterClient: (peerId: PeerId) => ICluster,
|
|
197
|
+
private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
|
|
198
|
+
private readonly localCluster?: {
|
|
199
|
+
update: (record: ClusterRecord) => Promise<ClusterRecord>;
|
|
200
|
+
peerId: PeerId;
|
|
201
|
+
wasTransactionExecuted?: (messageHash: string) => boolean;
|
|
202
|
+
/** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
|
|
203
|
+
getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
|
|
204
|
+
/** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
|
|
205
|
+
getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
|
|
206
|
+
},
|
|
207
|
+
private readonly fretService?: FretService,
|
|
208
|
+
private readonly reputation?: IPeerReputation,
|
|
209
|
+
private readonly stateStore?: ITransactionStateStore,
|
|
210
|
+
clock?: ClusterCoordinatorClock
|
|
211
|
+
) {
|
|
212
|
+
this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
|
|
213
|
+
this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
|
|
214
|
+
this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
|
|
215
|
+
this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
|
|
216
|
+
this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
|
|
217
|
+
this.promiseImmediateRetries = cfg.promiseImmediateRetries ?? 1;
|
|
218
|
+
this.now = clock?.now ?? ((): number => Date.now());
|
|
219
|
+
this.setTimer = clock?.setTimer ?? defaultSetTimer;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Invoke one cluster member's `update`, retrying transient REMOTE failures up to
|
|
224
|
+
* `immediateRetries` times before surfacing the error. The local cluster is invoked
|
|
225
|
+
* exactly once — a local throw is a real fault (validation / merge / consensus), not a
|
|
226
|
+
* transient transport blip. A remote call rides a libp2p stream that a circuit-relay
|
|
227
|
+
* ("limited") connection can reset once a per-circuit cap or reservation lapses, which
|
|
228
|
+
* surfaces as a StreamResetError; an immediate retry on the (usually still-warm)
|
|
229
|
+
* connection recovers most of those without escalating the peer to a failure. Shared by
|
|
230
|
+
* the promise-collection, commit-collection, and commit-broadcast phases so all three
|
|
231
|
+
* react to a relayed reset the same way.
|
|
232
|
+
*/
|
|
233
|
+
private async updateMember(peerIdStr: string, record: ClusterRecord, immediateRetries: number, phase: string): Promise<ClusterRecord> {
|
|
234
|
+
const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
|
|
235
|
+
if (isLocal) {
|
|
236
|
+
return await this.localCluster!.update(record);
|
|
237
|
+
}
|
|
238
|
+
const maxAttempts = 1 + Math.max(0, immediateRetries);
|
|
239
|
+
let lastError: unknown;
|
|
240
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
241
|
+
try {
|
|
242
|
+
return await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
|
|
243
|
+
} catch (err) {
|
|
244
|
+
lastError = err;
|
|
245
|
+
if (attempt < maxAttempts) {
|
|
246
|
+
log('cluster-tx:member-update-retry', {
|
|
247
|
+
messageHash: record.messageHash,
|
|
248
|
+
peerId: peerIdStr,
|
|
249
|
+
phase,
|
|
250
|
+
attempt,
|
|
251
|
+
error: err instanceof Error ? err.message : String(err)
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
throw lastError;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Creates a base58btc string hash uniquely identifying a transaction. For a v2 record the caller
|
|
261
|
+
* threads in the {@link membershipDigest} of the peer set so the responsible membership is bound into
|
|
262
|
+
* the identity (two different peer sets ⇒ two different hashes). Omitting `membershipDigestValue`
|
|
263
|
+
* reproduces the legacy v1 hash byte-for-byte.
|
|
264
|
+
*
|
|
265
|
+
* NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
|
|
266
|
+
* which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
|
|
267
|
+
* `pend.priority` (single-collection) — is automatically covered here and by the derived
|
|
268
|
+
* promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
|
|
269
|
+
* cannot strip or inflate it without invalidating the message hash the members verify. No separate
|
|
270
|
+
* priority-hashing step is needed.
|
|
271
|
+
*/
|
|
272
|
+
private async createMessageHash(message: RepoMessage, membershipDigestValue?: string): Promise<string> {
|
|
273
|
+
return computeClusterMessageHash(message, membershipDigestValue);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* The ONE cohort lookup every accessor on this class derives from: the raw peer map when the key
|
|
278
|
+
* network answered, otherwise the reason it did not. A thrown `findCluster` is logged here and
|
|
279
|
+
* nowhere else. Callers that need the map (`executeClusterTransaction`, which builds the record's
|
|
280
|
+
* `peers`) go through {@link getClusterForBlock}; callers that need to know whether the cohort
|
|
281
|
+
* RESOLVED go through {@link resolveCohort}.
|
|
282
|
+
*/
|
|
283
|
+
private async lookupCluster(blockId: BlockId): Promise<{ peers: ClusterPeers } | { reason: string }> {
|
|
284
|
+
try {
|
|
285
|
+
const peers = await this.keyNetwork.findCluster(routingKeyForBlock(blockId));
|
|
286
|
+
const peerIds = Object.keys(peers ?? {});
|
|
287
|
+
log('cluster-tx:cluster-members', { blockId, peerIds });
|
|
288
|
+
return { peers: peers ?? {} };
|
|
289
|
+
} catch (e) {
|
|
290
|
+
log('WARN findCluster failed for %s: %o', blockId, e)
|
|
291
|
+
return { reason: `findCluster threw: ${(e as Error)?.message ?? String(e)}` };
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Gets all peers in the cluster for a specific block ID. Empty when the lookup failed — the
|
|
297
|
+
* consensus path treats "no cohort" and "lookup failed" alike (there is nobody to run consensus
|
|
298
|
+
* with either way); a caller that must tell them apart uses {@link resolveCohort}.
|
|
299
|
+
*/
|
|
300
|
+
private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
|
|
301
|
+
const outcome = await this.lookupCluster(blockId);
|
|
302
|
+
return 'peers' in outcome ? outcome.peers : {};
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Whether the block's cohort could be established, and who it is. The primitive behind
|
|
307
|
+
* {@link getClusterPeerIds} and {@link getClusterSize}: a lookup that threw and a lookup that named
|
|
308
|
+
* nobody used to reach every caller as the same empty list, and `CoordinatorRepo`'s solo
|
|
309
|
+
* short-circuit then acknowledged a write it had no idea where to send exactly as it acknowledged a
|
|
310
|
+
* write to a genuine cohort of one (GitHub #19). Both shapes are still `resolved: false` here —
|
|
311
|
+
* neither names a destination — but they are distinguishable from a resolved cohort, which is what
|
|
312
|
+
* the write's durability class needs (`unrouted` vs `local`).
|
|
313
|
+
*/
|
|
314
|
+
async resolveCohort(blockId: BlockId): Promise<CohortResolution> {
|
|
315
|
+
const outcome = await this.lookupCluster(blockId);
|
|
316
|
+
if ('reason' in outcome) return { resolved: false, reason: outcome.reason };
|
|
317
|
+
const peerIds = Object.keys(outcome.peers);
|
|
318
|
+
if (peerIds.length === 0) return { resolved: false, reason: 'findCluster named nobody' };
|
|
319
|
+
return { resolved: true, peerIds };
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* A node never runs a cluster transaction for a cohort it is not in. Behind members reconcile from the
|
|
324
|
+
* coordinator's own proof-carrying copy (its member applies before the merged record fans out), and a
|
|
325
|
+
* coordinator outside `record.peers` is not a reconcile target — so a cohort with no holder would stay
|
|
326
|
+
* behind and the commit durability gate would refuse, having first put this node's vote and storage
|
|
327
|
+
* where the cohort does not look. The invariant is held here, at the one place a record's `peers` is
|
|
328
|
+
* chosen, rather than left to the routing convention.
|
|
329
|
+
*
|
|
330
|
+
* Fires only on a RESOLVED cohort (at least one peer) that excludes the wired local member. An empty
|
|
331
|
+
* cohort is a failed lookup, not a cohort this node is outside of, so it is left to `executeTransaction`'s
|
|
332
|
+
* size checks; `CoordinatorRepo`'s solo short-circuit keeps unresolved and single-peer cohorts away from
|
|
333
|
+
* this method altogether in any case. After its responsibility check, what remains is a multi-member
|
|
334
|
+
* cohort that changed inside the responsibility cache's staleness window. With no local member wired the guard does not apply: that
|
|
335
|
+
* bypass exists for wiring without an identity (direct constructors, some tests), never for production.
|
|
336
|
+
*/
|
|
337
|
+
private assertLocalMemberInCohort(blockId: BlockId, peers: ClusterPeers): void {
|
|
338
|
+
if (!this.localCluster) return;
|
|
339
|
+
const peerIds = Object.keys(peers);
|
|
340
|
+
const selfId = this.localCluster.peerId.toString();
|
|
341
|
+
if (peerIds.length === 0 || peerIds.includes(selfId)) return;
|
|
342
|
+
log('cluster-tx:not-in-cohort', { blockId, selfId, peerIds });
|
|
343
|
+
throw new ResponsibilityRefusalError('not-responsible', [blockId],
|
|
344
|
+
`refusing to coordinate a cluster transaction for a cohort this node is not in: ${peerIds.join(', ')}`);
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
|
|
348
|
+
const peerCount = Object.keys(peers ?? {}).length;
|
|
349
|
+
const record: ClusterRecord = {
|
|
350
|
+
messageHash,
|
|
351
|
+
peers,
|
|
352
|
+
// v2: bind the responsible membership into the signed identity. messageHash was computed over
|
|
353
|
+
// this same digest, so a different peer set would have produced a different messageHash.
|
|
354
|
+
membershipVersion: CURRENT_MEMBERSHIP_VERSION,
|
|
355
|
+
membershipDigest: membershipDigestValue,
|
|
356
|
+
message,
|
|
357
|
+
promises: {},
|
|
358
|
+
commits: {},
|
|
359
|
+
suggestedClusterSize: peerCount || undefined,
|
|
360
|
+
minRequiredSize: this.cfg.allowClusterDownsize ? undefined : this.cfg.clusterSize
|
|
361
|
+
};
|
|
362
|
+
|
|
363
|
+
// Add network size hint if available
|
|
364
|
+
if (this.fretService) {
|
|
365
|
+
try {
|
|
366
|
+
const estimate = this.fretService.getNetworkSizeEstimate();
|
|
367
|
+
if (estimate.size_estimate > 0) {
|
|
368
|
+
record.networkSizeHint = estimate.size_estimate;
|
|
369
|
+
record.networkSizeConfidence = estimate.confidence;
|
|
370
|
+
}
|
|
371
|
+
} catch (err) {
|
|
372
|
+
// Ignore errors getting size estimate
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
return record;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Initiates a 2-phase transaction for a specific block ID.
|
|
381
|
+
* Returns the cluster record and whether the local cluster already executed the operations.
|
|
382
|
+
*/
|
|
383
|
+
async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
|
|
384
|
+
record: ClusterRecord;
|
|
385
|
+
localExecuted: boolean;
|
|
386
|
+
/**
|
|
387
|
+
* Local storage's verdict for a pend operation this node's own cluster member applied during
|
|
388
|
+
* consensus, when the member retained one. Meaningful only when `localExecuted` is true;
|
|
389
|
+
* absent for non-pend messages, for a member that predates the retention, or after the
|
|
390
|
+
* retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
|
|
391
|
+
*/
|
|
392
|
+
localPendResult?: PendResult;
|
|
393
|
+
/**
|
|
394
|
+
* Local storage's verdict for a commit operation this node's own cluster member applied
|
|
395
|
+
* during consensus, when the member retained one. Same availability contract as
|
|
396
|
+
* `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
|
|
397
|
+
* win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
|
|
398
|
+
* success no member durably stored.
|
|
399
|
+
*/
|
|
400
|
+
localCommitResult?: CommitResult;
|
|
401
|
+
/**
|
|
402
|
+
* Conflict-shaped pend refusals reported by OTHER cohort members on their consensus responses
|
|
403
|
+
* (`ClusterRecord.applyOutcomes`), keyed by peer id. This is the arm `localPendResult` cannot
|
|
404
|
+
* cover: the refusing member is frequently not the coordinating node, and its verdict used to
|
|
405
|
+
* stay on that member while the writer was told the pend won. Unsigned advisory data — an
|
|
406
|
+
* entry means "retry", never "this write was invalid". Absent when nobody reported one.
|
|
407
|
+
*
|
|
408
|
+
* Residual: a member that reaches consensus only via the scheduled commit-retry timer applies
|
|
409
|
+
* after this method has already resolved, so its refusal arrives too late to appear here. The
|
|
410
|
+
* member-side commit-promise guard (`validateCommitAgainstRefusedPend`) is the backstop for
|
|
411
|
+
* that path.
|
|
412
|
+
*/
|
|
413
|
+
cohortPendRefusals?: { [peerId: string]: StaleFailure };
|
|
414
|
+
/**
|
|
415
|
+
* What OTHER cohort members reported about durably holding a commit after applying it at
|
|
416
|
+
* consensus (`ClusterRecord.applyOutcomes[peer].commit`), keyed by peer id — successes AND
|
|
417
|
+
* refusals, because `CoordinatorRepo.commit`'s durability gate counts the successes against
|
|
418
|
+
* the cohort the commit ran on and acknowledges only a majority. Each member's verdict is
|
|
419
|
+
* measured after its own reconcile, so a member that pulled the revision from a cohort peer
|
|
420
|
+
* reports success. Self is excluded for the same reason as `cohortPendRefusals` (its verdict
|
|
421
|
+
* travels as `localCommitResult`). Unsigned advisory data: a false success is one holder the
|
|
422
|
+
* member's signed approve vote already admitted to the majority; a false refusal is retry
|
|
423
|
+
* pressure. Absent when nobody reported one (a pend message, or pre-upgrade members).
|
|
424
|
+
*
|
|
425
|
+
* Same residual as `cohortPendRefusals`: a member reached only by the scheduled commit-retry
|
|
426
|
+
* timer applies after this method has resolved, and its report arrives too late to count —
|
|
427
|
+
* the gate then refuses honestly and the writer re-drives.
|
|
428
|
+
*/
|
|
429
|
+
cohortCommitOutcomes?: { [peerId: string]: CommitResult };
|
|
430
|
+
}> {
|
|
431
|
+
// The coordinating block id is derived HERE, from the key this method is already handed, rather
|
|
432
|
+
// than being set by each caller's message builder: a member's membership admission gate derives
|
|
433
|
+
// its own cohort view from this field, and a builder that forgets it silently downgrades the gate
|
|
434
|
+
// to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
|
|
435
|
+
// admitted at pend, refused at commit). Doing it at the single choke point means a future message
|
|
436
|
+
// builder cannot reintroduce the gap.
|
|
437
|
+
//
|
|
438
|
+
// Two constraints this shape exists to satisfy:
|
|
439
|
+
// - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
|
|
440
|
+
// N concurrent calls, one per block. In-place mutation would leak one block's id into another
|
|
441
|
+
// block's transaction.
|
|
442
|
+
// - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
|
|
443
|
+
// not just its first block, so this must not overwrite it. Tested on `length`, not on the
|
|
444
|
+
// field: an empty list carries no id for a member to derive from, so preserving one would be
|
|
445
|
+
// the same silent downgrade to the fallback floor this choke point exists to prevent.
|
|
446
|
+
const coordinated: RepoMessage = message.coordinatingBlockIds?.length
|
|
447
|
+
? message
|
|
448
|
+
: { ...message, coordinatingBlockIds: [blockId] };
|
|
449
|
+
|
|
450
|
+
// Get the cluster peers for this block
|
|
451
|
+
const peers = await this.getClusterForBlock(blockId);
|
|
452
|
+
this.assertLocalMemberInCohort(blockId, peers);
|
|
453
|
+
|
|
454
|
+
// Bind the responsible membership into the transaction identity (v2): the digest is folded into
|
|
455
|
+
// the messageHash below, so two different peer sets produce two different messageHashes rather
|
|
456
|
+
// than one hash with a silent internal disagreement about who is responsible.
|
|
457
|
+
const membershipDigestValue = await membershipDigest(peers);
|
|
458
|
+
|
|
459
|
+
// Create a unique hash for this transaction (over message + membership digest). Hashing the
|
|
460
|
+
// coordinating-block-bearing copy is what makes the field tamper-evident in transit — and it also
|
|
461
|
+
// makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
|
|
462
|
+
// identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
|
|
463
|
+
const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
|
|
464
|
+
|
|
465
|
+
// Create a cluster record for this transaction
|
|
466
|
+
const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
|
|
467
|
+
log('cluster-tx:start', {
|
|
468
|
+
messageHash,
|
|
469
|
+
blockId,
|
|
470
|
+
peerCount: Object.keys(peers ?? {}).length,
|
|
471
|
+
allowDownsize: this.cfg.allowClusterDownsize,
|
|
472
|
+
configuredSize: this.cfg.clusterSize,
|
|
473
|
+
suggestedSize: record.suggestedClusterSize,
|
|
474
|
+
minRequiredSize: record.minRequiredSize
|
|
475
|
+
});
|
|
476
|
+
|
|
477
|
+
// Create a new pending transaction
|
|
478
|
+
const transactionPromise = this.executeTransaction(peers, record);
|
|
479
|
+
const pending = new Pending(transactionPromise);
|
|
480
|
+
|
|
481
|
+
// Store the transaction state
|
|
482
|
+
const state: ClusterTransactionState = {
|
|
483
|
+
messageHash,
|
|
484
|
+
record,
|
|
485
|
+
pending,
|
|
486
|
+
lastUpdate: this.now()
|
|
487
|
+
};
|
|
488
|
+
this.transactions.set(messageHash, state);
|
|
489
|
+
this.persistCoordinatorState(messageHash, record, 'promising');
|
|
490
|
+
log('cluster-tx:transaction-store', {
|
|
491
|
+
messageHash,
|
|
492
|
+
transactionKeys: Array.from(this.transactions.keys())
|
|
493
|
+
});
|
|
494
|
+
|
|
495
|
+
// Wait for the transaction to complete
|
|
496
|
+
try {
|
|
497
|
+
const result = await pending.result();
|
|
498
|
+
// Check if the local cluster already executed the operations during consensus
|
|
499
|
+
const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
|
|
500
|
+
const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
|
|
501
|
+
const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
|
|
502
|
+
// Self is excluded: this node's own member verdict is already carried, more directly and
|
|
503
|
+
// without the wire round trip, by `localPendResult` — and leaving it in both places would
|
|
504
|
+
// make the coordinator's "prefer local" rule ambiguous.
|
|
505
|
+
// Re-checked here rather than trusted: members are supposed to report only conflict-shaped
|
|
506
|
+
// refusals, but the field arrives off the wire, so anything else (a success, a bare-reason
|
|
507
|
+
// fault, a malformed entry) is dropped instead of being handed to a caller that would read
|
|
508
|
+
// it as a retryable conflict.
|
|
509
|
+
const selfId = this.localCluster?.peerId.toString();
|
|
510
|
+
const cohortPendRefusals: { [peerId: string]: StaleFailure } = {};
|
|
511
|
+
// The commit arm is re-checked the same way, to the shape the gate reads: a plain
|
|
512
|
+
// `success: true`, or an object whose `success` is `false`. Anything else off the wire is
|
|
513
|
+
// dropped rather than counted as a holder.
|
|
514
|
+
const cohortCommitOutcomes: { [peerId: string]: CommitResult } = {};
|
|
515
|
+
for (const [peerId, outcome] of Object.entries(result.applyOutcomes ?? {})) {
|
|
516
|
+
if (peerId === selfId) continue;
|
|
517
|
+
const pend = outcome?.pend;
|
|
518
|
+
if (pend !== undefined && !pend.success && isConflictFailure(pend)) {
|
|
519
|
+
cohortPendRefusals[peerId] = pend;
|
|
520
|
+
}
|
|
521
|
+
const commit = outcome?.commit;
|
|
522
|
+
if (commit !== null && typeof commit === 'object' && (commit.success === true || commit.success === false)) {
|
|
523
|
+
cohortCommitOutcomes[peerId] = commit;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
return {
|
|
527
|
+
record: result,
|
|
528
|
+
localExecuted,
|
|
529
|
+
...(localPendResult === undefined ? {} : { localPendResult }),
|
|
530
|
+
...(localCommitResult === undefined ? {} : { localCommitResult }),
|
|
531
|
+
...(Object.keys(cohortPendRefusals).length === 0 ? {} : { cohortPendRefusals }),
|
|
532
|
+
...(Object.keys(cohortCommitOutcomes).length === 0 ? {} : { cohortCommitOutcomes })
|
|
533
|
+
};
|
|
534
|
+
} finally {
|
|
535
|
+
const stored = this.transactions.get(messageHash);
|
|
536
|
+
const retrySnapshot = stored?.retry ? {
|
|
537
|
+
attempt: stored.retry.attempt,
|
|
538
|
+
pending: Array.from(stored.retry.pendingPeers ?? [])
|
|
539
|
+
} : undefined;
|
|
540
|
+
log('cluster-tx:complete', {
|
|
541
|
+
messageHash,
|
|
542
|
+
finalPromises: stored ? Object.keys(stored.record.promises ?? {}) : undefined,
|
|
543
|
+
finalCommits: stored ? Object.keys(stored.record.commits ?? {}) : undefined,
|
|
544
|
+
retry: retrySnapshot
|
|
545
|
+
});
|
|
546
|
+
// Don't remove transaction immediately if retries are scheduled
|
|
547
|
+
// Let the retry completion or abort handle cleanup
|
|
548
|
+
if (!stored?.retry) {
|
|
549
|
+
// Wait a bit before cleanup to allow any in-flight responses to arrive
|
|
550
|
+
this.setTimer(() => {
|
|
551
|
+
this.transactions.delete(messageHash);
|
|
552
|
+
this.deleteCoordinatorState(messageHash);
|
|
553
|
+
log('cluster-tx:transaction-remove', {
|
|
554
|
+
messageHash,
|
|
555
|
+
remaining: Array.from(this.transactions.keys())
|
|
556
|
+
});
|
|
557
|
+
}, 100);
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* Executes the full transaction process
|
|
564
|
+
*/
|
|
565
|
+
private async executeTransaction(peers: ClusterPeers, record: ClusterRecord): Promise<ClusterRecord> {
|
|
566
|
+
const peerCount = Object.keys(peers).length;
|
|
567
|
+
|
|
568
|
+
// Validate against minimum cluster size
|
|
569
|
+
if (peerCount < this.cfg.minAbsoluteClusterSize) {
|
|
570
|
+
const validated = await this.validateSmallCluster(peerCount, peers);
|
|
571
|
+
if (!validated) {
|
|
572
|
+
log('cluster-tx:reject-too-small', {
|
|
573
|
+
peerCount,
|
|
574
|
+
minRequired: this.cfg.minAbsoluteClusterSize
|
|
575
|
+
});
|
|
576
|
+
throw new Error(`Cluster size ${peerCount} below minimum ${this.cfg.minAbsoluteClusterSize} and not validated`);
|
|
577
|
+
}
|
|
578
|
+
log('cluster-tx:small-cluster-validated', { peerCount });
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
// Check configured cluster size
|
|
582
|
+
if (!this.cfg.allowClusterDownsize && peerCount < this.cfg.clusterSize) {
|
|
583
|
+
log('cluster-tx:reject-downsize', { peerCount, required: this.cfg.clusterSize });
|
|
584
|
+
throw new Error(`Cluster size ${peerCount} below configured minimum ${this.cfg.clusterSize}`);
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
// Collect promises with super-majority requirement
|
|
588
|
+
const promised = await this.collectPromises(peers, record);
|
|
589
|
+
const superMajority = Math.ceil(peerCount * this.cfg.superMajorityThreshold);
|
|
590
|
+
|
|
591
|
+
// Count approvals, rejections and the two RETRYABLE refusals separately. A `conflict` vote is a
|
|
592
|
+
// member saying "not now — I hold the race winner"; a `held` vote is a member saying "not now —
|
|
593
|
+
// a different unresolved action holds these blocks in my storage". Neither may count toward
|
|
594
|
+
// approvals OR rejections, or a transient refusal would masquerade as a validator rejection
|
|
595
|
+
// (permanent) or as silence (indistinguishable from an unreachable cohort) — both wrong.
|
|
596
|
+
const promises = promised.record.promises;
|
|
597
|
+
const approvalCount = Object.values(promises).filter(sig => sig.type === 'approve').length;
|
|
598
|
+
const rejectionCount = Object.values(promises).filter(sig => sig.type === 'reject').length;
|
|
599
|
+
const conflictCount = Object.values(promises).filter(sig => sig.type === 'conflict').length;
|
|
600
|
+
const heldCount = Object.values(promises).filter(sig => sig.type === 'held').length;
|
|
601
|
+
|
|
602
|
+
// Check if rejections make super-majority impossible
|
|
603
|
+
// If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
|
|
604
|
+
const maxAllowedRejections = peerCount - superMajority;
|
|
605
|
+
// Whether the merged record itself PROVES super-majority unreachable — the same sum a member
|
|
606
|
+
// re-derives as `ConflictSuperseded`/`Rejected` from the signed votes, which is what makes an
|
|
607
|
+
// abandonment broadcast proof-carrying rather than an unauthenticated "forget this".
|
|
608
|
+
const refusalsProveUnreachable = rejectionCount + conflictCount + heldCount > maxAllowedRejections;
|
|
609
|
+
if (rejectionCount > maxAllowedRejections) {
|
|
610
|
+
const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
|
|
611
|
+
.flatMap(([peerId, sig]) => sig.type === 'reject' ? [[peerId, sig.rejectReason ?? 'unknown'] as const] : []));
|
|
612
|
+
const rejectReasons = Object.entries(rejectReasonsByPeer)
|
|
613
|
+
.map(([peerId, reason]) => `${peerId}: ${reason}`)
|
|
614
|
+
.join('; ');
|
|
615
|
+
log('cluster-tx:rejected-by-validators', {
|
|
616
|
+
messageHash: record.messageHash,
|
|
617
|
+
peerCount,
|
|
618
|
+
rejections: rejectionCount,
|
|
619
|
+
maxAllowed: maxAllowedRejections,
|
|
620
|
+
reasons: rejectReasons
|
|
621
|
+
});
|
|
622
|
+
this.updateTransactionRecord(promised.record, 'rejected-by-validators');
|
|
623
|
+
// Abandoning here without telling anyone leaves every member that voted holding this
|
|
624
|
+
// transaction in its own reservation table, blocking its blocks until that member's
|
|
625
|
+
// staleness sweep fires — and each retry we throw back to the caller plants a fresh
|
|
626
|
+
// reservation, so the block never frees. The merged record carries enough signed
|
|
627
|
+
// rejections to *prove* the transaction is dead, so replaying it to the cohort makes
|
|
628
|
+
// every member recompute `Rejected` and clear immediately. Proof-carrying, so a member
|
|
629
|
+
// need not trust us: it verifies the signatures it is shown.
|
|
630
|
+
this.broadcastAbandonment(promised.record, 'rejected-by-validators');
|
|
631
|
+
throw new ValidatorRejectionError(
|
|
632
|
+
`Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
|
|
633
|
+
rejectReasonsByPeer);
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
// A conflict-answered shortfall is a LOST RACE, not a validator verdict and not silence.
|
|
637
|
+
// Checked after the rejection threshold (a genuine validator rejection still wins) and
|
|
638
|
+
// before the generic shortfall (which must stay reserved for the genuinely-silent cohort).
|
|
639
|
+
if (conflictCount > 0 && approvalCount < superMajority) {
|
|
640
|
+
const conflicts = Object.fromEntries(Object.entries(promises)
|
|
641
|
+
.flatMap(([peerId, sig]) => sig.type === 'conflict' ? [[peerId, sig.conflictWith] as const] : []));
|
|
642
|
+
log('cluster-tx:conflict-race-lost', {
|
|
643
|
+
messageHash: record.messageHash,
|
|
644
|
+
peerCount,
|
|
645
|
+
approvals: approvalCount,
|
|
646
|
+
rejections: rejectionCount,
|
|
647
|
+
conflicts,
|
|
648
|
+
superMajority
|
|
649
|
+
});
|
|
650
|
+
this.updateTransactionRecord(promised.record, 'conflict-race-lost');
|
|
651
|
+
// Broadcast only when the merged record itself PROVES the transaction can no longer reach
|
|
652
|
+
// super-majority (members re-derive ConflictSuperseded/Rejected from the signed votes and
|
|
653
|
+
// clear their reservations immediately). Below that bar the record proves nothing and a
|
|
654
|
+
// broadcast would be the unauthenticated "forget this" the shortfall NOTE below refuses.
|
|
655
|
+
if (refusalsProveUnreachable) {
|
|
656
|
+
this.broadcastAbandonment(promised.record, 'conflict-race-lost');
|
|
657
|
+
}
|
|
658
|
+
throw new ConflictRaceLostError(
|
|
659
|
+
`Conflict race lost: ${conflictCount}/${peerCount} member(s) hold a conflicting winner (${approvalCount}/${superMajority} approvals)`,
|
|
660
|
+
conflicts);
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
// A `held`-answered shortfall is the OTHER retryable refusal: the pend queued behind a rival's
|
|
664
|
+
// unresolved reservation. Checked after the conflict branch so a lost race still wins when both
|
|
665
|
+
// answer — a conflict vote names the winning transaction's messageHash, which is strictly more
|
|
666
|
+
// actionable than an action id — and, like it, before the generic shortfall, which must stay
|
|
667
|
+
// reserved for the genuinely-silent cohort.
|
|
668
|
+
if (heldCount > 0 && approvalCount < superMajority) {
|
|
669
|
+
const heldBy = Object.fromEntries(Object.entries(promises)
|
|
670
|
+
.flatMap(([peerId, sig]) => sig.type === 'held' ? [[peerId, sig.heldBy] as const] : []));
|
|
671
|
+
log('cluster-tx:pend-blocks-held', {
|
|
672
|
+
messageHash: record.messageHash,
|
|
673
|
+
peerCount,
|
|
674
|
+
approvals: approvalCount,
|
|
675
|
+
rejections: rejectionCount,
|
|
676
|
+
heldBy,
|
|
677
|
+
superMajority
|
|
678
|
+
});
|
|
679
|
+
this.updateTransactionRecord(promised.record, 'pend-blocks-held');
|
|
680
|
+
if (refusalsProveUnreachable) {
|
|
681
|
+
this.broadcastAbandonment(promised.record, 'pend-blocks-held');
|
|
682
|
+
}
|
|
683
|
+
throw new BlocksHeldError(
|
|
684
|
+
`Pend blocks held: ${heldCount}/${peerCount} member(s) hold an unresolved rival action (${approvalCount}/${superMajority} approvals)`,
|
|
685
|
+
heldBy);
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
if (peerCount > 1 && approvalCount < superMajority) {
|
|
689
|
+
log('cluster-tx:supermajority-failed', {
|
|
690
|
+
messageHash: record.messageHash,
|
|
691
|
+
peerCount,
|
|
692
|
+
approvals: approvalCount,
|
|
693
|
+
rejections: rejectionCount,
|
|
694
|
+
superMajority,
|
|
695
|
+
threshold: this.cfg.superMajorityThreshold
|
|
696
|
+
});
|
|
697
|
+
this.updateTransactionRecord(promised.record, 'supermajority-failed');
|
|
698
|
+
// NOTE: deliberately NOT broadcast, unlike the rejected-by-validators branch above. With
|
|
699
|
+
// conflict-answered shortfalls peeled off above, we get here only because peers did not
|
|
700
|
+
// answer at all, so the record carries no signed evidence that the transaction is dead — a
|
|
701
|
+
// broadcast would be an unauthenticated "forget this" that any caller could use to clear a
|
|
702
|
+
// live transaction out of a member's reservation table. Members that DID vote are freed by
|
|
703
|
+
// their own staleness sweep instead.
|
|
704
|
+
// NOTE: the message below is load-bearing wire text — the consuming repo
|
|
705
|
+
// (sereus cadre-core control-write-retry) matches it verbatim to retry a genuinely-silent
|
|
706
|
+
// cohort. Keep it byte-identical, and never fold `conflict` or `held` votes into its
|
|
707
|
+
// rejection count.
|
|
708
|
+
throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
// Mark as disputed when minority rejections exist but super-majority approves
|
|
712
|
+
if (rejectionCount > 0 && approvalCount >= superMajority) {
|
|
713
|
+
const rejectingPeers: string[] = [];
|
|
714
|
+
const rejectReasons: { [peerId: string]: string } = {};
|
|
715
|
+
for (const [peerId, sig] of Object.entries(promises)) {
|
|
716
|
+
if (sig.type === 'reject') {
|
|
717
|
+
rejectingPeers.push(peerId);
|
|
718
|
+
rejectReasons[peerId] = sig.rejectReason ?? 'unknown';
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
promised.record.disputed = true;
|
|
722
|
+
promised.record.disputeEvidence = { rejectingPeers, rejectReasons };
|
|
723
|
+
log('cluster-tx:disputed', {
|
|
724
|
+
messageHash: record.messageHash,
|
|
725
|
+
rejectingPeers,
|
|
726
|
+
rejectReasons,
|
|
727
|
+
approvalCount,
|
|
728
|
+
rejectionCount,
|
|
729
|
+
peerCount
|
|
730
|
+
});
|
|
731
|
+
// [dispute-subsystem-dormant] Evidence is computed and persisted but initiateDispute() is
|
|
732
|
+
// intentionally NOT called here. Dispute origination stays dormant pending arbitrator-set
|
|
733
|
+
// anchoring — without it a forged synthetic cohort passes resolution.
|
|
734
|
+
// Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
|
|
735
|
+
// Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
|
|
739
|
+
return await this.commitTransaction(promised.record);
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
/**
|
|
743
|
+
* The block's cohort peer ids as currently derivable. Empty when the cohort did not resolve
|
|
744
|
+
* ({@link resolveCohort}: `findCluster` threw, or named nobody), so a caller branching on
|
|
745
|
+
* `length <= 1` is also taking the degraded-routing branch. Derived from `resolveCohort` rather
|
|
746
|
+
* than re-deriving the cohort, so there is exactly one lookup rule.
|
|
747
|
+
*/
|
|
748
|
+
async getClusterPeerIds(blockId: BlockId): Promise<string[]> {
|
|
749
|
+
const cohort = await this.resolveCohort(blockId);
|
|
750
|
+
return cohort.resolved ? [...cohort.peerIds] : [];
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
/** {@link getClusterPeerIds}, counted. Derived from it rather than re-deriving the cohort, so the
|
|
754
|
+
* size a caller branches on and the ids it logs can never come from two different rules. */
|
|
755
|
+
async getClusterSize(blockId: BlockId): Promise<number> {
|
|
756
|
+
return (await this.getClusterPeerIds(blockId)).length;
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
/**
|
|
760
|
+
* Validate that a small cluster size is legitimate by querying remote peers
|
|
761
|
+
* for their network size estimates. Returns true if estimates roughly agree.
|
|
762
|
+
*/
|
|
763
|
+
private async validateSmallCluster(localSize: number, _peers: ClusterPeers): Promise<boolean> {
|
|
764
|
+
// If we have FRET and it shows confident estimate
|
|
765
|
+
if (this.fretService) {
|
|
766
|
+
try {
|
|
767
|
+
const estimate = this.fretService.getNetworkSizeEstimate();
|
|
768
|
+
if (estimate.confidence > 0.5) {
|
|
769
|
+
// Check if FRET estimate roughly matches observed cluster size
|
|
770
|
+
const orderOfMagnitude = Math.floor(Math.log10(estimate.size_estimate + 1));
|
|
771
|
+
const localOrderOfMagnitude = Math.floor(Math.log10(localSize + 1));
|
|
772
|
+
|
|
773
|
+
// If within same order of magnitude, accept it
|
|
774
|
+
if (Math.abs(orderOfMagnitude - localOrderOfMagnitude) <= 1) {
|
|
775
|
+
log('cluster-tx:small-cluster-validated-by-fret', {
|
|
776
|
+
localSize,
|
|
777
|
+
fretEstimate: estimate.size_estimate,
|
|
778
|
+
confidence: estimate.confidence,
|
|
779
|
+
sources: estimate.sources
|
|
780
|
+
});
|
|
781
|
+
return true;
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
} catch (err) {
|
|
785
|
+
// Ignore errors
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
// Fallback: with no confident network-size estimate, fail CLOSED by default.
|
|
790
|
+
// An undersized cluster with no way to justify its size is unsafe (a lone/
|
|
791
|
+
// near-lone node could rubber-stamp its own writes), so reject unless the
|
|
792
|
+
// operator has explicitly opted in via allowUnvalidatedSmallCluster (e.g.
|
|
793
|
+
// single-node / local dev knowingly running below the floor).
|
|
794
|
+
const admit = this.cfg.allowUnvalidatedSmallCluster ?? false;
|
|
795
|
+
log('cluster-tx:small-cluster-no-confident-estimate', {
|
|
796
|
+
localSize,
|
|
797
|
+
reason: 'no-confident-network-size-estimate',
|
|
798
|
+
admit
|
|
799
|
+
});
|
|
800
|
+
return admit;
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
/**
|
|
804
|
+
* Collects promises from all peers in the cluster
|
|
805
|
+
*/
|
|
806
|
+
private async collectPromises(peers: ClusterPeers, record: ClusterRecord): Promise<{ record: ClusterRecord }> {
|
|
807
|
+
const peerIds = Object.keys(peers);
|
|
808
|
+
const summary: ClusterLogPeerOutcome[] = [];
|
|
809
|
+
if (verbose) {
|
|
810
|
+
const peerDetail = peerIds.map(id => ({
|
|
811
|
+
id: id.substring(0, 12),
|
|
812
|
+
addrs: peers[id]?.multiaddrs?.length ?? 0
|
|
813
|
+
}));
|
|
814
|
+
log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
|
|
815
|
+
}
|
|
816
|
+
// For each peer, create a client and request a promise. A remote promise rides
|
|
817
|
+
// a libp2p stream that a relayed (limited) connection can reset transiently, so
|
|
818
|
+
// each remote request gets `promiseImmediateRetries` in-line re-attempts before
|
|
819
|
+
// it counts as a failure — without this a single relayed reset drops the peer and
|
|
820
|
+
// sinks super-majority (the commit broadcast already has the same guard).
|
|
821
|
+
const promiseRequests = peerIds.map(peerIdStr => {
|
|
822
|
+
const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
|
|
823
|
+
log('cluster-tx:promise-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
|
|
824
|
+
return new Pending(this.updateMember(peerIdStr, record, this.promiseImmediateRetries, 'promise'));
|
|
825
|
+
});
|
|
826
|
+
|
|
827
|
+
// Wait for all promises to complete
|
|
828
|
+
const results = await Promise.all(promiseRequests.map((p, idx) => p.result().then(res => {
|
|
829
|
+
const peerIdStr = peerIds[idx]!;
|
|
830
|
+
log('cluster-tx:promise-response', {
|
|
831
|
+
messageHash: record.messageHash,
|
|
832
|
+
peerId: peerIdStr,
|
|
833
|
+
success: true,
|
|
834
|
+
returnedPromises: Object.keys(res.promises ?? {}),
|
|
835
|
+
returnedCommits: Object.keys(res.commits ?? {})
|
|
836
|
+
});
|
|
837
|
+
summary.push({ peerId: peerIdStr, success: true });
|
|
838
|
+
return res;
|
|
839
|
+
}).catch(err => {
|
|
840
|
+
const peerIdStr = peerIds[idx]!;
|
|
841
|
+
log('cluster-tx:promise-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
|
|
842
|
+
summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
|
|
843
|
+
this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `promise:${record.messageHash}`);
|
|
844
|
+
return null;
|
|
845
|
+
})));
|
|
846
|
+
const successes = summary.filter(entry => entry.success).map(entry => entry.peerId);
|
|
847
|
+
const failures = summary.filter(entry => !entry.success);
|
|
848
|
+
log('cluster-tx:promise-summary', {
|
|
849
|
+
messageHash: record.messageHash,
|
|
850
|
+
successes,
|
|
851
|
+
failures
|
|
852
|
+
});
|
|
853
|
+
|
|
854
|
+
log('cluster-tx:promise-merge-begin', {
|
|
855
|
+
messageHash: record.messageHash,
|
|
856
|
+
initialPromises: Object.keys(record.promises ?? {}),
|
|
857
|
+
transactionsKeys: Array.from(this.transactions.keys()),
|
|
858
|
+
hasTransaction: this.transactions.has(record.messageHash)
|
|
859
|
+
});
|
|
860
|
+
|
|
861
|
+
// Merge all promises into the record
|
|
862
|
+
for (const result of results.filter(Boolean) as ClusterRecord[]) {
|
|
863
|
+
log('cluster-tx:promise-merge-input', {
|
|
864
|
+
messageHash: record.messageHash,
|
|
865
|
+
resultFrom: Object.keys(result.promises ?? {}),
|
|
866
|
+
recordBefore: Object.keys(record.promises ?? {})
|
|
867
|
+
});
|
|
868
|
+
const resultPromises = Object.keys(result.promises ?? {});
|
|
869
|
+
log('cluster-tx:promise-merge-result', {
|
|
870
|
+
messageHash: record.messageHash,
|
|
871
|
+
peerPromises: resultPromises
|
|
872
|
+
});
|
|
873
|
+
if (typeof record.suggestedClusterSize === 'number' && typeof result.suggestedClusterSize === 'number') {
|
|
874
|
+
const expected = result.suggestedClusterSize;
|
|
875
|
+
const actual = Object.keys(peers).length;
|
|
876
|
+
const maxDiff = Math.ceil(Math.max(1, expected * this.cfg.clusterSizeTolerance));
|
|
877
|
+
if (Math.abs(actual - expected) > maxDiff) {
|
|
878
|
+
log('cluster-tx:size-variance', { expected, actual, tolerance: this.cfg.clusterSizeTolerance });
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
record.promises = { ...record.promises, ...result.promises };
|
|
882
|
+
log('cluster-tx:promise-merge-after', {
|
|
883
|
+
messageHash: record.messageHash,
|
|
884
|
+
mergedPromises: Object.keys(record.promises ?? {})
|
|
885
|
+
});
|
|
886
|
+
}
|
|
887
|
+
log('cluster-tx:promise-merge', {
|
|
888
|
+
messageHash: record.messageHash,
|
|
889
|
+
mergedPromises: Object.keys(record.promises ?? {})
|
|
890
|
+
});
|
|
891
|
+
log('cluster-tx:promise-merge-end', {
|
|
892
|
+
messageHash: record.messageHash,
|
|
893
|
+
finalPromises: Object.keys(record.promises ?? {}),
|
|
894
|
+
transactionsEntry: this.transactions.get(record.messageHash)
|
|
895
|
+
});
|
|
896
|
+
this.updateTransactionRecord(record, 'after-promises');
|
|
897
|
+
return { record };
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
/**
|
|
901
|
+
* Commits the transaction to all peers in the cluster
|
|
902
|
+
*/
|
|
903
|
+
private async commitTransaction(record: ClusterRecord): Promise<ClusterRecord> {
|
|
904
|
+
// For each peer, create a client and send the commit
|
|
905
|
+
const peerIds = Object.keys(record.peers);
|
|
906
|
+
const summary: ClusterLogPeerOutcome[] = [];
|
|
907
|
+
if (verbose) {
|
|
908
|
+
const peerDetail = peerIds.map(id => ({
|
|
909
|
+
id: id.substring(0, 12),
|
|
910
|
+
addrs: record.peers[id]?.multiaddrs?.length ?? 0
|
|
911
|
+
}));
|
|
912
|
+
log('cluster-tx:commit-peers', { messageHash: record.messageHash, peers: peerDetail });
|
|
913
|
+
}
|
|
914
|
+
// Send the record with promises to all peers
|
|
915
|
+
// Each peer will add its own commit signature
|
|
916
|
+
const commitPayload = {
|
|
917
|
+
...record
|
|
918
|
+
};
|
|
919
|
+
// No per-peer immediate retry here: a commit-collection failure is recovered
|
|
920
|
+
// downstream by broadcastMergedRecord's in-line retry and the scheduled
|
|
921
|
+
// commit-retry timer. (The promise phase has no such backstop, which is why
|
|
922
|
+
// collectPromises gets the immediate retry instead.)
|
|
923
|
+
const commitRequests = peerIds.map(peerIdStr => {
|
|
924
|
+
const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
|
|
925
|
+
log('cluster-tx:commit-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
|
|
926
|
+
const promise = isLocal
|
|
927
|
+
? this.localCluster!.update(commitPayload)
|
|
928
|
+
: this.createClusterClient(peerIdFromString(peerIdStr)).update(commitPayload);
|
|
929
|
+
return new Pending(promise);
|
|
930
|
+
});
|
|
931
|
+
|
|
932
|
+
// Wait for all commits to complete
|
|
933
|
+
const results = await Promise.all(commitRequests.map((p, idx) => p.result().then(res => {
|
|
934
|
+
const peerIdStr = peerIds[idx]!;
|
|
935
|
+
log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: true });
|
|
936
|
+
summary.push({ peerId: peerIdStr, success: true });
|
|
937
|
+
return res;
|
|
938
|
+
}).catch(err => {
|
|
939
|
+
const peerIdStr = peerIds[idx]!;
|
|
940
|
+
log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
|
|
941
|
+
summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
|
|
942
|
+
this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `commit:${record.messageHash}`);
|
|
943
|
+
return null;
|
|
944
|
+
})));
|
|
945
|
+
const commitSuccesses = summary.filter(entry => entry.success).map(entry => entry.peerId);
|
|
946
|
+
const commitFailures = summary.filter(entry => !entry.success);
|
|
947
|
+
log('cluster-tx:commit-summary', {
|
|
948
|
+
messageHash: record.messageHash,
|
|
949
|
+
successes: commitSuccesses,
|
|
950
|
+
failures: commitFailures
|
|
951
|
+
});
|
|
952
|
+
log('cluster-tx:commit-merge-begin', {
|
|
953
|
+
messageHash: record.messageHash,
|
|
954
|
+
initialCommits: Object.keys(record.commits ?? {}),
|
|
955
|
+
transactionsEntry: this.transactions.get(record.messageHash)
|
|
956
|
+
});
|
|
957
|
+
|
|
958
|
+
// A member can reach consensus during THIS round rather than during the broadcast below (a
|
|
959
|
+
// record that already carries commits — a retried delivery), so its apply verdicts arrive on
|
|
960
|
+
// these responses. Collect both; the broadcast's copy wins on overlap, being the later of the two.
|
|
961
|
+
mergeApplyOutcomes(record, collectApplyOutcomes(results.map((response, idx) => ({ peerId: peerIds[idx]!, response }))));
|
|
962
|
+
|
|
963
|
+
// Merge all commits into the record
|
|
964
|
+
for (const result of results.filter(Boolean) as ClusterRecord[]) {
|
|
965
|
+
log('cluster-tx:commit-merge-input', {
|
|
966
|
+
messageHash: record.messageHash,
|
|
967
|
+
resultFrom: Object.keys(result.commits ?? {}),
|
|
968
|
+
recordBefore: Object.keys(record.commits ?? {})
|
|
969
|
+
});
|
|
970
|
+
log('cluster-tx:commit-merge-result', {
|
|
971
|
+
messageHash: record.messageHash,
|
|
972
|
+
peerCommits: Object.keys(result.commits ?? {})
|
|
973
|
+
});
|
|
974
|
+
record.commits = { ...record.commits, ...result.commits };
|
|
975
|
+
log('cluster-tx:commit-merge-after', {
|
|
976
|
+
messageHash: record.messageHash,
|
|
977
|
+
mergedCommits: Object.keys(record.commits ?? {})
|
|
978
|
+
});
|
|
979
|
+
}
|
|
980
|
+
log('cluster-tx:commit-merge', {
|
|
981
|
+
messageHash: record.messageHash,
|
|
982
|
+
mergedCommits: Object.keys(record.commits ?? {})
|
|
983
|
+
});
|
|
984
|
+
log('cluster-tx:commit-merge-end', {
|
|
985
|
+
messageHash: record.messageHash,
|
|
986
|
+
finalCommits: Object.keys(record.commits ?? {}),
|
|
987
|
+
transactionsEntry: this.transactions.get(record.messageHash)
|
|
988
|
+
});
|
|
989
|
+
this.updateTransactionRecord(record, 'after-commit');
|
|
990
|
+
|
|
991
|
+
// Check for simple majority (>50%) - this proves commitment
|
|
992
|
+
const peerCount = Object.keys(record.peers).length;
|
|
993
|
+
const simpleMajority = Math.floor(peerCount * this.cfg.simpleMajorityThreshold) + 1;
|
|
994
|
+
const commitCount = Object.keys(record.commits).length;
|
|
995
|
+
|
|
996
|
+
if (commitCount >= simpleMajority) {
|
|
997
|
+
log('cluster-tx:commit-majority-reached', {
|
|
998
|
+
messageHash: record.messageHash,
|
|
999
|
+
commitCount,
|
|
1000
|
+
simpleMajority,
|
|
1001
|
+
peerCount,
|
|
1002
|
+
threshold: this.cfg.simpleMajorityThreshold
|
|
1003
|
+
});
|
|
1004
|
+
// Broadcast the merged record (with all commit signatures) to ALL peers
|
|
1005
|
+
// so each peer can independently reach consensus and execute the operations.
|
|
1006
|
+
// Without this, only the coordinator's local cluster executes — remote peers
|
|
1007
|
+
// never see enough commits to reach consensus on their own.
|
|
1008
|
+
const { failures: broadcastFailures, applyOutcomes } = await this.broadcastMergedRecord(record, peerIds);
|
|
1009
|
+
mergeApplyOutcomes(record, applyOutcomes);
|
|
1010
|
+
if (broadcastFailures.length > 0) {
|
|
1011
|
+
this.scheduleCommitRetry(record.messageHash, record, broadcastFailures);
|
|
1012
|
+
} else {
|
|
1013
|
+
this.clearRetry(record.messageHash);
|
|
1014
|
+
}
|
|
1015
|
+
} else {
|
|
1016
|
+
const missingPeers = commitFailures.map(entry => entry.peerId);
|
|
1017
|
+
if (missingPeers.length > 0) {
|
|
1018
|
+
this.scheduleCommitRetry(record.messageHash, record, missingPeers);
|
|
1019
|
+
} else {
|
|
1020
|
+
this.clearRetry(record.messageHash);
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
return record;
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
/**
|
|
1027
|
+
* Broadcast the merged commit record to every peer, with `commitBroadcastImmediateRetries`
|
|
1028
|
+
* in-line re-attempts per peer before giving up. The libp2p connection used during
|
|
1029
|
+
* the prior commit phase is typically still warm, so a single immediate retry recovers
|
|
1030
|
+
* most transient stream errors without falling back to the scheduled retry timer.
|
|
1031
|
+
* Local cluster is invoked exactly once — local failures are fatal, not transient.
|
|
1032
|
+
*
|
|
1033
|
+
* **Delivery order is load-bearing: this node's own member first, awaited, then the remote
|
|
1034
|
+
* members in parallel.** This broadcast is where members apply the commit, and a member that is
|
|
1035
|
+
* behind (it never saw the pend, or holds no base for the block) reconciles the committed
|
|
1036
|
+
* revision from `record.peers` DURING its apply. The coordinator's own member is the one peer
|
|
1037
|
+
* guaranteed to hold the revision by then — provided it has actually applied, which a single
|
|
1038
|
+
* `Promise.all` over every peer did not guarantee: the remote members' reconciles raced the
|
|
1039
|
+
* local apply and found no holder. Its copy also carries the cohort's commit proof
|
|
1040
|
+
* (`buildBlockCommitProof`), which `createReconcileBlock` accepts from a single holder, so a
|
|
1041
|
+
* whole cohort of behind members can heal from it. The cost is one in-process apply before the
|
|
1042
|
+
* network fan-out; no extra round trip. The commit round in `commitTransaction` may stay
|
|
1043
|
+
* parallel: on the first pass the record it carries has no commit signatures yet, so no member
|
|
1044
|
+
* can reach consensus (and apply) there. The scheduled retry (`retryCommits`) does re-send a
|
|
1045
|
+
* record that already carries them, in parallel — but by then this node's member applied in the
|
|
1046
|
+
* first broadcast unless it was itself among the failed deliveries, which is the retry residual
|
|
1047
|
+
* documented on `executeClusterTransaction`. A coordinator outside `record.peers` is not a
|
|
1048
|
+
* reconcile target and gains nothing from this ordering; the durability gate in
|
|
1049
|
+
* `CoordinatorRepo.commit` is what makes that shape refuse rather than acknowledge.
|
|
1050
|
+
*
|
|
1051
|
+
* NOTE: when the coordinating member is ITSELF behind (it never saw the pend), its reconcile
|
|
1052
|
+
* runs here before any remote member has applied, finds no holder, and reports not-durable; the
|
|
1053
|
+
* remote members then apply and may carry the majority on their own. Fine while the coordinator
|
|
1054
|
+
* ordinarily saw the pend; if coordinators are routinely picked after the pend phase, deliver
|
|
1055
|
+
* local-first only when the local member holds the pend, or reconcile it once more afterwards.
|
|
1056
|
+
*/
|
|
1057
|
+
private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[]; applyOutcomes?: ClusterRecord['applyOutcomes'] }> {
|
|
1058
|
+
const deliver = async (peerIdStr: string) => {
|
|
1059
|
+
try {
|
|
1060
|
+
const response = await this.updateMember(peerIdStr, record, this.commitBroadcastImmediateRetries, 'commit-broadcast');
|
|
1061
|
+
return { peerId: peerIdStr, success: true as const, response };
|
|
1062
|
+
} catch (err) {
|
|
1063
|
+
log('cluster-tx:consensus-broadcast-error', {
|
|
1064
|
+
messageHash: record.messageHash,
|
|
1065
|
+
peerId: peerIdStr,
|
|
1066
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1067
|
+
});
|
|
1068
|
+
return { peerId: peerIdStr, success: false as const, response: undefined };
|
|
1069
|
+
}
|
|
1070
|
+
};
|
|
1071
|
+
const selfId = this.localCluster?.peerId.toString();
|
|
1072
|
+
const localFirst = peerIds.filter(id => id === selfId);
|
|
1073
|
+
const remote = peerIds.filter(id => id !== selfId);
|
|
1074
|
+
const localResults = await Promise.all(localFirst.map(deliver));
|
|
1075
|
+
const remoteResults = await Promise.all(remote.map(deliver));
|
|
1076
|
+
const results = [...localResults, ...remoteResults];
|
|
1077
|
+
const failures = results.filter(r => !r.success).map(r => r.peerId);
|
|
1078
|
+
// This broadcast is where members actually apply the operations, so their responses carry the
|
|
1079
|
+
// only report the coordinator ever gets of what each member's OWN storage said. Collecting it
|
|
1080
|
+
// here is what lets a pend refused by a non-coordinating member reach the writer as a conflict
|
|
1081
|
+
// instead of the fabricated success that used to fork the block.
|
|
1082
|
+
//
|
|
1083
|
+
// Each peer's entry is taken from that peer's OWN response and re-keyed under the peer we
|
|
1084
|
+
// asked, so a member cannot report an outcome on another member's behalf by echoing a record
|
|
1085
|
+
// full of entries. Unsigned and advisory either way — see ClusterRecord.applyOutcomes.
|
|
1086
|
+
const applyOutcomes = collectApplyOutcomes(results);
|
|
1087
|
+
return { failures, ...(applyOutcomes === undefined ? {} : { applyOutcomes }) };
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
/**
|
|
1091
|
+
* Fire-and-forget replay of an abandoned transaction's record to every peer in its cohort.
|
|
1092
|
+
*
|
|
1093
|
+
* Called only where the record itself proves the transaction is dead (enough signed rejections that
|
|
1094
|
+
* super-majority is unreachable). Each member re-derives `TransactionPhase.Rejected` from the votes
|
|
1095
|
+
* it verifies and drops the entry from its own reservation table, freeing the blocks immediately
|
|
1096
|
+
* instead of after its 2 s staleness window. No new message type and no wire-format change — this is
|
|
1097
|
+
* the same `update()` every other phase uses.
|
|
1098
|
+
*
|
|
1099
|
+
* Never awaited into the caller's throw and never rethrows: an abandonment must not turn into a
|
|
1100
|
+
* *different* failure, and the staleness sweep remains the backstop if delivery fails.
|
|
1101
|
+
*/
|
|
1102
|
+
private broadcastAbandonment(record: ClusterRecord, reason: string): void {
|
|
1103
|
+
const peerIds = Object.keys(record.peers);
|
|
1104
|
+
log('cluster-tx:abandon-broadcast', { messageHash: record.messageHash, reason, peerIds });
|
|
1105
|
+
void Promise.all(peerIds.map(async peerIdStr => {
|
|
1106
|
+
try {
|
|
1107
|
+
await this.updateMember(peerIdStr, record, 0, 'abandon-broadcast');
|
|
1108
|
+
} catch (err) {
|
|
1109
|
+
log('cluster-tx:abandon-broadcast-error', {
|
|
1110
|
+
messageHash: record.messageHash,
|
|
1111
|
+
peerId: peerIdStr,
|
|
1112
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1113
|
+
});
|
|
1114
|
+
}
|
|
1115
|
+
}));
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
private updateTransactionRecord(record: ClusterRecord, stage: string): void {
|
|
1119
|
+
const state = this.transactions.get(record.messageHash);
|
|
1120
|
+
if (!state) {
|
|
1121
|
+
log('cluster-tx:transaction-update-miss', { messageHash: record.messageHash, stage });
|
|
1122
|
+
return;
|
|
1123
|
+
}
|
|
1124
|
+
state.record = { ...record };
|
|
1125
|
+
state.lastUpdate = this.now();
|
|
1126
|
+
log('cluster-tx:transaction-update', {
|
|
1127
|
+
messageHash: record.messageHash,
|
|
1128
|
+
stage,
|
|
1129
|
+
promises: Object.keys(record.promises ?? {}),
|
|
1130
|
+
commits: Object.keys(record.commits ?? {})
|
|
1131
|
+
});
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
private scheduleCommitRetry(messageHash: string, _record: ClusterRecord, missingPeers: string[]): void {
|
|
1135
|
+
const state = this.transactions.get(messageHash);
|
|
1136
|
+
if (!state) {
|
|
1137
|
+
return;
|
|
1138
|
+
}
|
|
1139
|
+
const existing = state.retry;
|
|
1140
|
+
const nextAttempt = (existing?.attempt ?? 0) + 1;
|
|
1141
|
+
if (nextAttempt > this.retryMaxAttempts) {
|
|
1142
|
+
log('cluster-tx:retry-abort', { messageHash, missingPeers });
|
|
1143
|
+
return;
|
|
1144
|
+
}
|
|
1145
|
+
if (missingPeers.length === 0) {
|
|
1146
|
+
this.clearRetry(messageHash);
|
|
1147
|
+
return;
|
|
1148
|
+
}
|
|
1149
|
+
const pendingPeers = new Set(missingPeers);
|
|
1150
|
+
const baseInterval = existing ? Math.min(existing.intervalMs * this.retryBackoffFactor, this.retryMaxIntervalMs) : this.retryInitialIntervalMs;
|
|
1151
|
+
existing?.cancel?.();
|
|
1152
|
+
const cancel = this.setTimer(() => {
|
|
1153
|
+
void this.retryCommits(messageHash);
|
|
1154
|
+
}, baseInterval);
|
|
1155
|
+
state.retry = {
|
|
1156
|
+
pendingPeers,
|
|
1157
|
+
attempt: nextAttempt,
|
|
1158
|
+
intervalMs: baseInterval,
|
|
1159
|
+
cancel
|
|
1160
|
+
};
|
|
1161
|
+
this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
|
|
1162
|
+
pendingPeers: Array.from(pendingPeers),
|
|
1163
|
+
attempt: nextAttempt,
|
|
1164
|
+
intervalMs: baseInterval
|
|
1165
|
+
});
|
|
1166
|
+
log('cluster-tx:retry-scheduled', { messageHash, attempt: nextAttempt, missingPeers, delayMs: baseInterval });
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
private async retryCommits(messageHash: string): Promise<void> {
|
|
1170
|
+
const state = this.transactions.get(messageHash);
|
|
1171
|
+
if (!state?.retry) {
|
|
1172
|
+
return;
|
|
1173
|
+
}
|
|
1174
|
+
const { pendingPeers, attempt } = state.retry;
|
|
1175
|
+
if (pendingPeers.size === 0) {
|
|
1176
|
+
this.clearRetry(messageHash);
|
|
1177
|
+
return;
|
|
1178
|
+
}
|
|
1179
|
+
const peerIds = Array.from(pendingPeers);
|
|
1180
|
+
const record = state.record;
|
|
1181
|
+
log('cluster-tx:retry-start', { messageHash, attempt, peerIds });
|
|
1182
|
+
const results = await Promise.all(peerIds.map(async peerIdStr => {
|
|
1183
|
+
const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
|
|
1184
|
+
const payload: ClusterRecord = {
|
|
1185
|
+
...record,
|
|
1186
|
+
commits: record.commits
|
|
1187
|
+
};
|
|
1188
|
+
try {
|
|
1189
|
+
const res = isLocal
|
|
1190
|
+
? await this.localCluster!.update(payload)
|
|
1191
|
+
: await this.createClusterClient(peerIdFromString(peerIdStr)).update(payload);
|
|
1192
|
+
state.record.commits = { ...state.record.commits, ...res.commits };
|
|
1193
|
+
return { peerId: peerIdStr, success: true as const };
|
|
1194
|
+
} catch (err) {
|
|
1195
|
+
return {
|
|
1196
|
+
peerId: peerIdStr,
|
|
1197
|
+
success: false as const,
|
|
1198
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1199
|
+
};
|
|
1200
|
+
}
|
|
1201
|
+
}));
|
|
1202
|
+
const successes = results.filter(r => r.success).map(r => r.peerId);
|
|
1203
|
+
const failures = results.filter(r => !r.success);
|
|
1204
|
+
for (const peerId of successes) {
|
|
1205
|
+
pendingPeers.delete(peerId);
|
|
1206
|
+
}
|
|
1207
|
+
log('cluster-tx:retry-complete', { messageHash, attempt, successes, failures });
|
|
1208
|
+
if (pendingPeers.size === 0) {
|
|
1209
|
+
log('cluster-tx:retry-finished', { messageHash });
|
|
1210
|
+
this.clearRetry(messageHash);
|
|
1211
|
+
return;
|
|
1212
|
+
}
|
|
1213
|
+
if (!this.transactions.has(messageHash)) {
|
|
1214
|
+
return;
|
|
1215
|
+
}
|
|
1216
|
+
this.scheduleCommitRetry(messageHash, state.record, Array.from(pendingPeers));
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
private clearRetry(messageHash: string): void {
|
|
1220
|
+
const state = this.transactions.get(messageHash);
|
|
1221
|
+
if (!state?.retry) {
|
|
1222
|
+
return;
|
|
1223
|
+
}
|
|
1224
|
+
state.retry.cancel?.();
|
|
1225
|
+
state.retry = undefined;
|
|
1226
|
+
// Clean up the transaction after retry is complete
|
|
1227
|
+
this.setTimer(() => {
|
|
1228
|
+
this.transactions.delete(messageHash);
|
|
1229
|
+
this.deleteCoordinatorState(messageHash);
|
|
1230
|
+
log('cluster-tx:transaction-remove', {
|
|
1231
|
+
messageHash,
|
|
1232
|
+
remaining: Array.from(this.transactions.keys())
|
|
1233
|
+
});
|
|
1234
|
+
}, 100);
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1237
|
+
/** Fire-and-forget persist — errors are logged, never thrown. */
|
|
1238
|
+
private persistCoordinatorState(
|
|
1239
|
+
messageHash: string,
|
|
1240
|
+
record: ClusterRecord,
|
|
1241
|
+
phase: 'promising' | 'committing' | 'broadcasting',
|
|
1242
|
+
retryState?: { pendingPeers: string[]; attempt: number; intervalMs: number }
|
|
1243
|
+
): void {
|
|
1244
|
+
if (!this.stateStore) return;
|
|
1245
|
+
this.stateStore.saveCoordinatorState(messageHash, {
|
|
1246
|
+
messageHash,
|
|
1247
|
+
record,
|
|
1248
|
+
lastUpdate: this.now(),
|
|
1249
|
+
phase,
|
|
1250
|
+
retryState
|
|
1251
|
+
}).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
/** Fire-and-forget delete — errors are logged, never thrown. */
|
|
1255
|
+
private deleteCoordinatorState(messageHash: string): void {
|
|
1256
|
+
if (!this.stateStore) return;
|
|
1257
|
+
this.stateStore.deleteCoordinatorState(messageHash)
|
|
1258
|
+
.catch(err => log('cluster-tx:persist-delete-error', { messageHash, error: (err as Error).message }));
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
/**
|
|
1262
|
+
* Recover coordinator transactions from persistent store after a restart.
|
|
1263
|
+
* Called during node startup, before accepting new requests.
|
|
1264
|
+
*/
|
|
1265
|
+
async recoverTransactions(): Promise<void> {
|
|
1266
|
+
if (!this.stateStore) return;
|
|
1267
|
+
const states = await this.stateStore.getAllCoordinatorStates();
|
|
1268
|
+
for (const state of states) {
|
|
1269
|
+
const { messageHash } = state;
|
|
1270
|
+
// Expired — clean up
|
|
1271
|
+
if (state.record.message.expiration && state.record.message.expiration < this.now()) {
|
|
1272
|
+
log('cluster-tx:recovery-expired', { messageHash });
|
|
1273
|
+
await this.stateStore.deleteCoordinatorState(messageHash);
|
|
1274
|
+
continue;
|
|
1275
|
+
}
|
|
1276
|
+
// Broadcasting phase with retry state — resume retries
|
|
1277
|
+
if (state.phase === 'broadcasting' && state.retryState) {
|
|
1278
|
+
log('cluster-tx:recovery-resume-broadcast', { messageHash, attempt: state.retryState.attempt });
|
|
1279
|
+
const pending = new Pending(Promise.resolve(state.record));
|
|
1280
|
+
const txState: ClusterTransactionState = {
|
|
1281
|
+
messageHash,
|
|
1282
|
+
record: state.record,
|
|
1283
|
+
pending,
|
|
1284
|
+
lastUpdate: state.lastUpdate
|
|
1285
|
+
};
|
|
1286
|
+
this.transactions.set(messageHash, txState);
|
|
1287
|
+
// Schedule retry from where we left off
|
|
1288
|
+
this.scheduleCommitRetry(messageHash, state.record, state.retryState.pendingPeers);
|
|
1289
|
+
continue;
|
|
1290
|
+
}
|
|
1291
|
+
// Promising or committing — cannot resume (caller context is gone)
|
|
1292
|
+
log('cluster-tx:recovery-stale', { messageHash, phase: state.phase });
|
|
1293
|
+
await this.stateStore.deleteCoordinatorState(messageHash);
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
}
|