@optimystic/db-p2p 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/cluster-repo.d.ts +58 -9
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +113 -21
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +100 -31
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +288 -192
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +2 -0
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +7 -5
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +24 -6
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/package.json +2 -2
- package/src/cluster/cluster-repo.ts +116 -21
- package/src/repo/cluster-coordinator.ts +1409 -1296
- package/src/repo/coordinator-repo.ts +9 -5
- package/src/storage/storage-repo.ts +1781 -1762
|
@@ -1,1296 +1,1409 @@
|
|
|
1
|
-
import { peerIdFromString } from "@libp2p/peer-id";
|
|
2
|
-
import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult, StaleFailure } from "@optimystic/db-core";
|
|
3
|
-
import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, isConflictFailure, membershipDigest, routingKeyForBlock } from "@optimystic/db-core";
|
|
4
|
-
import { Pending } from "@optimystic/db-core";
|
|
5
|
-
import type { PeerId } from "@libp2p/interface";
|
|
6
|
-
import { createLogger, verbose } from '../logger.js'
|
|
7
|
-
import type { ClusterLogPeerOutcome } from './types.js'
|
|
8
|
-
import type { FretService } from "p2p-fret";
|
|
9
|
-
import type { IPeerReputation } from "../reputation/types.js";
|
|
10
|
-
import { PenaltyReason } from "../reputation/types.js";
|
|
11
|
-
import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
|
|
12
|
-
import { ResponsibilityRefusalError } from "./responsibility.js";
|
|
13
|
-
|
|
14
|
-
const log = createLogger('cluster')
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* Pick each peer's OWN {@link ClusterRecord.applyOutcomes} entry out of the record that peer answered
|
|
18
|
-
* with, and key it under the peer we actually asked.
|
|
19
|
-
*
|
|
20
|
-
* Taking only `response.applyOutcomes[peerId]` — rather than spreading the whole map — is what keeps
|
|
21
|
-
* one member from reporting outcomes on other members' behalf: a peer that echoes back a record full
|
|
22
|
-
* of entries contributes exactly one, its own. The field is unsigned advisory data (see its doc
|
|
23
|
-
* comment for why that is safe), so this is a shaping rule, not a security boundary.
|
|
24
|
-
*
|
|
25
|
-
* Returns `undefined` when no peer reported anything, so the common case adds no empty object to the
|
|
26
|
-
* record.
|
|
27
|
-
*/
|
|
28
|
-
function collectApplyOutcomes(
|
|
29
|
-
responses: ReadonlyArray<{ peerId: string; response?: ClusterRecord | null }>
|
|
30
|
-
): ClusterRecord['applyOutcomes'] {
|
|
31
|
-
let collected: NonNullable<ClusterRecord['applyOutcomes']> | undefined;
|
|
32
|
-
for (const { peerId, response } of responses) {
|
|
33
|
-
const own = response?.applyOutcomes?.[peerId];
|
|
34
|
-
if (own === undefined) continue;
|
|
35
|
-
collected ??= {};
|
|
36
|
-
collected[peerId] = own;
|
|
37
|
-
}
|
|
38
|
-
return collected;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
/** Fold collected outcomes into a record in place, later report winning per peer. No-op for `undefined`. */
|
|
42
|
-
function mergeApplyOutcomes(record: ClusterRecord, collected: ClusterRecord['applyOutcomes']): void {
|
|
43
|
-
if (collected === undefined) return;
|
|
44
|
-
record.applyOutcomes = { ...record.applyOutcomes, ...collected };
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
*
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
*
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
* The
|
|
129
|
-
*
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
*/
|
|
143
|
-
export
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
/**
|
|
171
|
-
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
*/
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
*
|
|
261
|
-
*
|
|
262
|
-
*
|
|
263
|
-
*
|
|
264
|
-
*
|
|
265
|
-
*
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
*
|
|
297
|
-
*
|
|
298
|
-
*
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
*
|
|
312
|
-
* the
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
*
|
|
331
|
-
*
|
|
332
|
-
*
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
const
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
return
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
record: ClusterRecord
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
*
|
|
423
|
-
*
|
|
424
|
-
*
|
|
425
|
-
*
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
//
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
//
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
this.
|
|
489
|
-
this.
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
const
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
//
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
log('cluster-tx:
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
});
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
//
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
//
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
}
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
}
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
log('cluster-tx:promise-merge-
|
|
892
|
-
messageHash: record.messageHash,
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
//
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
record.
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
this.updateTransactionRecord(record, 'after-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
}
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
}
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
*
|
|
1034
|
-
*
|
|
1035
|
-
*
|
|
1036
|
-
*
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
*
|
|
1096
|
-
*
|
|
1097
|
-
*
|
|
1098
|
-
*
|
|
1099
|
-
*
|
|
1100
|
-
*
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
const
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
if (
|
|
1146
|
-
this.
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
const
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
state.
|
|
1225
|
-
state
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
this.
|
|
1258
|
-
.
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1
|
+
import { peerIdFromString } from "@libp2p/peer-id";
|
|
2
|
+
import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult, StaleFailure } from "@optimystic/db-core";
|
|
3
|
+
import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, isConflictFailure, membershipDigest, routingKeyForBlock } from "@optimystic/db-core";
|
|
4
|
+
import { Pending } from "@optimystic/db-core";
|
|
5
|
+
import type { PeerId } from "@libp2p/interface";
|
|
6
|
+
import { createLogger, verbose } from '../logger.js'
|
|
7
|
+
import type { ClusterLogPeerOutcome } from './types.js'
|
|
8
|
+
import type { FretService } from "p2p-fret";
|
|
9
|
+
import type { IPeerReputation } from "../reputation/types.js";
|
|
10
|
+
import { PenaltyReason } from "../reputation/types.js";
|
|
11
|
+
import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
|
|
12
|
+
import { ResponsibilityRefusalError } from "./responsibility.js";
|
|
13
|
+
|
|
14
|
+
const log = createLogger('cluster')
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Pick each peer's OWN {@link ClusterRecord.applyOutcomes} entry out of the record that peer answered
|
|
18
|
+
* with, and key it under the peer we actually asked.
|
|
19
|
+
*
|
|
20
|
+
* Taking only `response.applyOutcomes[peerId]` — rather than spreading the whole map — is what keeps
|
|
21
|
+
* one member from reporting outcomes on other members' behalf: a peer that echoes back a record full
|
|
22
|
+
* of entries contributes exactly one, its own. The field is unsigned advisory data (see its doc
|
|
23
|
+
* comment for why that is safe), so this is a shaping rule, not a security boundary.
|
|
24
|
+
*
|
|
25
|
+
* Returns `undefined` when no peer reported anything, so the common case adds no empty object to the
|
|
26
|
+
* record.
|
|
27
|
+
*/
|
|
28
|
+
function collectApplyOutcomes(
|
|
29
|
+
responses: ReadonlyArray<{ peerId: string; response?: ClusterRecord | null }>
|
|
30
|
+
): ClusterRecord['applyOutcomes'] {
|
|
31
|
+
let collected: NonNullable<ClusterRecord['applyOutcomes']> | undefined;
|
|
32
|
+
for (const { peerId, response } of responses) {
|
|
33
|
+
const own = response?.applyOutcomes?.[peerId];
|
|
34
|
+
if (own === undefined) continue;
|
|
35
|
+
collected ??= {};
|
|
36
|
+
collected[peerId] = own;
|
|
37
|
+
}
|
|
38
|
+
return collected;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Fold collected outcomes into a record in place, later report winning per peer. No-op for `undefined`. */
|
|
42
|
+
function mergeApplyOutcomes(record: ClusterRecord, collected: ClusterRecord['applyOutcomes']): void {
|
|
43
|
+
if (collected === undefined) return;
|
|
44
|
+
record.applyOutcomes = { ...record.applyOutcomes, ...collected };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Fold the commit signatures of every answered response into a record in place. */
|
|
48
|
+
function mergeCommits(record: ClusterRecord, responses: ReadonlyArray<{ response?: ClusterRecord | null }>): void {
|
|
49
|
+
for (const { response } of responses) {
|
|
50
|
+
if (response) record.commits = { ...record.commits, ...response.commits };
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** One member's answer to a delivery; `response` is absent, and `error` present, when it failed. */
|
|
55
|
+
interface MemberDelivery {
|
|
56
|
+
peerId: string;
|
|
57
|
+
success: boolean;
|
|
58
|
+
response?: ClusterRecord;
|
|
59
|
+
error?: string;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* The members of `deliveries` that still need the consensus record: one whose delivery failed, one
|
|
64
|
+
* whose response does not report having run the consensus apply (`MemberApplyOutcome.executed`,
|
|
65
|
+
* which a member on an older build never sets), and one that reports a refused commit —
|
|
66
|
+
* sending it the record again gives a behind member another reconcile once the coordinating member
|
|
67
|
+
* holds the revision (`ClusterMember.handleAlreadyExecuted`). Each member is judged by its own entry
|
|
68
|
+
* in its own response, as {@link collectApplyOutcomes} takes it.
|
|
69
|
+
*/
|
|
70
|
+
function membersAwaitingConsensus(deliveries: readonly MemberDelivery[]): string[] {
|
|
71
|
+
return deliveries
|
|
72
|
+
.filter(({ peerId, response }) => {
|
|
73
|
+
const own = response?.applyOutcomes?.[peerId];
|
|
74
|
+
return own?.executed !== true || own.commit?.success === false;
|
|
75
|
+
})
|
|
76
|
+
.map(({ peerId }) => peerId);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Consensus refused a transaction: enough members voted reject that super-majority became
|
|
81
|
+
* impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
|
|
82
|
+
* "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
|
|
83
|
+
* rejection reasons — those are free-form text that is part of each member's signed vote payload
|
|
84
|
+
* (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
|
|
85
|
+
* `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
|
|
86
|
+
* loss (confirmed against local storage) or a genuine validation fault.
|
|
87
|
+
*/
|
|
88
|
+
export class ValidatorRejectionError extends Error {
|
|
89
|
+
constructor(
|
|
90
|
+
message: string,
|
|
91
|
+
/** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
|
|
92
|
+
readonly rejectReasons: Record<string, string>
|
|
93
|
+
) {
|
|
94
|
+
super(message);
|
|
95
|
+
this.name = 'ValidatorRejectionError';
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The transaction lost a conflict race: one or more members answered with a signed `conflict`
|
|
101
|
+
* vote (they hold a rival transaction that won the deterministic race on the same blocks) and
|
|
102
|
+
* approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} — nobody
|
|
103
|
+
* judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
|
|
104
|
+
* `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
|
|
105
|
+
* `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
|
|
106
|
+
* as a thrown error only from other paths. The commit conversion matters as much as the pend one:
|
|
107
|
+
* at the moment this is thrown zero members approved and the members hold the winner — nothing of
|
|
108
|
+
* the loser landed — yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
|
|
109
|
+
* (it treats throws as transport faults), and that re-driven commit races into the window after
|
|
110
|
+
* members apply the winner and clear its reservation, where it can assemble a consensus no member
|
|
111
|
+
* will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
|
|
112
|
+
* writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
|
|
113
|
+
* and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
|
|
114
|
+
* parsed out of prose.
|
|
115
|
+
*/
|
|
116
|
+
export class ConflictRaceLostError extends Error {
|
|
117
|
+
constructor(
|
|
118
|
+
message: string,
|
|
119
|
+
/** peerId → messageHash of the rival transaction that member holds as the race winner. */
|
|
120
|
+
readonly conflicts: Record<string, string>
|
|
121
|
+
) {
|
|
122
|
+
super(message);
|
|
123
|
+
this.name = 'ConflictRaceLostError';
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* The transaction's pend could not proceed because one or more members answered with a signed `held`
|
|
129
|
+
* vote: the requested blocks are reserved by a different unresolved action in that member's durable
|
|
130
|
+
* storage. Sibling of {@link ConflictRaceLostError} and retryable for the same reason — nobody judged
|
|
131
|
+
* this write invalid; it queued behind a reservation that disappears when the holder commits or
|
|
132
|
+
* cancels.
|
|
133
|
+
*
|
|
134
|
+
* The two are separate because they name different things. A conflict vote names the winning rival's
|
|
135
|
+
* `messageHash`, which the member holds whole; a held vote can only name the rival's **action id**,
|
|
136
|
+
* because it fires in the window where the rival has left the member's in-memory table but not yet its
|
|
137
|
+
* storage. `CoordinatorRepo.pend` converts this into a `StaleFailure` with `conflict: true` so the
|
|
138
|
+
* normal retry machinery (`isConflictFailure`) absorbs it, exactly as it does a lost race.
|
|
139
|
+
*
|
|
140
|
+
* Only a PEND record can produce it: `held` votes come from `ClusterMember.validatePendOperations`,
|
|
141
|
+
* which inspects pend operations only, so `CoordinatorRepo.commit` never meets one.
|
|
142
|
+
*/
|
|
143
|
+
export class BlocksHeldError extends Error {
|
|
144
|
+
constructor(
|
|
145
|
+
message: string,
|
|
146
|
+
/** peerId → actionId of the unresolved action that member's storage says holds the blocks. */
|
|
147
|
+
readonly heldBy: Record<string, string>
|
|
148
|
+
) {
|
|
149
|
+
super(message);
|
|
150
|
+
this.name = 'BlocksHeldError';
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
|
|
155
|
+
export type TimerCancel = () => void;
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Production timer binding: a one-shot `setTimeout` whose handle is **unref'd** so a pending
|
|
159
|
+
* commit-retry (or the deferred transaction cleanup) never keeps an otherwise-idle process alive.
|
|
160
|
+
* The returned handle clears the timeout (idempotent). Mirrors the reactivity rotation
|
|
161
|
+
* re-registration scheduler's `defaultSetTimer` (see reactivity/rotation-rereg-scheduler.ts).
|
|
162
|
+
*/
|
|
163
|
+
function defaultSetTimer(fn: () => void, delayMs: number): TimerCancel {
|
|
164
|
+
const handle = setTimeout(fn, delayMs);
|
|
165
|
+
// An idle retry/cleanup timer must not pin a process (mirror rotation re-registration + push-state gossip).
|
|
166
|
+
(handle as { unref?: () => void }).unref?.();
|
|
167
|
+
return (): void => clearTimeout(handle);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Optional injection seam for deterministic time. Production leaves both undefined and gets
|
|
172
|
+
* `Date.now` + an unref'd `setTimeout`; tests inject a fake clock + timer queue so scheduled
|
|
173
|
+
* commit-retries fire in virtual (not wall-clock) time.
|
|
174
|
+
*/
|
|
175
|
+
export interface ClusterCoordinatorClock {
|
|
176
|
+
/** Clock (Unix ms). Defaults to `Date.now`. */
|
|
177
|
+
now?: () => number;
|
|
178
|
+
/** Schedule a one-shot timer, returning a cancel handle. Defaults to an unref'd `setTimeout`. */
|
|
179
|
+
setTimer?: (fn: () => void, delayMs: number) => TimerCancel;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Manages the state of cluster transactions for a specific block ID
|
|
184
|
+
*/
|
|
185
|
+
interface CommitRetryState {
|
|
186
|
+
pendingPeers: Set<string>;
|
|
187
|
+
attempt: number;
|
|
188
|
+
intervalMs: number;
|
|
189
|
+
cancel?: TimerCancel;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
interface ClusterTransactionState {
|
|
193
|
+
messageHash: string;
|
|
194
|
+
record: ClusterRecord;
|
|
195
|
+
pending: Pending<ClusterRecord>;
|
|
196
|
+
lastUpdate: number;
|
|
197
|
+
promiseTimeout?: NodeJS.Timeout;
|
|
198
|
+
resolutionTimeout?: NodeJS.Timeout;
|
|
199
|
+
retry?: CommitRetryState;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** Manages distributed transactions across clusters */
|
|
203
|
+
/**
|
|
204
|
+
* What a cohort lookup established about a block's cohort. `resolved: false` covers BOTH a lookup
|
|
205
|
+
* that threw and one that answered with nobody: neither names a destination for a write, and the
|
|
206
|
+
* durability class both produce is the same (`unrouted`). `reason` is for logs only — never branch
|
|
207
|
+
* on it.
|
|
208
|
+
*/
|
|
209
|
+
export type CohortResolution =
|
|
210
|
+
| { readonly resolved: true; readonly peerIds: readonly string[] }
|
|
211
|
+
| { readonly resolved: false; readonly reason: string };
|
|
212
|
+
|
|
213
|
+
export class ClusterCoordinator {
|
|
214
|
+
private transactions: Map<string, ClusterTransactionState> = new Map();
|
|
215
|
+
private readonly retryInitialIntervalMs: number;
|
|
216
|
+
private readonly retryBackoffFactor: number;
|
|
217
|
+
private readonly retryMaxIntervalMs: number;
|
|
218
|
+
private readonly retryMaxAttempts: number;
|
|
219
|
+
private readonly commitBroadcastImmediateRetries: number;
|
|
220
|
+
private readonly promiseImmediateRetries: number;
|
|
221
|
+
/** Injected clock/timer seam; production defaults to `Date.now` + unref'd `setTimeout`. */
|
|
222
|
+
private readonly now: () => number;
|
|
223
|
+
private readonly setTimer: (fn: () => void, delayMs: number) => TimerCancel;
|
|
224
|
+
|
|
225
|
+
constructor(
|
|
226
|
+
private readonly keyNetwork: IKeyNetwork,
|
|
227
|
+
/** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
|
|
228
|
+
private readonly createClusterClient: (peerId: PeerId) => ICluster,
|
|
229
|
+
private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
|
|
230
|
+
private readonly localCluster?: {
|
|
231
|
+
update: (record: ClusterRecord) => Promise<ClusterRecord>;
|
|
232
|
+
peerId: PeerId;
|
|
233
|
+
wasTransactionExecuted?: (messageHash: string) => boolean;
|
|
234
|
+
/** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
|
|
235
|
+
getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
|
|
236
|
+
/** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
|
|
237
|
+
getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
|
|
238
|
+
/** One more reconcile for a behind-refused commit, once remote members hold it; see ClusterMember.reconcileRefusedCommit. */
|
|
239
|
+
reconcileRefusedCommit?: (record: ClusterRecord) => Promise<void>;
|
|
240
|
+
},
|
|
241
|
+
private readonly fretService?: FretService,
|
|
242
|
+
private readonly reputation?: IPeerReputation,
|
|
243
|
+
private readonly stateStore?: ITransactionStateStore,
|
|
244
|
+
clock?: ClusterCoordinatorClock
|
|
245
|
+
) {
|
|
246
|
+
this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
|
|
247
|
+
this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
|
|
248
|
+
this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
|
|
249
|
+
this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
|
|
250
|
+
this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
|
|
251
|
+
this.promiseImmediateRetries = cfg.promiseImmediateRetries ?? 1;
|
|
252
|
+
this.now = clock?.now ?? ((): number => Date.now());
|
|
253
|
+
this.setTimer = clock?.setTimer ?? defaultSetTimer;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Invoke one cluster member's `update`, retrying transient REMOTE failures up to
|
|
258
|
+
* `immediateRetries` times before surfacing the error. The local cluster is invoked
|
|
259
|
+
* exactly once — a local throw is a real fault (validation / merge / consensus), not a
|
|
260
|
+
* transient transport blip. A remote call rides a libp2p stream that a circuit-relay
|
|
261
|
+
* ("limited") connection can reset once a per-circuit cap or reservation lapses, which
|
|
262
|
+
* surfaces as a StreamResetError; an immediate retry on the (usually still-warm)
|
|
263
|
+
* connection recovers most of those without escalating the peer to a failure. Shared by
|
|
264
|
+
* the promise-collection, commit-collection, and commit-broadcast phases so all three
|
|
265
|
+
* react to a relayed reset the same way.
|
|
266
|
+
*/
|
|
267
|
+
private async updateMember(peerIdStr: string, record: ClusterRecord, immediateRetries: number, phase: string): Promise<ClusterRecord> {
|
|
268
|
+
const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
|
|
269
|
+
if (isLocal) {
|
|
270
|
+
return await this.localCluster!.update(record);
|
|
271
|
+
}
|
|
272
|
+
const maxAttempts = 1 + Math.max(0, immediateRetries);
|
|
273
|
+
let lastError: unknown;
|
|
274
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
275
|
+
try {
|
|
276
|
+
return await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
|
|
277
|
+
} catch (err) {
|
|
278
|
+
lastError = err;
|
|
279
|
+
if (attempt < maxAttempts) {
|
|
280
|
+
log('cluster-tx:member-update-retry', {
|
|
281
|
+
messageHash: record.messageHash,
|
|
282
|
+
peerId: peerIdStr,
|
|
283
|
+
phase,
|
|
284
|
+
attempt,
|
|
285
|
+
error: err instanceof Error ? err.message : String(err)
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
throw lastError;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Creates a base58btc string hash uniquely identifying a transaction. For a v2 record the caller
|
|
295
|
+
* threads in the {@link membershipDigest} of the peer set so the responsible membership is bound into
|
|
296
|
+
* the identity (two different peer sets ⇒ two different hashes). Omitting `membershipDigestValue`
|
|
297
|
+
* reproduces the legacy v1 hash byte-for-byte.
|
|
298
|
+
*
|
|
299
|
+
* NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
|
|
300
|
+
* which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
|
|
301
|
+
* `pend.priority` (single-collection) — is automatically covered here and by the derived
|
|
302
|
+
* promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
|
|
303
|
+
* cannot strip or inflate it without invalidating the message hash the members verify. No separate
|
|
304
|
+
* priority-hashing step is needed.
|
|
305
|
+
*/
|
|
306
|
+
private async createMessageHash(message: RepoMessage, membershipDigestValue?: string): Promise<string> {
|
|
307
|
+
return computeClusterMessageHash(message, membershipDigestValue);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* The ONE cohort lookup every accessor on this class derives from: the raw peer map when the key
|
|
312
|
+
* network answered, otherwise the reason it did not. A thrown `findCluster` is logged here and
|
|
313
|
+
* nowhere else. Callers that need the map (`executeClusterTransaction`, which builds the record's
|
|
314
|
+
* `peers`) go through {@link getClusterForBlock}; callers that need to know whether the cohort
|
|
315
|
+
* RESOLVED go through {@link resolveCohort}.
|
|
316
|
+
*/
|
|
317
|
+
private async lookupCluster(blockId: BlockId): Promise<{ peers: ClusterPeers } | { reason: string }> {
|
|
318
|
+
try {
|
|
319
|
+
const peers = await this.keyNetwork.findCluster(routingKeyForBlock(blockId));
|
|
320
|
+
const peerIds = Object.keys(peers ?? {});
|
|
321
|
+
log('cluster-tx:cluster-members', { blockId, peerIds });
|
|
322
|
+
return { peers: peers ?? {} };
|
|
323
|
+
} catch (e) {
|
|
324
|
+
log('WARN findCluster failed for %s: %o', blockId, e)
|
|
325
|
+
return { reason: `findCluster threw: ${(e as Error)?.message ?? String(e)}` };
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* Gets all peers in the cluster for a specific block ID. Empty when the lookup failed — the
|
|
331
|
+
* consensus path treats "no cohort" and "lookup failed" alike (there is nobody to run consensus
|
|
332
|
+
* with either way); a caller that must tell them apart uses {@link resolveCohort}.
|
|
333
|
+
*/
|
|
334
|
+
private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
|
|
335
|
+
const outcome = await this.lookupCluster(blockId);
|
|
336
|
+
return 'peers' in outcome ? outcome.peers : {};
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* Whether the block's cohort could be established, and who it is. The primitive behind
|
|
341
|
+
* {@link getClusterPeerIds} and {@link getClusterSize}: a lookup that threw and a lookup that named
|
|
342
|
+
* nobody used to reach every caller as the same empty list, and `CoordinatorRepo`'s solo
|
|
343
|
+
* short-circuit then acknowledged a write it had no idea where to send exactly as it acknowledged a
|
|
344
|
+
* write to a genuine cohort of one (GitHub #19). Both shapes are still `resolved: false` here —
|
|
345
|
+
* neither names a destination — but they are distinguishable from a resolved cohort, which is what
|
|
346
|
+
* the write's durability class needs (`unrouted` vs `local`).
|
|
347
|
+
*/
|
|
348
|
+
async resolveCohort(blockId: BlockId): Promise<CohortResolution> {
|
|
349
|
+
const outcome = await this.lookupCluster(blockId);
|
|
350
|
+
if ('reason' in outcome) return { resolved: false, reason: outcome.reason };
|
|
351
|
+
const peerIds = Object.keys(outcome.peers);
|
|
352
|
+
if (peerIds.length === 0) return { resolved: false, reason: 'findCluster named nobody' };
|
|
353
|
+
return { resolved: true, peerIds };
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/**
|
|
357
|
+
* A node never runs a cluster transaction for a cohort it is not in. Behind members reconcile from the
|
|
358
|
+
* coordinator's own proof-carrying copy (its member applies before the consensus broadcast, and a
|
|
359
|
+
* member that applied earlier, on receipt of the commit round, is sent the record again once it has),
|
|
360
|
+
* and a coordinator outside `record.peers` is not a reconcile target — so a cohort with no holder would stay
|
|
361
|
+
* behind and the commit durability gate would refuse, having first put this node's vote and storage
|
|
362
|
+
* where the cohort does not look. The invariant is held here, at the one place a record's `peers` is
|
|
363
|
+
* chosen, rather than left to the routing convention.
|
|
364
|
+
*
|
|
365
|
+
* Fires only on a RESOLVED cohort (at least one peer) that excludes the wired local member. An empty
|
|
366
|
+
* cohort is a failed lookup, not a cohort this node is outside of, so it is left to `executeTransaction`'s
|
|
367
|
+
* size checks; `CoordinatorRepo`'s solo short-circuit keeps unresolved and single-peer cohorts away from
|
|
368
|
+
* this method altogether in any case. After its responsibility check, what remains is a multi-member
|
|
369
|
+
* cohort that changed inside the responsibility cache's staleness window. With no local member wired the guard does not apply: that
|
|
370
|
+
* bypass exists for wiring without an identity (direct constructors, some tests), never for production.
|
|
371
|
+
*/
|
|
372
|
+
private assertLocalMemberInCohort(blockId: BlockId, peers: ClusterPeers): void {
|
|
373
|
+
if (!this.localCluster) return;
|
|
374
|
+
const peerIds = Object.keys(peers);
|
|
375
|
+
const selfId = this.localCluster.peerId.toString();
|
|
376
|
+
if (peerIds.length === 0 || peerIds.includes(selfId)) return;
|
|
377
|
+
log('cluster-tx:not-in-cohort', { blockId, selfId, peerIds });
|
|
378
|
+
throw new ResponsibilityRefusalError('not-responsible', [blockId],
|
|
379
|
+
`refusing to coordinate a cluster transaction for a cohort this node is not in: ${peerIds.join(', ')}`);
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
|
|
383
|
+
const peerCount = Object.keys(peers ?? {}).length;
|
|
384
|
+
const record: ClusterRecord = {
|
|
385
|
+
messageHash,
|
|
386
|
+
peers,
|
|
387
|
+
// v2: bind the responsible membership into the signed identity. messageHash was computed over
|
|
388
|
+
// this same digest, so a different peer set would have produced a different messageHash.
|
|
389
|
+
membershipVersion: CURRENT_MEMBERSHIP_VERSION,
|
|
390
|
+
membershipDigest: membershipDigestValue,
|
|
391
|
+
message,
|
|
392
|
+
promises: {},
|
|
393
|
+
commits: {},
|
|
394
|
+
suggestedClusterSize: peerCount || undefined,
|
|
395
|
+
minRequiredSize: this.cfg.allowClusterDownsize ? undefined : this.cfg.clusterSize
|
|
396
|
+
};
|
|
397
|
+
|
|
398
|
+
// Add network size hint if available
|
|
399
|
+
if (this.fretService) {
|
|
400
|
+
try {
|
|
401
|
+
const estimate = this.fretService.getNetworkSizeEstimate();
|
|
402
|
+
if (estimate.size_estimate > 0) {
|
|
403
|
+
record.networkSizeHint = estimate.size_estimate;
|
|
404
|
+
record.networkSizeConfidence = estimate.confidence;
|
|
405
|
+
}
|
|
406
|
+
} catch (err) {
|
|
407
|
+
// Ignore errors getting size estimate
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
return record;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/**
|
|
415
|
+
* Initiates a 2-phase transaction for a specific block ID.
|
|
416
|
+
* Returns the cluster record and whether the local cluster already executed the operations.
|
|
417
|
+
*/
|
|
418
|
+
async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
|
|
419
|
+
record: ClusterRecord;
|
|
420
|
+
localExecuted: boolean;
|
|
421
|
+
/**
|
|
422
|
+
* Local storage's verdict for a pend operation this node's own cluster member applied during
|
|
423
|
+
* consensus, when the member retained one. Meaningful only when `localExecuted` is true;
|
|
424
|
+
* absent for non-pend messages, for a member that predates the retention, or after the
|
|
425
|
+
* retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
|
|
426
|
+
*/
|
|
427
|
+
localPendResult?: PendResult;
|
|
428
|
+
/**
|
|
429
|
+
* Local storage's verdict for a commit operation this node's own cluster member applied
|
|
430
|
+
* during consensus, when the member retained one. Same availability contract as
|
|
431
|
+
* `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
|
|
432
|
+
* win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
|
|
433
|
+
* success no member durably stored. Read after the consensus broadcast, so a behind member's
|
|
434
|
+
* verdict already reflects the reconcile it ran against the remote members that applied in the
|
|
435
|
+
* commit round, and any second one `broadcastMergedRecord` gave it.
|
|
436
|
+
*/
|
|
437
|
+
localCommitResult?: CommitResult;
|
|
438
|
+
/**
|
|
439
|
+
* Conflict-shaped pend refusals reported by OTHER cohort members on their consensus responses
|
|
440
|
+
* (`ClusterRecord.applyOutcomes`), keyed by peer id. This is the arm `localPendResult` cannot
|
|
441
|
+
* cover: the refusing member is frequently not the coordinating node, and its verdict used to
|
|
442
|
+
* stay on that member while the writer was told the pend won. Unsigned advisory data — an
|
|
443
|
+
* entry means "retry", never "this write was invalid". Absent when nobody reported one.
|
|
444
|
+
*
|
|
445
|
+
* Residual: a member that reaches consensus only via the scheduled commit-retry timer applies
|
|
446
|
+
* after this method has already resolved, so its refusal arrives too late to appear here. The
|
|
447
|
+
* member-side commit-promise guard (`validateCommitAgainstRefusedPend`) is the backstop for
|
|
448
|
+
* that path.
|
|
449
|
+
*/
|
|
450
|
+
cohortPendRefusals?: { [peerId: string]: StaleFailure };
|
|
451
|
+
/**
|
|
452
|
+
* What OTHER cohort members reported about durably holding a commit after applying it at
|
|
453
|
+
* consensus (`ClusterRecord.applyOutcomes[peer].commit`), keyed by peer id — successes AND
|
|
454
|
+
* refusals, because `CoordinatorRepo.commit`'s durability gate counts the successes against
|
|
455
|
+
* the cohort the commit ran on and acknowledges only a majority. Each member's verdict is
|
|
456
|
+
* measured after its own reconcile, so a member that pulled the revision from a cohort peer
|
|
457
|
+
* reports success. Self is excluded for the same reason as `cohortPendRefusals` (its verdict
|
|
458
|
+
* travels as `localCommitResult`). Unsigned advisory data: a false success is one holder the
|
|
459
|
+
* member's signed approve vote already admitted to the majority; a false refusal is retry
|
|
460
|
+
* pressure. Absent when nobody reported one (a pend message, or pre-upgrade members).
|
|
461
|
+
*
|
|
462
|
+
* Same residual as `cohortPendRefusals`: a member reached only by the scheduled commit-retry
|
|
463
|
+
* timer applies after this method has resolved, and its report arrives too late to count —
|
|
464
|
+
* the gate then refuses honestly and the writer re-drives.
|
|
465
|
+
*/
|
|
466
|
+
cohortCommitOutcomes?: { [peerId: string]: CommitResult };
|
|
467
|
+
}> {
|
|
468
|
+
// The coordinating block id is derived HERE, from the key this method is already handed, rather
|
|
469
|
+
// than being set by each caller's message builder: a member's membership admission gate derives
|
|
470
|
+
// its own cohort view from this field, and a builder that forgets it silently downgrades the gate
|
|
471
|
+
// to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
|
|
472
|
+
// admitted at pend, refused at commit). Doing it at the single choke point means a future message
|
|
473
|
+
// builder cannot reintroduce the gap.
|
|
474
|
+
//
|
|
475
|
+
// Two constraints this shape exists to satisfy:
|
|
476
|
+
// - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
|
|
477
|
+
// N concurrent calls, one per block. In-place mutation would leak one block's id into another
|
|
478
|
+
// block's transaction.
|
|
479
|
+
// - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
|
|
480
|
+
// not just its first block, so this must not overwrite it. Tested on `length`, not on the
|
|
481
|
+
// field: an empty list carries no id for a member to derive from, so preserving one would be
|
|
482
|
+
// the same silent downgrade to the fallback floor this choke point exists to prevent.
|
|
483
|
+
const coordinated: RepoMessage = message.coordinatingBlockIds?.length
|
|
484
|
+
? message
|
|
485
|
+
: { ...message, coordinatingBlockIds: [blockId] };
|
|
486
|
+
|
|
487
|
+
// Get the cluster peers for this block
|
|
488
|
+
const peers = await this.getClusterForBlock(blockId);
|
|
489
|
+
this.assertLocalMemberInCohort(blockId, peers);
|
|
490
|
+
|
|
491
|
+
// Bind the responsible membership into the transaction identity (v2): the digest is folded into
|
|
492
|
+
// the messageHash below, so two different peer sets produce two different messageHashes rather
|
|
493
|
+
// than one hash with a silent internal disagreement about who is responsible.
|
|
494
|
+
const membershipDigestValue = await membershipDigest(peers);
|
|
495
|
+
|
|
496
|
+
// Create a unique hash for this transaction (over message + membership digest). Hashing the
|
|
497
|
+
// coordinating-block-bearing copy is what makes the field tamper-evident in transit — and it also
|
|
498
|
+
// makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
|
|
499
|
+
// identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
|
|
500
|
+
const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
|
|
501
|
+
|
|
502
|
+
// Create a cluster record for this transaction
|
|
503
|
+
const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
|
|
504
|
+
log('cluster-tx:start', {
|
|
505
|
+
messageHash,
|
|
506
|
+
blockId,
|
|
507
|
+
peerCount: Object.keys(peers ?? {}).length,
|
|
508
|
+
allowDownsize: this.cfg.allowClusterDownsize,
|
|
509
|
+
configuredSize: this.cfg.clusterSize,
|
|
510
|
+
suggestedSize: record.suggestedClusterSize,
|
|
511
|
+
minRequiredSize: record.minRequiredSize
|
|
512
|
+
});
|
|
513
|
+
|
|
514
|
+
// Create a new pending transaction
|
|
515
|
+
const transactionPromise = this.executeTransaction(peers, record);
|
|
516
|
+
const pending = new Pending(transactionPromise);
|
|
517
|
+
|
|
518
|
+
// Store the transaction state
|
|
519
|
+
const state: ClusterTransactionState = {
|
|
520
|
+
messageHash,
|
|
521
|
+
record,
|
|
522
|
+
pending,
|
|
523
|
+
lastUpdate: this.now()
|
|
524
|
+
};
|
|
525
|
+
this.transactions.set(messageHash, state);
|
|
526
|
+
this.persistCoordinatorState(messageHash, record, 'promising');
|
|
527
|
+
log('cluster-tx:transaction-store', {
|
|
528
|
+
messageHash,
|
|
529
|
+
transactionKeys: Array.from(this.transactions.keys())
|
|
530
|
+
});
|
|
531
|
+
|
|
532
|
+
// Wait for the transaction to complete
|
|
533
|
+
try {
|
|
534
|
+
const result = await pending.result();
|
|
535
|
+
// Check if the local cluster already executed the operations during consensus
|
|
536
|
+
const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
|
|
537
|
+
const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
|
|
538
|
+
const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
|
|
539
|
+
// Self is excluded: this node's own member verdict is already carried, more directly and
|
|
540
|
+
// without the wire round trip, by `localPendResult` — and leaving it in both places would
|
|
541
|
+
// make the coordinator's "prefer local" rule ambiguous.
|
|
542
|
+
// Re-checked here rather than trusted: members are supposed to report only conflict-shaped
|
|
543
|
+
// refusals, but the field arrives off the wire, so anything else (a success, a bare-reason
|
|
544
|
+
// fault, a malformed entry) is dropped instead of being handed to a caller that would read
|
|
545
|
+
// it as a retryable conflict.
|
|
546
|
+
const selfId = this.localCluster?.peerId.toString();
|
|
547
|
+
const cohortPendRefusals: { [peerId: string]: StaleFailure } = {};
|
|
548
|
+
// The commit arm is re-checked the same way, to the shape the gate reads: a plain
|
|
549
|
+
// `success: true`, or an object whose `success` is `false`. Anything else off the wire is
|
|
550
|
+
// dropped rather than counted as a holder.
|
|
551
|
+
const cohortCommitOutcomes: { [peerId: string]: CommitResult } = {};
|
|
552
|
+
for (const [peerId, outcome] of Object.entries(result.applyOutcomes ?? {})) {
|
|
553
|
+
if (peerId === selfId) continue;
|
|
554
|
+
const pend = outcome?.pend;
|
|
555
|
+
if (pend !== undefined && !pend.success && isConflictFailure(pend)) {
|
|
556
|
+
cohortPendRefusals[peerId] = pend;
|
|
557
|
+
}
|
|
558
|
+
const commit = outcome?.commit;
|
|
559
|
+
if (commit !== null && typeof commit === 'object' && (commit.success === true || commit.success === false)) {
|
|
560
|
+
cohortCommitOutcomes[peerId] = commit;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
return {
|
|
564
|
+
record: result,
|
|
565
|
+
localExecuted,
|
|
566
|
+
...(localPendResult === undefined ? {} : { localPendResult }),
|
|
567
|
+
...(localCommitResult === undefined ? {} : { localCommitResult }),
|
|
568
|
+
...(Object.keys(cohortPendRefusals).length === 0 ? {} : { cohortPendRefusals }),
|
|
569
|
+
...(Object.keys(cohortCommitOutcomes).length === 0 ? {} : { cohortCommitOutcomes })
|
|
570
|
+
};
|
|
571
|
+
} finally {
|
|
572
|
+
const stored = this.transactions.get(messageHash);
|
|
573
|
+
const retrySnapshot = stored?.retry ? {
|
|
574
|
+
attempt: stored.retry.attempt,
|
|
575
|
+
pending: Array.from(stored.retry.pendingPeers ?? [])
|
|
576
|
+
} : undefined;
|
|
577
|
+
log('cluster-tx:complete', {
|
|
578
|
+
messageHash,
|
|
579
|
+
finalPromises: stored ? Object.keys(stored.record.promises ?? {}) : undefined,
|
|
580
|
+
finalCommits: stored ? Object.keys(stored.record.commits ?? {}) : undefined,
|
|
581
|
+
retry: retrySnapshot
|
|
582
|
+
});
|
|
583
|
+
// Don't remove transaction immediately if retries are scheduled
|
|
584
|
+
// Let the retry completion or abort handle cleanup
|
|
585
|
+
if (!stored?.retry) {
|
|
586
|
+
// Wait a bit before cleanup to allow any in-flight responses to arrive
|
|
587
|
+
this.setTimer(() => {
|
|
588
|
+
this.transactions.delete(messageHash);
|
|
589
|
+
this.deleteCoordinatorState(messageHash);
|
|
590
|
+
log('cluster-tx:transaction-remove', {
|
|
591
|
+
messageHash,
|
|
592
|
+
remaining: Array.from(this.transactions.keys())
|
|
593
|
+
});
|
|
594
|
+
}, 100);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
/**
|
|
600
|
+
* Executes the full transaction process
|
|
601
|
+
*/
|
|
602
|
+
private async executeTransaction(peers: ClusterPeers, record: ClusterRecord): Promise<ClusterRecord> {
|
|
603
|
+
const peerCount = Object.keys(peers).length;
|
|
604
|
+
|
|
605
|
+
// Validate against minimum cluster size
|
|
606
|
+
if (peerCount < this.cfg.minAbsoluteClusterSize) {
|
|
607
|
+
const validated = await this.validateSmallCluster(peerCount, peers);
|
|
608
|
+
if (!validated) {
|
|
609
|
+
log('cluster-tx:reject-too-small', {
|
|
610
|
+
peerCount,
|
|
611
|
+
minRequired: this.cfg.minAbsoluteClusterSize
|
|
612
|
+
});
|
|
613
|
+
throw new Error(`Cluster size ${peerCount} below minimum ${this.cfg.minAbsoluteClusterSize} and not validated`);
|
|
614
|
+
}
|
|
615
|
+
log('cluster-tx:small-cluster-validated', { peerCount });
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
// Check configured cluster size
|
|
619
|
+
if (!this.cfg.allowClusterDownsize && peerCount < this.cfg.clusterSize) {
|
|
620
|
+
log('cluster-tx:reject-downsize', { peerCount, required: this.cfg.clusterSize });
|
|
621
|
+
throw new Error(`Cluster size ${peerCount} below configured minimum ${this.cfg.clusterSize}`);
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
// Collect promises with super-majority requirement
|
|
625
|
+
const promised = await this.collectPromises(peers, record);
|
|
626
|
+
const superMajority = Math.ceil(peerCount * this.cfg.superMajorityThreshold);
|
|
627
|
+
|
|
628
|
+
// Count approvals, rejections and the two RETRYABLE refusals separately. A `conflict` vote is a
|
|
629
|
+
// member saying "not now — I hold the race winner"; a `held` vote is a member saying "not now —
|
|
630
|
+
// a different unresolved action holds these blocks in my storage". Neither may count toward
|
|
631
|
+
// approvals OR rejections, or a transient refusal would masquerade as a validator rejection
|
|
632
|
+
// (permanent) or as silence (indistinguishable from an unreachable cohort) — both wrong.
|
|
633
|
+
const promises = promised.record.promises;
|
|
634
|
+
const approvalCount = Object.values(promises).filter(sig => sig.type === 'approve').length;
|
|
635
|
+
const rejectionCount = Object.values(promises).filter(sig => sig.type === 'reject').length;
|
|
636
|
+
const conflictCount = Object.values(promises).filter(sig => sig.type === 'conflict').length;
|
|
637
|
+
const heldCount = Object.values(promises).filter(sig => sig.type === 'held').length;
|
|
638
|
+
|
|
639
|
+
// Check if rejections make super-majority impossible
|
|
640
|
+
// If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
|
|
641
|
+
const maxAllowedRejections = peerCount - superMajority;
|
|
642
|
+
// Whether the merged record itself PROVES super-majority unreachable — the same sum a member
|
|
643
|
+
// re-derives as `ConflictSuperseded`/`Rejected` from the signed votes, which is what makes an
|
|
644
|
+
// abandonment broadcast proof-carrying rather than an unauthenticated "forget this".
|
|
645
|
+
const refusalsProveUnreachable = rejectionCount + conflictCount + heldCount > maxAllowedRejections;
|
|
646
|
+
if (rejectionCount > maxAllowedRejections) {
|
|
647
|
+
const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
|
|
648
|
+
.flatMap(([peerId, sig]) => sig.type === 'reject' ? [[peerId, sig.rejectReason ?? 'unknown'] as const] : []));
|
|
649
|
+
const rejectReasons = Object.entries(rejectReasonsByPeer)
|
|
650
|
+
.map(([peerId, reason]) => `${peerId}: ${reason}`)
|
|
651
|
+
.join('; ');
|
|
652
|
+
log('cluster-tx:rejected-by-validators', {
|
|
653
|
+
messageHash: record.messageHash,
|
|
654
|
+
peerCount,
|
|
655
|
+
rejections: rejectionCount,
|
|
656
|
+
maxAllowed: maxAllowedRejections,
|
|
657
|
+
reasons: rejectReasons
|
|
658
|
+
});
|
|
659
|
+
this.updateTransactionRecord(promised.record, 'rejected-by-validators');
|
|
660
|
+
// Abandoning here without telling anyone leaves every member that voted holding this
|
|
661
|
+
// transaction in its own reservation table, blocking its blocks until that member's
|
|
662
|
+
// staleness sweep fires — and each retry we throw back to the caller plants a fresh
|
|
663
|
+
// reservation, so the block never frees. The merged record carries enough signed
|
|
664
|
+
// rejections to *prove* the transaction is dead, so replaying it to the cohort makes
|
|
665
|
+
// every member recompute `Rejected` and clear immediately. Proof-carrying, so a member
|
|
666
|
+
// need not trust us: it verifies the signatures it is shown.
|
|
667
|
+
this.broadcastAbandonment(promised.record, 'rejected-by-validators');
|
|
668
|
+
throw new ValidatorRejectionError(
|
|
669
|
+
`Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
|
|
670
|
+
rejectReasonsByPeer);
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
// A conflict-answered shortfall is a LOST RACE, not a validator verdict and not silence.
|
|
674
|
+
// Checked after the rejection threshold (a genuine validator rejection still wins) and
|
|
675
|
+
// before the generic shortfall (which must stay reserved for the genuinely-silent cohort).
|
|
676
|
+
if (conflictCount > 0 && approvalCount < superMajority) {
|
|
677
|
+
const conflicts = Object.fromEntries(Object.entries(promises)
|
|
678
|
+
.flatMap(([peerId, sig]) => sig.type === 'conflict' ? [[peerId, sig.conflictWith] as const] : []));
|
|
679
|
+
log('cluster-tx:conflict-race-lost', {
|
|
680
|
+
messageHash: record.messageHash,
|
|
681
|
+
peerCount,
|
|
682
|
+
approvals: approvalCount,
|
|
683
|
+
rejections: rejectionCount,
|
|
684
|
+
conflicts,
|
|
685
|
+
superMajority
|
|
686
|
+
});
|
|
687
|
+
this.updateTransactionRecord(promised.record, 'conflict-race-lost');
|
|
688
|
+
// Broadcast only when the merged record itself PROVES the transaction can no longer reach
|
|
689
|
+
// super-majority (members re-derive ConflictSuperseded/Rejected from the signed votes and
|
|
690
|
+
// clear their reservations immediately). Below that bar the record proves nothing and a
|
|
691
|
+
// broadcast would be the unauthenticated "forget this" the shortfall NOTE below refuses.
|
|
692
|
+
if (refusalsProveUnreachable) {
|
|
693
|
+
this.broadcastAbandonment(promised.record, 'conflict-race-lost');
|
|
694
|
+
}
|
|
695
|
+
throw new ConflictRaceLostError(
|
|
696
|
+
`Conflict race lost: ${conflictCount}/${peerCount} member(s) hold a conflicting winner (${approvalCount}/${superMajority} approvals)`,
|
|
697
|
+
conflicts);
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
// A `held`-answered shortfall is the OTHER retryable refusal: the pend queued behind a rival's
|
|
701
|
+
// unresolved reservation. Checked after the conflict branch so a lost race still wins when both
|
|
702
|
+
// answer — a conflict vote names the winning transaction's messageHash, which is strictly more
|
|
703
|
+
// actionable than an action id — and, like it, before the generic shortfall, which must stay
|
|
704
|
+
// reserved for the genuinely-silent cohort.
|
|
705
|
+
if (heldCount > 0 && approvalCount < superMajority) {
|
|
706
|
+
const heldBy = Object.fromEntries(Object.entries(promises)
|
|
707
|
+
.flatMap(([peerId, sig]) => sig.type === 'held' ? [[peerId, sig.heldBy] as const] : []));
|
|
708
|
+
log('cluster-tx:pend-blocks-held', {
|
|
709
|
+
messageHash: record.messageHash,
|
|
710
|
+
peerCount,
|
|
711
|
+
approvals: approvalCount,
|
|
712
|
+
rejections: rejectionCount,
|
|
713
|
+
heldBy,
|
|
714
|
+
superMajority
|
|
715
|
+
});
|
|
716
|
+
this.updateTransactionRecord(promised.record, 'pend-blocks-held');
|
|
717
|
+
if (refusalsProveUnreachable) {
|
|
718
|
+
this.broadcastAbandonment(promised.record, 'pend-blocks-held');
|
|
719
|
+
}
|
|
720
|
+
throw new BlocksHeldError(
|
|
721
|
+
`Pend blocks held: ${heldCount}/${peerCount} member(s) hold an unresolved rival action (${approvalCount}/${superMajority} approvals)`,
|
|
722
|
+
heldBy);
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
if (peerCount > 1 && approvalCount < superMajority) {
|
|
726
|
+
log('cluster-tx:supermajority-failed', {
|
|
727
|
+
messageHash: record.messageHash,
|
|
728
|
+
peerCount,
|
|
729
|
+
approvals: approvalCount,
|
|
730
|
+
rejections: rejectionCount,
|
|
731
|
+
superMajority,
|
|
732
|
+
threshold: this.cfg.superMajorityThreshold
|
|
733
|
+
});
|
|
734
|
+
this.updateTransactionRecord(promised.record, 'supermajority-failed');
|
|
735
|
+
// NOTE: deliberately NOT broadcast, unlike the rejected-by-validators branch above. With
|
|
736
|
+
// conflict-answered shortfalls peeled off above, we get here only because peers did not
|
|
737
|
+
// answer at all, so the record carries no signed evidence that the transaction is dead — a
|
|
738
|
+
// broadcast would be an unauthenticated "forget this" that any caller could use to clear a
|
|
739
|
+
// live transaction out of a member's reservation table. Members that DID vote are freed by
|
|
740
|
+
// their own staleness sweep instead.
|
|
741
|
+
// NOTE: the message below is load-bearing wire text — the consuming repo
|
|
742
|
+
// (sereus cadre-core control-write-retry) matches it verbatim to retry a genuinely-silent
|
|
743
|
+
// cohort. Keep it byte-identical, and never fold `conflict` or `held` votes into its
|
|
744
|
+
// rejection count.
|
|
745
|
+
throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
// Mark as disputed when minority rejections exist but super-majority approves
|
|
749
|
+
if (rejectionCount > 0 && approvalCount >= superMajority) {
|
|
750
|
+
const rejectingPeers: string[] = [];
|
|
751
|
+
const rejectReasons: { [peerId: string]: string } = {};
|
|
752
|
+
for (const [peerId, sig] of Object.entries(promises)) {
|
|
753
|
+
if (sig.type === 'reject') {
|
|
754
|
+
rejectingPeers.push(peerId);
|
|
755
|
+
rejectReasons[peerId] = sig.rejectReason ?? 'unknown';
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
promised.record.disputed = true;
|
|
759
|
+
promised.record.disputeEvidence = { rejectingPeers, rejectReasons };
|
|
760
|
+
log('cluster-tx:disputed', {
|
|
761
|
+
messageHash: record.messageHash,
|
|
762
|
+
rejectingPeers,
|
|
763
|
+
rejectReasons,
|
|
764
|
+
approvalCount,
|
|
765
|
+
rejectionCount,
|
|
766
|
+
peerCount
|
|
767
|
+
});
|
|
768
|
+
// [dispute-subsystem-dormant] Evidence is computed and persisted but initiateDispute() is
|
|
769
|
+
// intentionally NOT called here. Dispute origination stays dormant pending arbitrator-set
|
|
770
|
+
// anchoring — without it a forged synthetic cohort passes resolution.
|
|
771
|
+
// Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
|
|
772
|
+
// Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
|
|
776
|
+
return await this.commitTransaction(promised.record);
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
/**
|
|
780
|
+
* The block's cohort peer ids as currently derivable. Empty when the cohort did not resolve
|
|
781
|
+
* ({@link resolveCohort}: `findCluster` threw, or named nobody), so a caller branching on
|
|
782
|
+
* `length <= 1` is also taking the degraded-routing branch. Derived from `resolveCohort` rather
|
|
783
|
+
* than re-deriving the cohort, so there is exactly one lookup rule.
|
|
784
|
+
*/
|
|
785
|
+
async getClusterPeerIds(blockId: BlockId): Promise<string[]> {
|
|
786
|
+
const cohort = await this.resolveCohort(blockId);
|
|
787
|
+
return cohort.resolved ? [...cohort.peerIds] : [];
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
/** {@link getClusterPeerIds}, counted. Derived from it rather than re-deriving the cohort, so the
|
|
791
|
+
* size a caller branches on and the ids it logs can never come from two different rules. */
|
|
792
|
+
async getClusterSize(blockId: BlockId): Promise<number> {
|
|
793
|
+
return (await this.getClusterPeerIds(blockId)).length;
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
/**
|
|
797
|
+
* Validate that a small cluster size is legitimate by querying remote peers
|
|
798
|
+
* for their network size estimates. Returns true if estimates roughly agree.
|
|
799
|
+
*/
|
|
800
|
+
private async validateSmallCluster(localSize: number, _peers: ClusterPeers): Promise<boolean> {
|
|
801
|
+
// If we have FRET and it shows confident estimate
|
|
802
|
+
if (this.fretService) {
|
|
803
|
+
try {
|
|
804
|
+
const estimate = this.fretService.getNetworkSizeEstimate();
|
|
805
|
+
if (estimate.confidence > 0.5) {
|
|
806
|
+
// Check if FRET estimate roughly matches observed cluster size
|
|
807
|
+
const orderOfMagnitude = Math.floor(Math.log10(estimate.size_estimate + 1));
|
|
808
|
+
const localOrderOfMagnitude = Math.floor(Math.log10(localSize + 1));
|
|
809
|
+
|
|
810
|
+
// If within same order of magnitude, accept it
|
|
811
|
+
if (Math.abs(orderOfMagnitude - localOrderOfMagnitude) <= 1) {
|
|
812
|
+
log('cluster-tx:small-cluster-validated-by-fret', {
|
|
813
|
+
localSize,
|
|
814
|
+
fretEstimate: estimate.size_estimate,
|
|
815
|
+
confidence: estimate.confidence,
|
|
816
|
+
sources: estimate.sources
|
|
817
|
+
});
|
|
818
|
+
return true;
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
} catch (err) {
|
|
822
|
+
// Ignore errors
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
// Fallback: with no confident network-size estimate, fail CLOSED by default.
|
|
827
|
+
// An undersized cluster with no way to justify its size is unsafe (a lone/
|
|
828
|
+
// near-lone node could rubber-stamp its own writes), so reject unless the
|
|
829
|
+
// operator has explicitly opted in via allowUnvalidatedSmallCluster (e.g.
|
|
830
|
+
// single-node / local dev knowingly running below the floor).
|
|
831
|
+
const admit = this.cfg.allowUnvalidatedSmallCluster ?? false;
|
|
832
|
+
log('cluster-tx:small-cluster-no-confident-estimate', {
|
|
833
|
+
localSize,
|
|
834
|
+
reason: 'no-confident-network-size-estimate',
|
|
835
|
+
admit
|
|
836
|
+
});
|
|
837
|
+
return admit;
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
/**
|
|
841
|
+
* Collects promises from all peers in the cluster
|
|
842
|
+
*/
|
|
843
|
+
private async collectPromises(peers: ClusterPeers, record: ClusterRecord): Promise<{ record: ClusterRecord }> {
|
|
844
|
+
const peerIds = Object.keys(peers);
|
|
845
|
+
const summary: ClusterLogPeerOutcome[] = [];
|
|
846
|
+
if (verbose) {
|
|
847
|
+
const peerDetail = peerIds.map(id => ({
|
|
848
|
+
id: id.substring(0, 12),
|
|
849
|
+
addrs: peers[id]?.multiaddrs?.length ?? 0
|
|
850
|
+
}));
|
|
851
|
+
log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
|
|
852
|
+
}
|
|
853
|
+
// For each peer, create a client and request a promise. A remote promise rides
|
|
854
|
+
// a libp2p stream that a relayed (limited) connection can reset transiently, so
|
|
855
|
+
// each remote request gets `promiseImmediateRetries` in-line re-attempts before
|
|
856
|
+
// it counts as a failure — without this a single relayed reset drops the peer and
|
|
857
|
+
// sinks super-majority (the commit broadcast already has the same guard).
|
|
858
|
+
const promiseRequests = peerIds.map(peerIdStr => {
|
|
859
|
+
const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
|
|
860
|
+
log('cluster-tx:promise-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
|
|
861
|
+
return new Pending(this.updateMember(peerIdStr, record, this.promiseImmediateRetries, 'promise'));
|
|
862
|
+
});
|
|
863
|
+
|
|
864
|
+
// Wait for all promises to complete
|
|
865
|
+
const results = await Promise.all(promiseRequests.map((p, idx) => p.result().then(res => {
|
|
866
|
+
const peerIdStr = peerIds[idx]!;
|
|
867
|
+
log('cluster-tx:promise-response', {
|
|
868
|
+
messageHash: record.messageHash,
|
|
869
|
+
peerId: peerIdStr,
|
|
870
|
+
success: true,
|
|
871
|
+
returnedPromises: Object.keys(res.promises ?? {}),
|
|
872
|
+
returnedCommits: Object.keys(res.commits ?? {})
|
|
873
|
+
});
|
|
874
|
+
summary.push({ peerId: peerIdStr, success: true });
|
|
875
|
+
return res;
|
|
876
|
+
}).catch(err => {
|
|
877
|
+
const peerIdStr = peerIds[idx]!;
|
|
878
|
+
log('cluster-tx:promise-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
|
|
879
|
+
summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
|
|
880
|
+
this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `promise:${record.messageHash}`);
|
|
881
|
+
return null;
|
|
882
|
+
})));
|
|
883
|
+
const successes = summary.filter(entry => entry.success).map(entry => entry.peerId);
|
|
884
|
+
const failures = summary.filter(entry => !entry.success);
|
|
885
|
+
log('cluster-tx:promise-summary', {
|
|
886
|
+
messageHash: record.messageHash,
|
|
887
|
+
successes,
|
|
888
|
+
failures
|
|
889
|
+
});
|
|
890
|
+
|
|
891
|
+
log('cluster-tx:promise-merge-begin', {
|
|
892
|
+
messageHash: record.messageHash,
|
|
893
|
+
initialPromises: Object.keys(record.promises ?? {}),
|
|
894
|
+
transactionsKeys: Array.from(this.transactions.keys()),
|
|
895
|
+
hasTransaction: this.transactions.has(record.messageHash)
|
|
896
|
+
});
|
|
897
|
+
|
|
898
|
+
// Merge all promises into the record
|
|
899
|
+
for (const result of results.filter(Boolean) as ClusterRecord[]) {
|
|
900
|
+
log('cluster-tx:promise-merge-input', {
|
|
901
|
+
messageHash: record.messageHash,
|
|
902
|
+
resultFrom: Object.keys(result.promises ?? {}),
|
|
903
|
+
recordBefore: Object.keys(record.promises ?? {})
|
|
904
|
+
});
|
|
905
|
+
const resultPromises = Object.keys(result.promises ?? {});
|
|
906
|
+
log('cluster-tx:promise-merge-result', {
|
|
907
|
+
messageHash: record.messageHash,
|
|
908
|
+
peerPromises: resultPromises
|
|
909
|
+
});
|
|
910
|
+
if (typeof record.suggestedClusterSize === 'number' && typeof result.suggestedClusterSize === 'number') {
|
|
911
|
+
const expected = result.suggestedClusterSize;
|
|
912
|
+
const actual = Object.keys(peers).length;
|
|
913
|
+
const maxDiff = Math.ceil(Math.max(1, expected * this.cfg.clusterSizeTolerance));
|
|
914
|
+
if (Math.abs(actual - expected) > maxDiff) {
|
|
915
|
+
log('cluster-tx:size-variance', { expected, actual, tolerance: this.cfg.clusterSizeTolerance });
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
record.promises = { ...record.promises, ...result.promises };
|
|
919
|
+
log('cluster-tx:promise-merge-after', {
|
|
920
|
+
messageHash: record.messageHash,
|
|
921
|
+
mergedPromises: Object.keys(record.promises ?? {})
|
|
922
|
+
});
|
|
923
|
+
}
|
|
924
|
+
log('cluster-tx:promise-merge', {
|
|
925
|
+
messageHash: record.messageHash,
|
|
926
|
+
mergedPromises: Object.keys(record.promises ?? {})
|
|
927
|
+
});
|
|
928
|
+
log('cluster-tx:promise-merge-end', {
|
|
929
|
+
messageHash: record.messageHash,
|
|
930
|
+
finalPromises: Object.keys(record.promises ?? {}),
|
|
931
|
+
transactionsEntry: this.transactions.get(record.messageHash)
|
|
932
|
+
});
|
|
933
|
+
this.updateTransactionRecord(record, 'after-promises');
|
|
934
|
+
return { record };
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
/**
|
|
938
|
+
* The commit round, then the consensus delivery. Runs once the promise round reached super-majority.
|
|
939
|
+
*
|
|
940
|
+
* **This node's own member votes to commit first, in process, and its signature rides on the commit
|
|
941
|
+
* round** ({@link presignLocalCommit}). A remote member receiving that record adds its own commit,
|
|
942
|
+
* and in a cohort of two (2 of 2) or three (2 of 3) that is already the strict majority its phase
|
|
943
|
+
* loop needs for consensus, so it applies in the same delivery and answers with its apply report
|
|
944
|
+
* stamped on. What a member accepts does not change: it reaches consensus only on commit signatures
|
|
945
|
+
* it verified, and it signed its own commit only after seeing a super-majority of approved promises.
|
|
946
|
+
* It is the same kind of record the consensus broadcast carries, arriving one round earlier. In a
|
|
947
|
+
* cohort of four or more the coordinator's commit plus one member's is short of a majority, so
|
|
948
|
+
* nobody applies on receipt and the broadcast below works as it always did.
|
|
949
|
+
*
|
|
950
|
+
* Once the merged commits reach the majority, {@link broadcastMergedRecord} delivers the record to
|
|
951
|
+
* this node's member and then only to the remote members still needing it
|
|
952
|
+
* ({@link membersAwaitingConsensus}). With every remote member healthy in a small cohort that list is
|
|
953
|
+
* empty, so a consensus operation costs each remote member two calls (promise, commit) instead of
|
|
954
|
+
* three. When the pre-sign is unavailable the round runs as it did before — every member in
|
|
955
|
+
* parallel, this node's included — and the broadcast then reaches every member.
|
|
956
|
+
*/
|
|
957
|
+
private async commitTransaction(record: ClusterRecord): Promise<ClusterRecord> {
|
|
958
|
+
const selfId = this.localCluster?.peerId.toString();
|
|
959
|
+
const presigned = await this.presignLocalCommit(record);
|
|
960
|
+
const roundPeers = Object.keys(record.peers).filter(id => !presigned || id !== selfId);
|
|
961
|
+
const deliveries = await this.collectCommits(record, roundPeers);
|
|
962
|
+
// A member can reach consensus during THIS round (see above), so its apply report arrives on
|
|
963
|
+
// these responses. The broadcast's copy wins on overlap, being the later of the two.
|
|
964
|
+
mergeApplyOutcomes(record, collectApplyOutcomes(deliveries));
|
|
965
|
+
mergeCommits(record, deliveries);
|
|
966
|
+
log('cluster-tx:commit-merge', {
|
|
967
|
+
messageHash: record.messageHash,
|
|
968
|
+
presigned,
|
|
969
|
+
mergedCommits: Object.keys(record.commits)
|
|
970
|
+
});
|
|
971
|
+
this.updateTransactionRecord(record, 'after-commit');
|
|
972
|
+
|
|
973
|
+
if (!this.hasCommitMajority(record)) {
|
|
974
|
+
this.scheduleOrClearRetry(record, deliveries.filter(d => !d.success).map(d => d.peerId));
|
|
975
|
+
return record;
|
|
976
|
+
}
|
|
977
|
+
log('cluster-tx:commit-majority-reached', {
|
|
978
|
+
messageHash: record.messageHash,
|
|
979
|
+
commitCount: Object.keys(record.commits).length,
|
|
980
|
+
peerCount: Object.keys(record.peers).length,
|
|
981
|
+
threshold: this.cfg.simpleMajorityThreshold
|
|
982
|
+
});
|
|
983
|
+
// This node's member is not in the list: the broadcast decides its delivery itself.
|
|
984
|
+
const awaiting = membersAwaitingConsensus(deliveries.filter(d => d.peerId !== selfId));
|
|
985
|
+
const { failures, applyOutcomes } = await this.broadcastMergedRecord(record, awaiting);
|
|
986
|
+
mergeApplyOutcomes(record, applyOutcomes);
|
|
987
|
+
// The scheduled retry works from the stored copy, and reads its apply outcomes to decide on the
|
|
988
|
+
// coordinating member's second reconcile, so it needs the broadcast's too.
|
|
989
|
+
this.updateTransactionRecord(record, 'after-broadcast');
|
|
990
|
+
this.scheduleOrClearRetry(record, failures);
|
|
991
|
+
return record;
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
/**
|
|
995
|
+
* Have this node's own member vote to commit on the promise-complete record, in process, before the
|
|
996
|
+
* commit round goes out, and merge its signature into `record`. True when the member answered: the
|
|
997
|
+
* round then leaves it out, and the consensus broadcast delivers it the merged record (should it
|
|
998
|
+
* have answered without a commit — its phase was not `OurCommitNeeded` — it is no worse off than
|
|
999
|
+
* in the round, where it would have answered the same). False when there is no local member in the
|
|
1000
|
+
* cohort (some test wiring) or the member threw (an expired message, or `validateRecord` refused):
|
|
1001
|
+
* the round then runs with it included, as it always did.
|
|
1002
|
+
*
|
|
1003
|
+
* The member cannot reach consensus here: the record carries no commit yet, and its own is a
|
|
1004
|
+
* majority only in a cohort of one, which `CoordinatorRepo`'s solo path keeps away from this class.
|
|
1005
|
+
* Were one to arrive anyway, the member would apply here and the broadcast would skip it as
|
|
1006
|
+
* already executed.
|
|
1007
|
+
*/
|
|
1008
|
+
private async presignLocalCommit(record: ClusterRecord): Promise<boolean> {
|
|
1009
|
+
const selfId = this.localCluster?.peerId.toString();
|
|
1010
|
+
if (selfId === undefined || !(selfId in record.peers)) {
|
|
1011
|
+
return false;
|
|
1012
|
+
}
|
|
1013
|
+
try {
|
|
1014
|
+
const response = await this.localCluster!.update({ ...record });
|
|
1015
|
+
// Its promise too, not only its commit. A member whose promise round delivery failed (possible
|
|
1016
|
+
// only in a cohort of four or more, where super-majority can be reached without it) adds its
|
|
1017
|
+
// promise here and signs its commit over a commit hash covering it; a round that carried the
|
|
1018
|
+
// commit without the promise would fail every remote member's signature check.
|
|
1019
|
+
record.promises = { ...record.promises, ...response.promises };
|
|
1020
|
+
mergeCommits(record, [{ response }]);
|
|
1021
|
+
log('cluster-tx:commit-presign', { messageHash: record.messageHash, signed: response.commits[selfId] !== undefined });
|
|
1022
|
+
return true;
|
|
1023
|
+
} catch (err) {
|
|
1024
|
+
log('cluster-tx:commit-presign-error', {
|
|
1025
|
+
messageHash: record.messageHash,
|
|
1026
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1027
|
+
});
|
|
1028
|
+
return false;
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
/**
|
|
1033
|
+
* Send `record` to each of `peerIds` in parallel for its commit vote. No per-peer immediate retry:
|
|
1034
|
+
* a failure here is recovered by the consensus broadcast's in-line retry and the scheduled
|
|
1035
|
+
* commit-retry timer. (The promise round has no such backstop, which is why `collectPromises` gets
|
|
1036
|
+
* the immediate retry instead.)
|
|
1037
|
+
*/
|
|
1038
|
+
private async collectCommits(record: ClusterRecord, peerIds: readonly string[]): Promise<MemberDelivery[]> {
|
|
1039
|
+
if (verbose) {
|
|
1040
|
+
const peerDetail = peerIds.map(id => ({
|
|
1041
|
+
id: id.substring(0, 12),
|
|
1042
|
+
addrs: record.peers[id]?.multiaddrs?.length ?? 0
|
|
1043
|
+
}));
|
|
1044
|
+
log('cluster-tx:commit-peers', { messageHash: record.messageHash, peers: peerDetail });
|
|
1045
|
+
}
|
|
1046
|
+
// A snapshot: the members answer from the record as sent, and `record` is merged into only after
|
|
1047
|
+
// every answer is in.
|
|
1048
|
+
const payload: ClusterRecord = { ...record };
|
|
1049
|
+
const selfId = this.localCluster?.peerId.toString();
|
|
1050
|
+
const deliveries = await Promise.all(peerIds.map(peerId => {
|
|
1051
|
+
log('cluster-tx:commit-request', { messageHash: record.messageHash, peerId, isLocal: peerId === selfId });
|
|
1052
|
+
return this.deliver(payload, peerId, 0, 'commit');
|
|
1053
|
+
}));
|
|
1054
|
+
for (const { peerId, success } of deliveries) {
|
|
1055
|
+
if (!success) this.reputation?.reportPeer(peerId, PenaltyReason.ConsensusTimeout, `commit:${record.messageHash}`);
|
|
1056
|
+
}
|
|
1057
|
+
log('cluster-tx:commit-summary', {
|
|
1058
|
+
messageHash: record.messageHash,
|
|
1059
|
+
successes: deliveries.filter(d => d.success).map(d => d.peerId),
|
|
1060
|
+
failures: deliveries.filter(d => !d.success).map(({ peerId, error }) => ({ peerId, error }))
|
|
1061
|
+
});
|
|
1062
|
+
return deliveries;
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
/** Whether `record`'s commit signatures reach the simple majority (>50%) that proves the commit. */
|
|
1066
|
+
private hasCommitMajority(record: ClusterRecord): boolean {
|
|
1067
|
+
const peerCount = Object.keys(record.peers).length;
|
|
1068
|
+
return Object.keys(record.commits).length >= Math.floor(peerCount * this.cfg.simpleMajorityThreshold) + 1;
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
/** Schedule a commit retry for `missingPeers`, or clear any pending one when nobody is missing. */
|
|
1072
|
+
private scheduleOrClearRetry(record: ClusterRecord, missingPeers: string[]): void {
|
|
1073
|
+
if (missingPeers.length > 0) {
|
|
1074
|
+
this.scheduleCommitRetry(record.messageHash, record, missingPeers);
|
|
1075
|
+
} else {
|
|
1076
|
+
this.clearRetry(record.messageHash);
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
/**
|
|
1081
|
+
* One {@link updateMember} call whose failure is logged and returned rather than thrown, so a
|
|
1082
|
+
* parallel round can read every member's answer.
|
|
1083
|
+
*/
|
|
1084
|
+
private async deliver(record: ClusterRecord, peerId: string, immediateRetries: number, phase: string): Promise<MemberDelivery> {
|
|
1085
|
+
try {
|
|
1086
|
+
return { peerId, success: true, response: await this.updateMember(peerId, record, immediateRetries, phase) };
|
|
1087
|
+
} catch (err) {
|
|
1088
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
1089
|
+
log('cluster-tx:member-delivery-error', { messageHash: record.messageHash, peerId, phase, error });
|
|
1090
|
+
return { peerId, success: false, error };
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
/**
|
|
1095
|
+
* Deliver the consensus record — carrying a majority of commit signatures — to the members that
|
|
1096
|
+
* still have to apply it: this node's own member first, awaited, unless it already applied
|
|
1097
|
+
* ({@link deliverToLocalMember}); then `remoteTargets` in parallel. Each remote delivery gets
|
|
1098
|
+
* `commitBroadcastImmediateRetries` in-line re-attempts before it counts as failed: the connection
|
|
1099
|
+
* the commit round used is usually still warm, so an immediate retry recovers most transient stream
|
|
1100
|
+
* errors without falling back to the scheduled retry timer. This node's member is invoked exactly
|
|
1101
|
+
* once — a local failure is a real fault, not a transient one.
|
|
1102
|
+
*
|
|
1103
|
+
* **Delivery order is load-bearing: this node's own member first, then the remote members.** A
|
|
1104
|
+
* member that is behind (it never saw the pend, or holds no base for the block) reconciles the
|
|
1105
|
+
* committed revision from `record.peers` during its apply. Once the coordinating member has applied
|
|
1106
|
+
* it holds the revision, and its copy carries the cohort's commit proof (`buildBlockCommitProof`),
|
|
1107
|
+
* which `createReconcileBlock` accepts from a single holder, so a whole cohort of behind members can
|
|
1108
|
+
* heal from it. This is also why `remoteTargets` includes members that have ALREADY applied but
|
|
1109
|
+
* report a refused commit: in a cohort of three or fewer a remote member applies on receipt of the
|
|
1110
|
+
* commit round ({@link commitTransaction}), before this node's member, so a behind one reconciled
|
|
1111
|
+
* while nobody held the revision. Sending it the record again now gives it another reconcile
|
|
1112
|
+
* (`ClusterMember.handleAlreadyExecuted`), and its answer carries the refreshed verdict. A
|
|
1113
|
+
* coordinator outside `record.peers` is not a reconcile target and gains nothing from this order;
|
|
1114
|
+
* the durability gate in `CoordinatorRepo.commit` is what makes that shape refuse rather than
|
|
1115
|
+
* acknowledge.
|
|
1116
|
+
*
|
|
1117
|
+
* The mirror case — the coordinating member is ITSELF behind — mostly heals on its own: in a small
|
|
1118
|
+
* cohort a remote member applied during the commit round, so this node's member finds a holder on
|
|
1119
|
+
* its first reconcile. Where no remote member has applied yet (a cohort of four or more, where
|
|
1120
|
+
* nobody applies on receipt), that first reconcile runs before anyone holds the revision and
|
|
1121
|
+
* retains a refusal. So once a remote member reports holding the revision, this node's member gets
|
|
1122
|
+
* one more reconcile (`reconcileRefusedCommit`). The member skips it unless its retained refusal has
|
|
1123
|
+
* the behind shape, so only a behind coordinator pays the extra fetch. It finishes before this
|
|
1124
|
+
* method returns, so `executeClusterTransaction` reads the refreshed verdict.
|
|
1125
|
+
*/
|
|
1126
|
+
private async broadcastMergedRecord(record: ClusterRecord, remoteTargets: readonly string[]): Promise<{ failures: string[]; applyOutcomes?: ClusterRecord['applyOutcomes'] }> {
|
|
1127
|
+
const local = await this.deliverToLocalMember(record);
|
|
1128
|
+
const remote = await Promise.all(remoteTargets.map(peerId =>
|
|
1129
|
+
this.deliver(record, peerId, this.commitBroadcastImmediateRetries, 'commit-broadcast')));
|
|
1130
|
+
const deliveries = local === undefined ? remote : [local, ...remote];
|
|
1131
|
+
// This delivery is where most members apply the operations, so their responses carry the only
|
|
1132
|
+
// report the coordinator ever gets of what each member's OWN storage said. Collecting it here is
|
|
1133
|
+
// what lets a pend refused by a non-coordinating member reach the writer as a conflict instead of
|
|
1134
|
+
// the fabricated success that used to fork the block.
|
|
1135
|
+
//
|
|
1136
|
+
// Each peer's entry is taken from that peer's OWN response and re-keyed under the peer we asked,
|
|
1137
|
+
// so a member cannot report an outcome on another member's behalf by echoing a record full of
|
|
1138
|
+
// entries. Unsigned and advisory either way — see ClusterRecord.applyOutcomes.
|
|
1139
|
+
const applyOutcomes = collectApplyOutcomes(deliveries);
|
|
1140
|
+
// NOTE: after a healing second reconcile, `applyOutcomes[selfId].commit` still carries the
|
|
1141
|
+
// pre-reconcile refusal. Nothing reads the self entry today (the gate reads
|
|
1142
|
+
// `localCommitResult`); if anything starts to, re-stamp it from `getExecutedCommitResult` here.
|
|
1143
|
+
// NOTE: in a 3+ cohort this also runs when the remote holders already form a majority without
|
|
1144
|
+
// this member — one extra fetch that heals its copy; gate on the remote count if it ever shows up.
|
|
1145
|
+
if (this.localMemberHasApplied(record, local) && this.remoteMemberHolds(record, applyOutcomes)) {
|
|
1146
|
+
await this.reconcileLocalMemberAgain(record);
|
|
1147
|
+
}
|
|
1148
|
+
return {
|
|
1149
|
+
failures: deliveries.filter(d => !d.success).map(d => d.peerId),
|
|
1150
|
+
...(applyOutcomes === undefined ? {} : { applyOutcomes })
|
|
1151
|
+
};
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
/**
|
|
1155
|
+
* Deliver `record` to this node's own member, awaited. `undefined` — nothing sent — when there is
|
|
1156
|
+
* no local member in the cohort, or it has already applied the record.
|
|
1157
|
+
*/
|
|
1158
|
+
private async deliverToLocalMember(record: ClusterRecord): Promise<MemberDelivery | undefined> {
|
|
1159
|
+
const selfId = this.localCluster?.peerId.toString();
|
|
1160
|
+
if (selfId === undefined || !(selfId in record.peers) || this.localCluster!.wasTransactionExecuted?.(record.messageHash) === true) {
|
|
1161
|
+
return undefined;
|
|
1162
|
+
}
|
|
1163
|
+
return await this.deliver(record, selfId, 0, 'commit-broadcast');
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
/** This node's member is in the cohort and has applied the record: just now (`local`), or before. */
|
|
1167
|
+
private localMemberHasApplied(record: ClusterRecord, local: MemberDelivery | undefined): boolean {
|
|
1168
|
+
const selfId = this.localCluster?.peerId.toString();
|
|
1169
|
+
return selfId !== undefined && selfId in record.peers && (local?.success ?? true);
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
/** Whether any remote member reports holding the commit, on this delivery or an earlier one. */
|
|
1173
|
+
private remoteMemberHolds(record: ClusterRecord, latest: ClusterRecord['applyOutcomes']): boolean {
|
|
1174
|
+
const selfId = this.localCluster?.peerId.toString();
|
|
1175
|
+
const outcomes = { ...record.applyOutcomes, ...latest };
|
|
1176
|
+
return Object.keys(record.peers).some(id => id !== selfId && outcomes[id]?.commit?.success === true);
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
/**
|
|
1180
|
+
* Give this node's own member its second reconcile (see {@link broadcastMergedRecord}). The
|
|
1181
|
+
* member contract is never to throw; the catch keeps a broken seam from failing a transaction
|
|
1182
|
+
* the remote members already applied.
|
|
1183
|
+
*/
|
|
1184
|
+
private async reconcileLocalMemberAgain(record: ClusterRecord): Promise<void> {
|
|
1185
|
+
try {
|
|
1186
|
+
await this.localCluster?.reconcileRefusedCommit?.(record);
|
|
1187
|
+
} catch (err) {
|
|
1188
|
+
log('cluster-tx:local-reconcile-again-error', {
|
|
1189
|
+
messageHash: record.messageHash,
|
|
1190
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1191
|
+
});
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
|
|
1195
|
+
/**
|
|
1196
|
+
* Fire-and-forget replay of an abandoned transaction's record to every peer in its cohort.
|
|
1197
|
+
*
|
|
1198
|
+
* Called only where the record itself proves the transaction is dead (enough signed rejections that
|
|
1199
|
+
* super-majority is unreachable). Each member re-derives `TransactionPhase.Rejected` from the votes
|
|
1200
|
+
* it verifies and drops the entry from its own reservation table, freeing the blocks immediately
|
|
1201
|
+
* instead of after its 2 s staleness window. No new message type and no wire-format change — this is
|
|
1202
|
+
* the same `update()` every other phase uses.
|
|
1203
|
+
*
|
|
1204
|
+
* Never awaited into the caller's throw and never rethrows: an abandonment must not turn into a
|
|
1205
|
+
* *different* failure, and the staleness sweep remains the backstop if delivery fails.
|
|
1206
|
+
*/
|
|
1207
|
+
private broadcastAbandonment(record: ClusterRecord, reason: string): void {
|
|
1208
|
+
const peerIds = Object.keys(record.peers);
|
|
1209
|
+
log('cluster-tx:abandon-broadcast', { messageHash: record.messageHash, reason, peerIds });
|
|
1210
|
+
void Promise.all(peerIds.map(async peerIdStr => {
|
|
1211
|
+
try {
|
|
1212
|
+
await this.updateMember(peerIdStr, record, 0, 'abandon-broadcast');
|
|
1213
|
+
} catch (err) {
|
|
1214
|
+
log('cluster-tx:abandon-broadcast-error', {
|
|
1215
|
+
messageHash: record.messageHash,
|
|
1216
|
+
peerId: peerIdStr,
|
|
1217
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1218
|
+
});
|
|
1219
|
+
}
|
|
1220
|
+
}));
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
private updateTransactionRecord(record: ClusterRecord, stage: string): void {
|
|
1224
|
+
const state = this.transactions.get(record.messageHash);
|
|
1225
|
+
if (!state) {
|
|
1226
|
+
log('cluster-tx:transaction-update-miss', { messageHash: record.messageHash, stage });
|
|
1227
|
+
return;
|
|
1228
|
+
}
|
|
1229
|
+
state.record = { ...record };
|
|
1230
|
+
state.lastUpdate = this.now();
|
|
1231
|
+
log('cluster-tx:transaction-update', {
|
|
1232
|
+
messageHash: record.messageHash,
|
|
1233
|
+
stage,
|
|
1234
|
+
promises: Object.keys(record.promises ?? {}),
|
|
1235
|
+
commits: Object.keys(record.commits ?? {})
|
|
1236
|
+
});
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
private scheduleCommitRetry(messageHash: string, _record: ClusterRecord, missingPeers: string[]): void {
|
|
1240
|
+
const state = this.transactions.get(messageHash);
|
|
1241
|
+
if (!state) {
|
|
1242
|
+
return;
|
|
1243
|
+
}
|
|
1244
|
+
const existing = state.retry;
|
|
1245
|
+
const nextAttempt = (existing?.attempt ?? 0) + 1;
|
|
1246
|
+
if (nextAttempt > this.retryMaxAttempts) {
|
|
1247
|
+
log('cluster-tx:retry-abort', { messageHash, missingPeers });
|
|
1248
|
+
return;
|
|
1249
|
+
}
|
|
1250
|
+
if (missingPeers.length === 0) {
|
|
1251
|
+
this.clearRetry(messageHash);
|
|
1252
|
+
return;
|
|
1253
|
+
}
|
|
1254
|
+
const pendingPeers = new Set(missingPeers);
|
|
1255
|
+
const baseInterval = existing ? Math.min(existing.intervalMs * this.retryBackoffFactor, this.retryMaxIntervalMs) : this.retryInitialIntervalMs;
|
|
1256
|
+
existing?.cancel?.();
|
|
1257
|
+
const cancel = this.setTimer(() => {
|
|
1258
|
+
void this.retryCommits(messageHash);
|
|
1259
|
+
}, baseInterval);
|
|
1260
|
+
state.retry = {
|
|
1261
|
+
pendingPeers,
|
|
1262
|
+
attempt: nextAttempt,
|
|
1263
|
+
intervalMs: baseInterval,
|
|
1264
|
+
cancel
|
|
1265
|
+
};
|
|
1266
|
+
this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
|
|
1267
|
+
pendingPeers: Array.from(pendingPeers),
|
|
1268
|
+
attempt: nextAttempt,
|
|
1269
|
+
intervalMs: baseInterval
|
|
1270
|
+
});
|
|
1271
|
+
log('cluster-tx:retry-scheduled', { messageHash, attempt: nextAttempt, missingPeers, delayMs: baseInterval });
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
private async retryCommits(messageHash: string): Promise<void> {
|
|
1275
|
+
const state = this.transactions.get(messageHash);
|
|
1276
|
+
if (!state?.retry) {
|
|
1277
|
+
return;
|
|
1278
|
+
}
|
|
1279
|
+
const { pendingPeers, attempt } = state.retry;
|
|
1280
|
+
if (pendingPeers.size === 0) {
|
|
1281
|
+
this.clearRetry(messageHash);
|
|
1282
|
+
return;
|
|
1283
|
+
}
|
|
1284
|
+
const record = state.record;
|
|
1285
|
+
const selfId = this.localCluster?.peerId.toString();
|
|
1286
|
+
log('cluster-tx:retry-start', { messageHash, attempt, peerIds: Array.from(pendingPeers) });
|
|
1287
|
+
// Each pending member gets the record as it stands: it adds its commit, and applies once the
|
|
1288
|
+
// record then carries a majority, which in a small cohort this very delivery can complete. This
|
|
1289
|
+
// node's member is left to the consensus broadcast below once the record already carries a
|
|
1290
|
+
// majority; before that (the commit round failed on it too) it is asked for its commit like the rest.
|
|
1291
|
+
const payload: ClusterRecord = { ...record };
|
|
1292
|
+
const selfToBroadcast = this.hasCommitMajority(record);
|
|
1293
|
+
const deliveries = await Promise.all(Array.from(pendingPeers)
|
|
1294
|
+
.filter(peerId => !selfToBroadcast || peerId !== selfId)
|
|
1295
|
+
.map(peerId => this.deliver(payload, peerId, 0, 'commit-retry')));
|
|
1296
|
+
mergeCommits(record, deliveries);
|
|
1297
|
+
mergeApplyOutcomes(record, collectApplyOutcomes(deliveries));
|
|
1298
|
+
for (const { peerId, success } of deliveries) {
|
|
1299
|
+
if (success) pendingPeers.delete(peerId);
|
|
1300
|
+
}
|
|
1301
|
+
if (this.hasCommitMajority(record)) {
|
|
1302
|
+
// The retry may itself have assembled the majority (a two-member cohort whose remote member
|
|
1303
|
+
// missed the commit round), and then this node's member has not applied; a remote member that
|
|
1304
|
+
// applied on receipt before this node's member did may hold a behind refusal; and in a cohort
|
|
1305
|
+
// of four or more the members that answered here have not applied at all. The consensus
|
|
1306
|
+
// broadcast covers all three, in its usual order, and delivers this node's member unless it
|
|
1307
|
+
// already applied.
|
|
1308
|
+
const { failures, applyOutcomes } = await this.broadcastMergedRecord(record,
|
|
1309
|
+
membersAwaitingConsensus(deliveries.filter(d => d.success && d.peerId !== selfId)));
|
|
1310
|
+
mergeApplyOutcomes(record, applyOutcomes);
|
|
1311
|
+
if (selfId !== undefined) pendingPeers.delete(selfId);
|
|
1312
|
+
for (const peerId of failures) pendingPeers.add(peerId);
|
|
1313
|
+
}
|
|
1314
|
+
log('cluster-tx:retry-complete', {
|
|
1315
|
+
messageHash,
|
|
1316
|
+
attempt,
|
|
1317
|
+
successes: deliveries.filter(d => d.success).map(d => d.peerId),
|
|
1318
|
+
failures: deliveries.filter(d => !d.success).map(({ peerId, error }) => ({ peerId, error })),
|
|
1319
|
+
stillPending: Array.from(pendingPeers)
|
|
1320
|
+
});
|
|
1321
|
+
if (pendingPeers.size === 0) {
|
|
1322
|
+
log('cluster-tx:retry-finished', { messageHash });
|
|
1323
|
+
this.clearRetry(messageHash);
|
|
1324
|
+
return;
|
|
1325
|
+
}
|
|
1326
|
+
if (!this.transactions.has(messageHash)) {
|
|
1327
|
+
return;
|
|
1328
|
+
}
|
|
1329
|
+
this.scheduleCommitRetry(messageHash, state.record, Array.from(pendingPeers));
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
private clearRetry(messageHash: string): void {
|
|
1333
|
+
const state = this.transactions.get(messageHash);
|
|
1334
|
+
if (!state?.retry) {
|
|
1335
|
+
return;
|
|
1336
|
+
}
|
|
1337
|
+
state.retry.cancel?.();
|
|
1338
|
+
state.retry = undefined;
|
|
1339
|
+
// Clean up the transaction after retry is complete
|
|
1340
|
+
this.setTimer(() => {
|
|
1341
|
+
this.transactions.delete(messageHash);
|
|
1342
|
+
this.deleteCoordinatorState(messageHash);
|
|
1343
|
+
log('cluster-tx:transaction-remove', {
|
|
1344
|
+
messageHash,
|
|
1345
|
+
remaining: Array.from(this.transactions.keys())
|
|
1346
|
+
});
|
|
1347
|
+
}, 100);
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
/** Fire-and-forget persist — errors are logged, never thrown. */
|
|
1351
|
+
private persistCoordinatorState(
|
|
1352
|
+
messageHash: string,
|
|
1353
|
+
record: ClusterRecord,
|
|
1354
|
+
phase: 'promising' | 'committing' | 'broadcasting',
|
|
1355
|
+
retryState?: { pendingPeers: string[]; attempt: number; intervalMs: number }
|
|
1356
|
+
): void {
|
|
1357
|
+
if (!this.stateStore) return;
|
|
1358
|
+
this.stateStore.saveCoordinatorState(messageHash, {
|
|
1359
|
+
messageHash,
|
|
1360
|
+
record,
|
|
1361
|
+
lastUpdate: this.now(),
|
|
1362
|
+
phase,
|
|
1363
|
+
retryState
|
|
1364
|
+
}).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
/** Fire-and-forget delete — errors are logged, never thrown. */
|
|
1368
|
+
private deleteCoordinatorState(messageHash: string): void {
|
|
1369
|
+
if (!this.stateStore) return;
|
|
1370
|
+
this.stateStore.deleteCoordinatorState(messageHash)
|
|
1371
|
+
.catch(err => log('cluster-tx:persist-delete-error', { messageHash, error: (err as Error).message }));
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
/**
|
|
1375
|
+
* Recover coordinator transactions from persistent store after a restart.
|
|
1376
|
+
* Called during node startup, before accepting new requests.
|
|
1377
|
+
*/
|
|
1378
|
+
async recoverTransactions(): Promise<void> {
|
|
1379
|
+
if (!this.stateStore) return;
|
|
1380
|
+
const states = await this.stateStore.getAllCoordinatorStates();
|
|
1381
|
+
for (const state of states) {
|
|
1382
|
+
const { messageHash } = state;
|
|
1383
|
+
// Expired — clean up
|
|
1384
|
+
if (state.record.message.expiration && state.record.message.expiration < this.now()) {
|
|
1385
|
+
log('cluster-tx:recovery-expired', { messageHash });
|
|
1386
|
+
await this.stateStore.deleteCoordinatorState(messageHash);
|
|
1387
|
+
continue;
|
|
1388
|
+
}
|
|
1389
|
+
// Broadcasting phase with retry state — resume retries
|
|
1390
|
+
if (state.phase === 'broadcasting' && state.retryState) {
|
|
1391
|
+
log('cluster-tx:recovery-resume-broadcast', { messageHash, attempt: state.retryState.attempt });
|
|
1392
|
+
const pending = new Pending(Promise.resolve(state.record));
|
|
1393
|
+
const txState: ClusterTransactionState = {
|
|
1394
|
+
messageHash,
|
|
1395
|
+
record: state.record,
|
|
1396
|
+
pending,
|
|
1397
|
+
lastUpdate: state.lastUpdate
|
|
1398
|
+
};
|
|
1399
|
+
this.transactions.set(messageHash, txState);
|
|
1400
|
+
// Schedule retry from where we left off
|
|
1401
|
+
this.scheduleCommitRetry(messageHash, state.record, state.retryState.pendingPeers);
|
|
1402
|
+
continue;
|
|
1403
|
+
}
|
|
1404
|
+
// Promising or committing — cannot resume (caller context is gone)
|
|
1405
|
+
log('cluster-tx:recovery-stale', { messageHash, phase: state.phase });
|
|
1406
|
+
await this.stateStore.deleteCoordinatorState(messageHash);
|
|
1407
|
+
}
|
|
1408
|
+
}
|
|
1409
|
+
}
|