@optimystic/db-p2p 0.29.0 → 1.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/cluster-repo.d.ts +75 -16
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +205 -24
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +16 -1
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +66 -7
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +5 -1
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +45 -8
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts +6 -0
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +62 -2
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/package.json +2 -2
- package/src/cluster/cluster-repo.ts +2671 -2488
- package/src/repo/cluster-coordinator.ts +1113 -1039
- package/src/repo/coordinator-repo.ts +2687 -2648
- package/src/storage/storage-repo.ts +65 -2
|
@@ -1,1039 +1,1113 @@
|
|
|
1
|
-
import { peerIdFromString } from "@libp2p/peer-id";
|
|
2
|
-
import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult } from "@optimystic/db-core";
|
|
3
|
-
import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, membershipDigest } from "@optimystic/db-core";
|
|
4
|
-
import { Pending } from "@optimystic/db-core";
|
|
5
|
-
import type { PeerId } from "@libp2p/interface";
|
|
6
|
-
import { createLogger, verbose } from '../logger.js'
|
|
7
|
-
import type { ClusterLogPeerOutcome } from './types.js'
|
|
8
|
-
import type { FretService } from "p2p-fret";
|
|
9
|
-
import type { IPeerReputation } from "../reputation/types.js";
|
|
10
|
-
import { PenaltyReason } from "../reputation/types.js";
|
|
11
|
-
import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
|
|
12
|
-
|
|
13
|
-
const log = createLogger('cluster')
|
|
14
|
-
|
|
15
|
-
/**
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
/**
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
*
|
|
192
|
-
*
|
|
193
|
-
*
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
//
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
//
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
this.
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
//
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
const
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
});
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
peerCount
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
*
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
const
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
log('cluster-tx:
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
});
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
log('cluster-tx:promise-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
const
|
|
694
|
-
const summary
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
};
|
|
707
|
-
|
|
708
|
-
//
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
}
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
}
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
}
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
return;
|
|
955
|
-
}
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
}
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
): void {
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1
|
+
import { peerIdFromString } from "@libp2p/peer-id";
|
|
2
|
+
import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult, StaleFailure } from "@optimystic/db-core";
|
|
3
|
+
import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, isConflictFailure, membershipDigest } from "@optimystic/db-core";
|
|
4
|
+
import { Pending } from "@optimystic/db-core";
|
|
5
|
+
import type { PeerId } from "@libp2p/interface";
|
|
6
|
+
import { createLogger, verbose } from '../logger.js'
|
|
7
|
+
import type { ClusterLogPeerOutcome } from './types.js'
|
|
8
|
+
import type { FretService } from "p2p-fret";
|
|
9
|
+
import type { IPeerReputation } from "../reputation/types.js";
|
|
10
|
+
import { PenaltyReason } from "../reputation/types.js";
|
|
11
|
+
import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
|
|
12
|
+
|
|
13
|
+
const log = createLogger('cluster')
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Pick each peer's OWN {@link ClusterRecord.applyOutcomes} entry out of the record that peer answered
|
|
17
|
+
* with, and key it under the peer we actually asked.
|
|
18
|
+
*
|
|
19
|
+
* Taking only `response.applyOutcomes[peerId]` — rather than spreading the whole map — is what keeps
|
|
20
|
+
* one member from reporting outcomes on other members' behalf: a peer that echoes back a record full
|
|
21
|
+
* of entries contributes exactly one, its own. The field is unsigned advisory data (see its doc
|
|
22
|
+
* comment for why that is safe), so this is a shaping rule, not a security boundary.
|
|
23
|
+
*
|
|
24
|
+
* Returns `undefined` when no peer reported anything, so the common case adds no empty object to the
|
|
25
|
+
* record.
|
|
26
|
+
*/
|
|
27
|
+
function collectApplyOutcomes(
|
|
28
|
+
responses: ReadonlyArray<{ peerId: string; response?: ClusterRecord | null }>
|
|
29
|
+
): ClusterRecord['applyOutcomes'] {
|
|
30
|
+
let collected: NonNullable<ClusterRecord['applyOutcomes']> | undefined;
|
|
31
|
+
for (const { peerId, response } of responses) {
|
|
32
|
+
const own = response?.applyOutcomes?.[peerId];
|
|
33
|
+
if (own === undefined) continue;
|
|
34
|
+
collected ??= {};
|
|
35
|
+
collected[peerId] = own;
|
|
36
|
+
}
|
|
37
|
+
return collected;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Fold collected outcomes into a record in place, later report winning per peer. No-op for `undefined`. */
|
|
41
|
+
function mergeApplyOutcomes(record: ClusterRecord, collected: ClusterRecord['applyOutcomes']): void {
|
|
42
|
+
if (collected === undefined) return;
|
|
43
|
+
record.applyOutcomes = { ...record.applyOutcomes, ...collected };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Consensus refused a transaction: enough members voted reject that super-majority became
|
|
48
|
+
* impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
|
|
49
|
+
* "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
|
|
50
|
+
* rejection reasons — those are free-form text that is part of each member's signed vote payload
|
|
51
|
+
* (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
|
|
52
|
+
* `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
|
|
53
|
+
* loss (confirmed against local storage) or a genuine validation fault.
|
|
54
|
+
*/
|
|
55
|
+
export class ValidatorRejectionError extends Error {
|
|
56
|
+
constructor(
|
|
57
|
+
message: string,
|
|
58
|
+
/** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
|
|
59
|
+
readonly rejectReasons: Record<string, string>
|
|
60
|
+
) {
|
|
61
|
+
super(message);
|
|
62
|
+
this.name = 'ValidatorRejectionError';
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* The transaction lost a conflict race: one or more members answered with a signed `conflict`
|
|
68
|
+
* vote (they hold a rival transaction that won the deterministic race on the same blocks) and
|
|
69
|
+
* approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} — nobody
|
|
70
|
+
* judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
|
|
71
|
+
* `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
|
|
72
|
+
* `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
|
|
73
|
+
* as a thrown error only from other paths. The commit conversion matters as much as the pend one:
|
|
74
|
+
* at the moment this is thrown zero members approved and the members hold the winner — nothing of
|
|
75
|
+
* the loser landed — yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
|
|
76
|
+
* (it treats throws as transport faults), and that re-driven commit races into the window after
|
|
77
|
+
* members apply the winner and clear its reservation, where it can assemble a consensus no member
|
|
78
|
+
* will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
|
|
79
|
+
* writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
|
|
80
|
+
* and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
|
|
81
|
+
* parsed out of prose.
|
|
82
|
+
*/
|
|
83
|
+
export class ConflictRaceLostError extends Error {
|
|
84
|
+
constructor(
|
|
85
|
+
message: string,
|
|
86
|
+
/** peerId → messageHash of the rival transaction that member holds as the race winner. */
|
|
87
|
+
readonly conflicts: Record<string, string>
|
|
88
|
+
) {
|
|
89
|
+
super(message);
|
|
90
|
+
this.name = 'ConflictRaceLostError';
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
|
|
95
|
+
export type TimerCancel = () => void;
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Production timer binding: a one-shot `setTimeout` whose handle is **unref'd** so a pending
|
|
99
|
+
* commit-retry (or the deferred transaction cleanup) never keeps an otherwise-idle process alive.
|
|
100
|
+
* The returned handle clears the timeout (idempotent). Mirrors the reactivity rotation
|
|
101
|
+
* re-registration scheduler's `defaultSetTimer` (see reactivity/rotation-rereg-scheduler.ts).
|
|
102
|
+
*/
|
|
103
|
+
function defaultSetTimer(fn: () => void, delayMs: number): TimerCancel {
|
|
104
|
+
const handle = setTimeout(fn, delayMs);
|
|
105
|
+
// An idle retry/cleanup timer must not pin a process (mirror rotation re-registration + push-state gossip).
|
|
106
|
+
(handle as { unref?: () => void }).unref?.();
|
|
107
|
+
return (): void => clearTimeout(handle);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Optional injection seam for deterministic time. Production leaves both undefined and gets
|
|
112
|
+
* `Date.now` + an unref'd `setTimeout`; tests inject a fake clock + timer queue so scheduled
|
|
113
|
+
* commit-retries fire in virtual (not wall-clock) time.
|
|
114
|
+
*/
|
|
115
|
+
export interface ClusterCoordinatorClock {
|
|
116
|
+
/** Clock (Unix ms). Defaults to `Date.now`. */
|
|
117
|
+
now?: () => number;
|
|
118
|
+
/** Schedule a one-shot timer, returning a cancel handle. Defaults to an unref'd `setTimeout`. */
|
|
119
|
+
setTimer?: (fn: () => void, delayMs: number) => TimerCancel;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Manages the state of cluster transactions for a specific block ID
|
|
124
|
+
*/
|
|
125
|
+
interface CommitRetryState {
|
|
126
|
+
pendingPeers: Set<string>;
|
|
127
|
+
attempt: number;
|
|
128
|
+
intervalMs: number;
|
|
129
|
+
cancel?: TimerCancel;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
interface ClusterTransactionState {
|
|
133
|
+
messageHash: string;
|
|
134
|
+
record: ClusterRecord;
|
|
135
|
+
pending: Pending<ClusterRecord>;
|
|
136
|
+
lastUpdate: number;
|
|
137
|
+
promiseTimeout?: NodeJS.Timeout;
|
|
138
|
+
resolutionTimeout?: NodeJS.Timeout;
|
|
139
|
+
retry?: CommitRetryState;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Manages distributed transactions across clusters */
|
|
143
|
+
export class ClusterCoordinator {
|
|
144
|
+
private transactions: Map<string, ClusterTransactionState> = new Map();
|
|
145
|
+
private readonly retryInitialIntervalMs: number;
|
|
146
|
+
private readonly retryBackoffFactor: number;
|
|
147
|
+
private readonly retryMaxIntervalMs: number;
|
|
148
|
+
private readonly retryMaxAttempts: number;
|
|
149
|
+
private readonly commitBroadcastImmediateRetries: number;
|
|
150
|
+
private readonly promiseImmediateRetries: number;
|
|
151
|
+
/** Injected clock/timer seam; production defaults to `Date.now` + unref'd `setTimeout`. */
|
|
152
|
+
private readonly now: () => number;
|
|
153
|
+
private readonly setTimer: (fn: () => void, delayMs: number) => TimerCancel;
|
|
154
|
+
|
|
155
|
+
constructor(
|
|
156
|
+
private readonly keyNetwork: IKeyNetwork,
|
|
157
|
+
/** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
|
|
158
|
+
private readonly createClusterClient: (peerId: PeerId) => ICluster,
|
|
159
|
+
private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
|
|
160
|
+
private readonly localCluster?: {
|
|
161
|
+
update: (record: ClusterRecord) => Promise<ClusterRecord>;
|
|
162
|
+
peerId: PeerId;
|
|
163
|
+
wasTransactionExecuted?: (messageHash: string) => boolean;
|
|
164
|
+
/** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
|
|
165
|
+
getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
|
|
166
|
+
/** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
|
|
167
|
+
getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
|
|
168
|
+
},
|
|
169
|
+
private readonly fretService?: FretService,
|
|
170
|
+
private readonly reputation?: IPeerReputation,
|
|
171
|
+
private readonly stateStore?: ITransactionStateStore,
|
|
172
|
+
clock?: ClusterCoordinatorClock
|
|
173
|
+
) {
|
|
174
|
+
this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
|
|
175
|
+
this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
|
|
176
|
+
this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
|
|
177
|
+
this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
|
|
178
|
+
this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
|
|
179
|
+
this.promiseImmediateRetries = cfg.promiseImmediateRetries ?? 1;
|
|
180
|
+
this.now = clock?.now ?? ((): number => Date.now());
|
|
181
|
+
this.setTimer = clock?.setTimer ?? defaultSetTimer;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Invoke one cluster member's `update`, retrying transient REMOTE failures up to
|
|
186
|
+
* `immediateRetries` times before surfacing the error. The local cluster is invoked
|
|
187
|
+
* exactly once — a local throw is a real fault (validation / merge / consensus), not a
|
|
188
|
+
* transient transport blip. A remote call rides a libp2p stream that a circuit-relay
|
|
189
|
+
* ("limited") connection can reset once a per-circuit cap or reservation lapses, which
|
|
190
|
+
* surfaces as a StreamResetError; an immediate retry on the (usually still-warm)
|
|
191
|
+
* connection recovers most of those without escalating the peer to a failure. Shared by
|
|
192
|
+
* the promise-collection, commit-collection, and commit-broadcast phases so all three
|
|
193
|
+
* react to a relayed reset the same way.
|
|
194
|
+
*/
|
|
195
|
+
private async updateMember(peerIdStr: string, record: ClusterRecord, immediateRetries: number, phase: string): Promise<ClusterRecord> {
|
|
196
|
+
const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
|
|
197
|
+
if (isLocal) {
|
|
198
|
+
return await this.localCluster!.update(record);
|
|
199
|
+
}
|
|
200
|
+
const maxAttempts = 1 + Math.max(0, immediateRetries);
|
|
201
|
+
let lastError: unknown;
|
|
202
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
203
|
+
try {
|
|
204
|
+
return await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
|
|
205
|
+
} catch (err) {
|
|
206
|
+
lastError = err;
|
|
207
|
+
if (attempt < maxAttempts) {
|
|
208
|
+
log('cluster-tx:member-update-retry', {
|
|
209
|
+
messageHash: record.messageHash,
|
|
210
|
+
peerId: peerIdStr,
|
|
211
|
+
phase,
|
|
212
|
+
attempt,
|
|
213
|
+
error: err instanceof Error ? err.message : String(err)
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
throw lastError;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Creates a base58btc string hash uniquely identifying a transaction. For a v2 record the caller
|
|
223
|
+
* threads in the {@link membershipDigest} of the peer set so the responsible membership is bound into
|
|
224
|
+
* the identity (two different peer sets ⇒ two different hashes). Omitting `membershipDigestValue`
|
|
225
|
+
* reproduces the legacy v1 hash byte-for-byte.
|
|
226
|
+
*
|
|
227
|
+
* NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
|
|
228
|
+
* which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
|
|
229
|
+
* `pend.priority` (single-collection) — is automatically covered here and by the derived
|
|
230
|
+
* promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
|
|
231
|
+
* cannot strip or inflate it without invalidating the message hash the members verify. No separate
|
|
232
|
+
* priority-hashing step is needed.
|
|
233
|
+
*/
|
|
234
|
+
private async createMessageHash(message: RepoMessage, membershipDigestValue?: string): Promise<string> {
|
|
235
|
+
return computeClusterMessageHash(message, membershipDigestValue);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Gets all peers in the cluster for a specific block ID
|
|
240
|
+
*/
|
|
241
|
+
private async getClusterForBlock(blockId: BlockId): Promise<ClusterPeers> {
|
|
242
|
+
const blockIdBytes = new TextEncoder().encode(blockId);
|
|
243
|
+
try {
|
|
244
|
+
const peers = await this.keyNetwork.findCluster(blockIdBytes);
|
|
245
|
+
const peerIds = Object.keys(peers ?? {});
|
|
246
|
+
log('cluster-tx:cluster-members', { blockId, peerIds });
|
|
247
|
+
return peers;
|
|
248
|
+
} catch (e) {
|
|
249
|
+
log('WARN findCluster failed for %s: %o', blockId, e)
|
|
250
|
+
return {} as ClusterPeers
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
|
|
255
|
+
const peerCount = Object.keys(peers ?? {}).length;
|
|
256
|
+
const record: ClusterRecord = {
|
|
257
|
+
messageHash,
|
|
258
|
+
peers,
|
|
259
|
+
// v2: bind the responsible membership into the signed identity. messageHash was computed over
|
|
260
|
+
// this same digest, so a different peer set would have produced a different messageHash.
|
|
261
|
+
membershipVersion: CURRENT_MEMBERSHIP_VERSION,
|
|
262
|
+
membershipDigest: membershipDigestValue,
|
|
263
|
+
message,
|
|
264
|
+
promises: {},
|
|
265
|
+
commits: {},
|
|
266
|
+
suggestedClusterSize: peerCount || undefined,
|
|
267
|
+
minRequiredSize: this.cfg.allowClusterDownsize ? undefined : this.cfg.clusterSize
|
|
268
|
+
};
|
|
269
|
+
|
|
270
|
+
// Add network size hint if available
|
|
271
|
+
if (this.fretService) {
|
|
272
|
+
try {
|
|
273
|
+
const estimate = this.fretService.getNetworkSizeEstimate();
|
|
274
|
+
if (estimate.size_estimate > 0) {
|
|
275
|
+
record.networkSizeHint = estimate.size_estimate;
|
|
276
|
+
record.networkSizeConfidence = estimate.confidence;
|
|
277
|
+
}
|
|
278
|
+
} catch (err) {
|
|
279
|
+
// Ignore errors getting size estimate
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
return record;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Initiates a 2-phase transaction for a specific block ID.
|
|
288
|
+
* Returns the cluster record and whether the local cluster already executed the operations.
|
|
289
|
+
*/
|
|
290
|
+
async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
|
|
291
|
+
record: ClusterRecord;
|
|
292
|
+
localExecuted: boolean;
|
|
293
|
+
/**
|
|
294
|
+
* Local storage's verdict for a pend operation this node's own cluster member applied during
|
|
295
|
+
* consensus, when the member retained one. Meaningful only when `localExecuted` is true;
|
|
296
|
+
* absent for non-pend messages, for a member that predates the retention, or after the
|
|
297
|
+
* retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
|
|
298
|
+
*/
|
|
299
|
+
localPendResult?: PendResult;
|
|
300
|
+
/**
|
|
301
|
+
* Local storage's verdict for a commit operation this node's own cluster member applied
|
|
302
|
+
* during consensus, when the member retained one. Same availability contract as
|
|
303
|
+
* `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
|
|
304
|
+
* win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
|
|
305
|
+
* success no member durably stored.
|
|
306
|
+
*/
|
|
307
|
+
localCommitResult?: CommitResult;
|
|
308
|
+
/**
|
|
309
|
+
* Conflict-shaped pend refusals reported by OTHER cohort members on their consensus responses
|
|
310
|
+
* (`ClusterRecord.applyOutcomes`), keyed by peer id. This is the arm `localPendResult` cannot
|
|
311
|
+
* cover: the refusing member is frequently not the coordinating node, and its verdict used to
|
|
312
|
+
* stay on that member while the writer was told the pend won. Unsigned advisory data — an
|
|
313
|
+
* entry means "retry", never "this write was invalid". Absent when nobody reported one.
|
|
314
|
+
*
|
|
315
|
+
* Residual: a member that reaches consensus only via the scheduled commit-retry timer applies
|
|
316
|
+
* after this method has already resolved, so its refusal arrives too late to appear here. The
|
|
317
|
+
* member-side commit-promise guard (`validateCommitAgainstRefusedPend`) is the backstop for
|
|
318
|
+
* that path.
|
|
319
|
+
*/
|
|
320
|
+
cohortPendRefusals?: { [peerId: string]: StaleFailure };
|
|
321
|
+
}> {
|
|
322
|
+
// The coordinating block id is derived HERE, from the key this method is already handed, rather
|
|
323
|
+
// than being set by each caller's message builder: a member's membership admission gate derives
|
|
324
|
+
// its own cohort view from this field, and a builder that forgets it silently downgrades the gate
|
|
325
|
+
// to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
|
|
326
|
+
// admitted at pend, refused at commit). Doing it at the single choke point means a future message
|
|
327
|
+
// builder cannot reintroduce the gap.
|
|
328
|
+
//
|
|
329
|
+
// Two constraints this shape exists to satisfy:
|
|
330
|
+
// - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
|
|
331
|
+
// N concurrent calls, one per block. In-place mutation would leak one block's id into another
|
|
332
|
+
// block's transaction.
|
|
333
|
+
// - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
|
|
334
|
+
// not just its first block, so this must not overwrite it. Tested on `length`, not on the
|
|
335
|
+
// field: an empty list carries no id for a member to derive from, so preserving one would be
|
|
336
|
+
// the same silent downgrade to the fallback floor this choke point exists to prevent.
|
|
337
|
+
const coordinated: RepoMessage = message.coordinatingBlockIds?.length
|
|
338
|
+
? message
|
|
339
|
+
: { ...message, coordinatingBlockIds: [blockId] };
|
|
340
|
+
|
|
341
|
+
// Get the cluster peers for this block
|
|
342
|
+
const peers = await this.getClusterForBlock(blockId);
|
|
343
|
+
|
|
344
|
+
// Bind the responsible membership into the transaction identity (v2): the digest is folded into
|
|
345
|
+
// the messageHash below, so two different peer sets produce two different messageHashes rather
|
|
346
|
+
// than one hash with a silent internal disagreement about who is responsible.
|
|
347
|
+
const membershipDigestValue = await membershipDigest(peers);
|
|
348
|
+
|
|
349
|
+
// Create a unique hash for this transaction (over message + membership digest). Hashing the
|
|
350
|
+
// coordinating-block-bearing copy is what makes the field tamper-evident in transit — and it also
|
|
351
|
+
// makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
|
|
352
|
+
// identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
|
|
353
|
+
const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
|
|
354
|
+
|
|
355
|
+
// Create a cluster record for this transaction
|
|
356
|
+
const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
|
|
357
|
+
log('cluster-tx:start', {
|
|
358
|
+
messageHash,
|
|
359
|
+
blockId,
|
|
360
|
+
peerCount: Object.keys(peers ?? {}).length,
|
|
361
|
+
allowDownsize: this.cfg.allowClusterDownsize,
|
|
362
|
+
configuredSize: this.cfg.clusterSize,
|
|
363
|
+
suggestedSize: record.suggestedClusterSize,
|
|
364
|
+
minRequiredSize: record.minRequiredSize
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
// Create a new pending transaction
|
|
368
|
+
const transactionPromise = this.executeTransaction(peers, record);
|
|
369
|
+
const pending = new Pending(transactionPromise);
|
|
370
|
+
|
|
371
|
+
// Store the transaction state
|
|
372
|
+
const state: ClusterTransactionState = {
|
|
373
|
+
messageHash,
|
|
374
|
+
record,
|
|
375
|
+
pending,
|
|
376
|
+
lastUpdate: this.now()
|
|
377
|
+
};
|
|
378
|
+
this.transactions.set(messageHash, state);
|
|
379
|
+
this.persistCoordinatorState(messageHash, record, 'promising');
|
|
380
|
+
log('cluster-tx:transaction-store', {
|
|
381
|
+
messageHash,
|
|
382
|
+
transactionKeys: Array.from(this.transactions.keys())
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
// Wait for the transaction to complete
|
|
386
|
+
try {
|
|
387
|
+
const result = await pending.result();
|
|
388
|
+
// Check if the local cluster already executed the operations during consensus
|
|
389
|
+
const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
|
|
390
|
+
const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
|
|
391
|
+
const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
|
|
392
|
+
// Self is excluded: this node's own member verdict is already carried, more directly and
|
|
393
|
+
// without the wire round trip, by `localPendResult` — and leaving it in both places would
|
|
394
|
+
// make the coordinator's "prefer local" rule ambiguous.
|
|
395
|
+
// Re-checked here rather than trusted: members are supposed to report only conflict-shaped
|
|
396
|
+
// refusals, but the field arrives off the wire, so anything else (a success, a bare-reason
|
|
397
|
+
// fault, a malformed entry) is dropped instead of being handed to a caller that would read
|
|
398
|
+
// it as a retryable conflict.
|
|
399
|
+
const selfId = this.localCluster?.peerId.toString();
|
|
400
|
+
const cohortPendRefusals: { [peerId: string]: StaleFailure } = {};
|
|
401
|
+
for (const [peerId, outcome] of Object.entries(result.applyOutcomes ?? {})) {
|
|
402
|
+
const pend = outcome?.pend;
|
|
403
|
+
if (peerId === selfId || pend === undefined || pend.success || !isConflictFailure(pend)) continue;
|
|
404
|
+
cohortPendRefusals[peerId] = pend;
|
|
405
|
+
}
|
|
406
|
+
return {
|
|
407
|
+
record: result,
|
|
408
|
+
localExecuted,
|
|
409
|
+
...(localPendResult === undefined ? {} : { localPendResult }),
|
|
410
|
+
...(localCommitResult === undefined ? {} : { localCommitResult }),
|
|
411
|
+
...(Object.keys(cohortPendRefusals).length === 0 ? {} : { cohortPendRefusals })
|
|
412
|
+
};
|
|
413
|
+
} finally {
|
|
414
|
+
const stored = this.transactions.get(messageHash);
|
|
415
|
+
const retrySnapshot = stored?.retry ? {
|
|
416
|
+
attempt: stored.retry.attempt,
|
|
417
|
+
pending: Array.from(stored.retry.pendingPeers ?? [])
|
|
418
|
+
} : undefined;
|
|
419
|
+
log('cluster-tx:complete', {
|
|
420
|
+
messageHash,
|
|
421
|
+
finalPromises: stored ? Object.keys(stored.record.promises ?? {}) : undefined,
|
|
422
|
+
finalCommits: stored ? Object.keys(stored.record.commits ?? {}) : undefined,
|
|
423
|
+
retry: retrySnapshot
|
|
424
|
+
});
|
|
425
|
+
// Don't remove transaction immediately if retries are scheduled
|
|
426
|
+
// Let the retry completion or abort handle cleanup
|
|
427
|
+
if (!stored?.retry) {
|
|
428
|
+
// Wait a bit before cleanup to allow any in-flight responses to arrive
|
|
429
|
+
this.setTimer(() => {
|
|
430
|
+
this.transactions.delete(messageHash);
|
|
431
|
+
this.deleteCoordinatorState(messageHash);
|
|
432
|
+
log('cluster-tx:transaction-remove', {
|
|
433
|
+
messageHash,
|
|
434
|
+
remaining: Array.from(this.transactions.keys())
|
|
435
|
+
});
|
|
436
|
+
}, 100);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* Executes the full transaction process
|
|
443
|
+
*/
|
|
444
|
+
private async executeTransaction(peers: ClusterPeers, record: ClusterRecord): Promise<ClusterRecord> {
|
|
445
|
+
const peerCount = Object.keys(peers).length;
|
|
446
|
+
|
|
447
|
+
// Validate against minimum cluster size
|
|
448
|
+
if (peerCount < this.cfg.minAbsoluteClusterSize) {
|
|
449
|
+
const validated = await this.validateSmallCluster(peerCount, peers);
|
|
450
|
+
if (!validated) {
|
|
451
|
+
log('cluster-tx:reject-too-small', {
|
|
452
|
+
peerCount,
|
|
453
|
+
minRequired: this.cfg.minAbsoluteClusterSize
|
|
454
|
+
});
|
|
455
|
+
throw new Error(`Cluster size ${peerCount} below minimum ${this.cfg.minAbsoluteClusterSize} and not validated`);
|
|
456
|
+
}
|
|
457
|
+
log('cluster-tx:small-cluster-validated', { peerCount });
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
// Check configured cluster size
|
|
461
|
+
if (!this.cfg.allowClusterDownsize && peerCount < this.cfg.clusterSize) {
|
|
462
|
+
log('cluster-tx:reject-downsize', { peerCount, required: this.cfg.clusterSize });
|
|
463
|
+
throw new Error(`Cluster size ${peerCount} below configured minimum ${this.cfg.clusterSize}`);
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
// Collect promises with super-majority requirement
|
|
467
|
+
const promised = await this.collectPromises(peers, record);
|
|
468
|
+
const superMajority = Math.ceil(peerCount * this.cfg.superMajorityThreshold);
|
|
469
|
+
|
|
470
|
+
// Count approvals, rejections and conflict votes separately. A `conflict` vote is a member
|
|
471
|
+
// saying "not now — I hold the race winner": it must count toward NEITHER approvals NOR
|
|
472
|
+
// rejections, or a lost race would masquerade as a validator rejection (permanent) or as
|
|
473
|
+
// silence (indistinguishable from an unreachable cohort) — both wrong.
|
|
474
|
+
const promises = promised.record.promises;
|
|
475
|
+
const approvalCount = Object.values(promises).filter(sig => sig.type === 'approve').length;
|
|
476
|
+
const rejectionCount = Object.values(promises).filter(sig => sig.type === 'reject').length;
|
|
477
|
+
const conflictCount = Object.values(promises).filter(sig => sig.type === 'conflict').length;
|
|
478
|
+
|
|
479
|
+
// Check if rejections make super-majority impossible
|
|
480
|
+
// If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
|
|
481
|
+
const maxAllowedRejections = peerCount - superMajority;
|
|
482
|
+
if (rejectionCount > maxAllowedRejections) {
|
|
483
|
+
const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
|
|
484
|
+
.flatMap(([peerId, sig]) => sig.type === 'reject' ? [[peerId, sig.rejectReason ?? 'unknown'] as const] : []));
|
|
485
|
+
const rejectReasons = Object.entries(rejectReasonsByPeer)
|
|
486
|
+
.map(([peerId, reason]) => `${peerId}: ${reason}`)
|
|
487
|
+
.join('; ');
|
|
488
|
+
log('cluster-tx:rejected-by-validators', {
|
|
489
|
+
messageHash: record.messageHash,
|
|
490
|
+
peerCount,
|
|
491
|
+
rejections: rejectionCount,
|
|
492
|
+
maxAllowed: maxAllowedRejections,
|
|
493
|
+
reasons: rejectReasons
|
|
494
|
+
});
|
|
495
|
+
this.updateTransactionRecord(promised.record, 'rejected-by-validators');
|
|
496
|
+
// Abandoning here without telling anyone leaves every member that voted holding this
|
|
497
|
+
// transaction in its own reservation table, blocking its blocks until that member's
|
|
498
|
+
// staleness sweep fires — and each retry we throw back to the caller plants a fresh
|
|
499
|
+
// reservation, so the block never frees. The merged record carries enough signed
|
|
500
|
+
// rejections to *prove* the transaction is dead, so replaying it to the cohort makes
|
|
501
|
+
// every member recompute `Rejected` and clear immediately. Proof-carrying, so a member
|
|
502
|
+
// need not trust us: it verifies the signatures it is shown.
|
|
503
|
+
this.broadcastAbandonment(promised.record, 'rejected-by-validators');
|
|
504
|
+
throw new ValidatorRejectionError(
|
|
505
|
+
`Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
|
|
506
|
+
rejectReasonsByPeer);
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
// A conflict-answered shortfall is a LOST RACE, not a validator verdict and not silence.
|
|
510
|
+
// Checked after the rejection threshold (a genuine validator rejection still wins) and
|
|
511
|
+
// before the generic shortfall (which must stay reserved for the genuinely-silent cohort).
|
|
512
|
+
if (conflictCount > 0 && approvalCount < superMajority) {
|
|
513
|
+
const conflicts = Object.fromEntries(Object.entries(promises)
|
|
514
|
+
.flatMap(([peerId, sig]) => sig.type === 'conflict' ? [[peerId, sig.conflictWith] as const] : []));
|
|
515
|
+
log('cluster-tx:conflict-race-lost', {
|
|
516
|
+
messageHash: record.messageHash,
|
|
517
|
+
peerCount,
|
|
518
|
+
approvals: approvalCount,
|
|
519
|
+
rejections: rejectionCount,
|
|
520
|
+
conflicts,
|
|
521
|
+
superMajority
|
|
522
|
+
});
|
|
523
|
+
this.updateTransactionRecord(promised.record, 'conflict-race-lost');
|
|
524
|
+
// Broadcast only when the merged record itself PROVES the transaction can no longer reach
|
|
525
|
+
// super-majority (members re-derive ConflictSuperseded/Rejected from the signed votes and
|
|
526
|
+
// clear their reservations immediately). Below that bar the record proves nothing and a
|
|
527
|
+
// broadcast would be the unauthenticated "forget this" the shortfall NOTE below refuses.
|
|
528
|
+
if (rejectionCount + conflictCount > maxAllowedRejections) {
|
|
529
|
+
this.broadcastAbandonment(promised.record, 'conflict-race-lost');
|
|
530
|
+
}
|
|
531
|
+
throw new ConflictRaceLostError(
|
|
532
|
+
`Conflict race lost: ${conflictCount}/${peerCount} member(s) hold a conflicting winner (${approvalCount}/${superMajority} approvals)`,
|
|
533
|
+
conflicts);
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
if (peerCount > 1 && approvalCount < superMajority) {
|
|
537
|
+
log('cluster-tx:supermajority-failed', {
|
|
538
|
+
messageHash: record.messageHash,
|
|
539
|
+
peerCount,
|
|
540
|
+
approvals: approvalCount,
|
|
541
|
+
rejections: rejectionCount,
|
|
542
|
+
superMajority,
|
|
543
|
+
threshold: this.cfg.superMajorityThreshold
|
|
544
|
+
});
|
|
545
|
+
this.updateTransactionRecord(promised.record, 'supermajority-failed');
|
|
546
|
+
// NOTE: deliberately NOT broadcast, unlike the rejected-by-validators branch above. With
|
|
547
|
+
// conflict-answered shortfalls peeled off above, we get here only because peers did not
|
|
548
|
+
// answer at all, so the record carries no signed evidence that the transaction is dead — a
|
|
549
|
+
// broadcast would be an unauthenticated "forget this" that any caller could use to clear a
|
|
550
|
+
// live transaction out of a member's reservation table. Members that DID vote are freed by
|
|
551
|
+
// their own staleness sweep instead.
|
|
552
|
+
// NOTE: the message below is load-bearing wire text — the consuming repo
|
|
553
|
+
// (sereus cadre-core control-write-retry) matches it verbatim to retry a genuinely-silent
|
|
554
|
+
// cohort. Keep it byte-identical, and never fold conflict votes into its rejection count.
|
|
555
|
+
throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
// Mark as disputed when minority rejections exist but super-majority approves
|
|
559
|
+
if (rejectionCount > 0 && approvalCount >= superMajority) {
|
|
560
|
+
const rejectingPeers: string[] = [];
|
|
561
|
+
const rejectReasons: { [peerId: string]: string } = {};
|
|
562
|
+
for (const [peerId, sig] of Object.entries(promises)) {
|
|
563
|
+
if (sig.type === 'reject') {
|
|
564
|
+
rejectingPeers.push(peerId);
|
|
565
|
+
rejectReasons[peerId] = sig.rejectReason ?? 'unknown';
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
promised.record.disputed = true;
|
|
569
|
+
promised.record.disputeEvidence = { rejectingPeers, rejectReasons };
|
|
570
|
+
log('cluster-tx:disputed', {
|
|
571
|
+
messageHash: record.messageHash,
|
|
572
|
+
rejectingPeers,
|
|
573
|
+
rejectReasons,
|
|
574
|
+
approvalCount,
|
|
575
|
+
rejectionCount,
|
|
576
|
+
peerCount
|
|
577
|
+
});
|
|
578
|
+
// [dispute-subsystem-dormant] Evidence is computed and persisted but initiateDispute() is
|
|
579
|
+
// intentionally NOT called here. Dispute origination stays dormant pending arbitrator-set
|
|
580
|
+
// anchoring — without it a forged synthetic cohort passes resolution.
|
|
581
|
+
// Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
|
|
582
|
+
// Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
|
|
586
|
+
return await this.commitTransaction(promised.record);
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/**
|
|
590
|
+
* The block's cohort peer ids as currently derivable. Empty when `findCluster` fails
|
|
591
|
+
* (getClusterForBlock swallows the throw), so a caller branching on `length <= 1` is also taking
|
|
592
|
+
* the degraded-routing branch; `CoordinatorRepo.commit` uses the ids to log whether a solo cohort
|
|
593
|
+
* is genuinely just self or a routing failure.
|
|
594
|
+
*/
|
|
595
|
+
async getClusterPeerIds(blockId: BlockId): Promise<string[]> {
|
|
596
|
+
const peers = await this.getClusterForBlock(blockId);
|
|
597
|
+
return Object.keys(peers ?? {});
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
/** {@link getClusterPeerIds}, counted. Derived from it rather than re-deriving the cohort, so the
|
|
601
|
+
* size a caller branches on and the ids it logs can never come from two different rules. */
|
|
602
|
+
async getClusterSize(blockId: BlockId): Promise<number> {
|
|
603
|
+
return (await this.getClusterPeerIds(blockId)).length;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
/**
|
|
607
|
+
* Validate that a small cluster size is legitimate by querying remote peers
|
|
608
|
+
* for their network size estimates. Returns true if estimates roughly agree.
|
|
609
|
+
*/
|
|
610
|
+
private async validateSmallCluster(localSize: number, _peers: ClusterPeers): Promise<boolean> {
|
|
611
|
+
// If we have FRET and it shows confident estimate
|
|
612
|
+
if (this.fretService) {
|
|
613
|
+
try {
|
|
614
|
+
const estimate = this.fretService.getNetworkSizeEstimate();
|
|
615
|
+
if (estimate.confidence > 0.5) {
|
|
616
|
+
// Check if FRET estimate roughly matches observed cluster size
|
|
617
|
+
const orderOfMagnitude = Math.floor(Math.log10(estimate.size_estimate + 1));
|
|
618
|
+
const localOrderOfMagnitude = Math.floor(Math.log10(localSize + 1));
|
|
619
|
+
|
|
620
|
+
// If within same order of magnitude, accept it
|
|
621
|
+
if (Math.abs(orderOfMagnitude - localOrderOfMagnitude) <= 1) {
|
|
622
|
+
log('cluster-tx:small-cluster-validated-by-fret', {
|
|
623
|
+
localSize,
|
|
624
|
+
fretEstimate: estimate.size_estimate,
|
|
625
|
+
confidence: estimate.confidence,
|
|
626
|
+
sources: estimate.sources
|
|
627
|
+
});
|
|
628
|
+
return true;
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
} catch (err) {
|
|
632
|
+
// Ignore errors
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
// Fallback: with no confident network-size estimate, fail CLOSED by default.
|
|
637
|
+
// An undersized cluster with no way to justify its size is unsafe (a lone/
|
|
638
|
+
// near-lone node could rubber-stamp its own writes), so reject unless the
|
|
639
|
+
// operator has explicitly opted in via allowUnvalidatedSmallCluster (e.g.
|
|
640
|
+
// single-node / local dev knowingly running below the floor).
|
|
641
|
+
const admit = this.cfg.allowUnvalidatedSmallCluster ?? false;
|
|
642
|
+
log('cluster-tx:small-cluster-no-confident-estimate', {
|
|
643
|
+
localSize,
|
|
644
|
+
reason: 'no-confident-network-size-estimate',
|
|
645
|
+
admit
|
|
646
|
+
});
|
|
647
|
+
return admit;
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
/**
|
|
651
|
+
* Collects promises from all peers in the cluster
|
|
652
|
+
*/
|
|
653
|
+
private async collectPromises(peers: ClusterPeers, record: ClusterRecord): Promise<{ record: ClusterRecord }> {
|
|
654
|
+
const peerIds = Object.keys(peers);
|
|
655
|
+
const summary: ClusterLogPeerOutcome[] = [];
|
|
656
|
+
if (verbose) {
|
|
657
|
+
const peerDetail = peerIds.map(id => ({
|
|
658
|
+
id: id.substring(0, 12),
|
|
659
|
+
addrs: peers[id]?.multiaddrs?.length ?? 0
|
|
660
|
+
}));
|
|
661
|
+
log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
|
|
662
|
+
}
|
|
663
|
+
// For each peer, create a client and request a promise. A remote promise rides
|
|
664
|
+
// a libp2p stream that a relayed (limited) connection can reset transiently, so
|
|
665
|
+
// each remote request gets `promiseImmediateRetries` in-line re-attempts before
|
|
666
|
+
// it counts as a failure — without this a single relayed reset drops the peer and
|
|
667
|
+
// sinks super-majority (the commit broadcast already has the same guard).
|
|
668
|
+
const promiseRequests = peerIds.map(peerIdStr => {
|
|
669
|
+
const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
|
|
670
|
+
log('cluster-tx:promise-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
|
|
671
|
+
return new Pending(this.updateMember(peerIdStr, record, this.promiseImmediateRetries, 'promise'));
|
|
672
|
+
});
|
|
673
|
+
|
|
674
|
+
// Wait for all promises to complete
|
|
675
|
+
const results = await Promise.all(promiseRequests.map((p, idx) => p.result().then(res => {
|
|
676
|
+
const peerIdStr = peerIds[idx]!;
|
|
677
|
+
log('cluster-tx:promise-response', {
|
|
678
|
+
messageHash: record.messageHash,
|
|
679
|
+
peerId: peerIdStr,
|
|
680
|
+
success: true,
|
|
681
|
+
returnedPromises: Object.keys(res.promises ?? {}),
|
|
682
|
+
returnedCommits: Object.keys(res.commits ?? {})
|
|
683
|
+
});
|
|
684
|
+
summary.push({ peerId: peerIdStr, success: true });
|
|
685
|
+
return res;
|
|
686
|
+
}).catch(err => {
|
|
687
|
+
const peerIdStr = peerIds[idx]!;
|
|
688
|
+
log('cluster-tx:promise-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
|
|
689
|
+
summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
|
|
690
|
+
this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `promise:${record.messageHash}`);
|
|
691
|
+
return null;
|
|
692
|
+
})));
|
|
693
|
+
const successes = summary.filter(entry => entry.success).map(entry => entry.peerId);
|
|
694
|
+
const failures = summary.filter(entry => !entry.success);
|
|
695
|
+
log('cluster-tx:promise-summary', {
|
|
696
|
+
messageHash: record.messageHash,
|
|
697
|
+
successes,
|
|
698
|
+
failures
|
|
699
|
+
});
|
|
700
|
+
|
|
701
|
+
log('cluster-tx:promise-merge-begin', {
|
|
702
|
+
messageHash: record.messageHash,
|
|
703
|
+
initialPromises: Object.keys(record.promises ?? {}),
|
|
704
|
+
transactionsKeys: Array.from(this.transactions.keys()),
|
|
705
|
+
hasTransaction: this.transactions.has(record.messageHash)
|
|
706
|
+
});
|
|
707
|
+
|
|
708
|
+
// Merge all promises into the record
|
|
709
|
+
for (const result of results.filter(Boolean) as ClusterRecord[]) {
|
|
710
|
+
log('cluster-tx:promise-merge-input', {
|
|
711
|
+
messageHash: record.messageHash,
|
|
712
|
+
resultFrom: Object.keys(result.promises ?? {}),
|
|
713
|
+
recordBefore: Object.keys(record.promises ?? {})
|
|
714
|
+
});
|
|
715
|
+
const resultPromises = Object.keys(result.promises ?? {});
|
|
716
|
+
log('cluster-tx:promise-merge-result', {
|
|
717
|
+
messageHash: record.messageHash,
|
|
718
|
+
peerPromises: resultPromises
|
|
719
|
+
});
|
|
720
|
+
if (typeof record.suggestedClusterSize === 'number' && typeof result.suggestedClusterSize === 'number') {
|
|
721
|
+
const expected = result.suggestedClusterSize;
|
|
722
|
+
const actual = Object.keys(peers).length;
|
|
723
|
+
const maxDiff = Math.ceil(Math.max(1, expected * this.cfg.clusterSizeTolerance));
|
|
724
|
+
if (Math.abs(actual - expected) > maxDiff) {
|
|
725
|
+
log('cluster-tx:size-variance', { expected, actual, tolerance: this.cfg.clusterSizeTolerance });
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
record.promises = { ...record.promises, ...result.promises };
|
|
729
|
+
log('cluster-tx:promise-merge-after', {
|
|
730
|
+
messageHash: record.messageHash,
|
|
731
|
+
mergedPromises: Object.keys(record.promises ?? {})
|
|
732
|
+
});
|
|
733
|
+
}
|
|
734
|
+
log('cluster-tx:promise-merge', {
|
|
735
|
+
messageHash: record.messageHash,
|
|
736
|
+
mergedPromises: Object.keys(record.promises ?? {})
|
|
737
|
+
});
|
|
738
|
+
log('cluster-tx:promise-merge-end', {
|
|
739
|
+
messageHash: record.messageHash,
|
|
740
|
+
finalPromises: Object.keys(record.promises ?? {}),
|
|
741
|
+
transactionsEntry: this.transactions.get(record.messageHash)
|
|
742
|
+
});
|
|
743
|
+
this.updateTransactionRecord(record, 'after-promises');
|
|
744
|
+
return { record };
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
/**
|
|
748
|
+
* Commits the transaction to all peers in the cluster
|
|
749
|
+
*/
|
|
750
|
+
private async commitTransaction(record: ClusterRecord): Promise<ClusterRecord> {
|
|
751
|
+
// For each peer, create a client and send the commit
|
|
752
|
+
const peerIds = Object.keys(record.peers);
|
|
753
|
+
const summary: ClusterLogPeerOutcome[] = [];
|
|
754
|
+
if (verbose) {
|
|
755
|
+
const peerDetail = peerIds.map(id => ({
|
|
756
|
+
id: id.substring(0, 12),
|
|
757
|
+
addrs: record.peers[id]?.multiaddrs?.length ?? 0
|
|
758
|
+
}));
|
|
759
|
+
log('cluster-tx:commit-peers', { messageHash: record.messageHash, peers: peerDetail });
|
|
760
|
+
}
|
|
761
|
+
// Send the record with promises to all peers
|
|
762
|
+
// Each peer will add its own commit signature
|
|
763
|
+
const commitPayload = {
|
|
764
|
+
...record
|
|
765
|
+
};
|
|
766
|
+
// No per-peer immediate retry here: a commit-collection failure is recovered
|
|
767
|
+
// downstream by broadcastMergedRecord's in-line retry and the scheduled
|
|
768
|
+
// commit-retry timer. (The promise phase has no such backstop, which is why
|
|
769
|
+
// collectPromises gets the immediate retry instead.)
|
|
770
|
+
const commitRequests = peerIds.map(peerIdStr => {
|
|
771
|
+
const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
|
|
772
|
+
log('cluster-tx:commit-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
|
|
773
|
+
const promise = isLocal
|
|
774
|
+
? this.localCluster!.update(commitPayload)
|
|
775
|
+
: this.createClusterClient(peerIdFromString(peerIdStr)).update(commitPayload);
|
|
776
|
+
return new Pending(promise);
|
|
777
|
+
});
|
|
778
|
+
|
|
779
|
+
// Wait for all commits to complete
|
|
780
|
+
const results = await Promise.all(commitRequests.map((p, idx) => p.result().then(res => {
|
|
781
|
+
const peerIdStr = peerIds[idx]!;
|
|
782
|
+
log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: true });
|
|
783
|
+
summary.push({ peerId: peerIdStr, success: true });
|
|
784
|
+
return res;
|
|
785
|
+
}).catch(err => {
|
|
786
|
+
const peerIdStr = peerIds[idx]!;
|
|
787
|
+
log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
|
|
788
|
+
summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
|
|
789
|
+
this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `commit:${record.messageHash}`);
|
|
790
|
+
return null;
|
|
791
|
+
})));
|
|
792
|
+
const commitSuccesses = summary.filter(entry => entry.success).map(entry => entry.peerId);
|
|
793
|
+
const commitFailures = summary.filter(entry => !entry.success);
|
|
794
|
+
log('cluster-tx:commit-summary', {
|
|
795
|
+
messageHash: record.messageHash,
|
|
796
|
+
successes: commitSuccesses,
|
|
797
|
+
failures: commitFailures
|
|
798
|
+
});
|
|
799
|
+
log('cluster-tx:commit-merge-begin', {
|
|
800
|
+
messageHash: record.messageHash,
|
|
801
|
+
initialCommits: Object.keys(record.commits ?? {}),
|
|
802
|
+
transactionsEntry: this.transactions.get(record.messageHash)
|
|
803
|
+
});
|
|
804
|
+
|
|
805
|
+
// Members that already held super-majority promises reach consensus during THIS round rather
|
|
806
|
+
// than during the broadcast below, so their apply verdicts arrive on these responses. Collect
|
|
807
|
+
// both; the broadcast's copy wins on overlap, being the later of the two.
|
|
808
|
+
mergeApplyOutcomes(record, collectApplyOutcomes(results.map((response, idx) => ({ peerId: peerIds[idx]!, response }))));
|
|
809
|
+
|
|
810
|
+
// Merge all commits into the record
|
|
811
|
+
for (const result of results.filter(Boolean) as ClusterRecord[]) {
|
|
812
|
+
log('cluster-tx:commit-merge-input', {
|
|
813
|
+
messageHash: record.messageHash,
|
|
814
|
+
resultFrom: Object.keys(result.commits ?? {}),
|
|
815
|
+
recordBefore: Object.keys(record.commits ?? {})
|
|
816
|
+
});
|
|
817
|
+
log('cluster-tx:commit-merge-result', {
|
|
818
|
+
messageHash: record.messageHash,
|
|
819
|
+
peerCommits: Object.keys(result.commits ?? {})
|
|
820
|
+
});
|
|
821
|
+
record.commits = { ...record.commits, ...result.commits };
|
|
822
|
+
log('cluster-tx:commit-merge-after', {
|
|
823
|
+
messageHash: record.messageHash,
|
|
824
|
+
mergedCommits: Object.keys(record.commits ?? {})
|
|
825
|
+
});
|
|
826
|
+
}
|
|
827
|
+
log('cluster-tx:commit-merge', {
|
|
828
|
+
messageHash: record.messageHash,
|
|
829
|
+
mergedCommits: Object.keys(record.commits ?? {})
|
|
830
|
+
});
|
|
831
|
+
log('cluster-tx:commit-merge-end', {
|
|
832
|
+
messageHash: record.messageHash,
|
|
833
|
+
finalCommits: Object.keys(record.commits ?? {}),
|
|
834
|
+
transactionsEntry: this.transactions.get(record.messageHash)
|
|
835
|
+
});
|
|
836
|
+
this.updateTransactionRecord(record, 'after-commit');
|
|
837
|
+
|
|
838
|
+
// Check for simple majority (>50%) - this proves commitment
|
|
839
|
+
const peerCount = Object.keys(record.peers).length;
|
|
840
|
+
const simpleMajority = Math.floor(peerCount * this.cfg.simpleMajorityThreshold) + 1;
|
|
841
|
+
const commitCount = Object.keys(record.commits).length;
|
|
842
|
+
|
|
843
|
+
if (commitCount >= simpleMajority) {
|
|
844
|
+
log('cluster-tx:commit-majority-reached', {
|
|
845
|
+
messageHash: record.messageHash,
|
|
846
|
+
commitCount,
|
|
847
|
+
simpleMajority,
|
|
848
|
+
peerCount,
|
|
849
|
+
threshold: this.cfg.simpleMajorityThreshold
|
|
850
|
+
});
|
|
851
|
+
// Broadcast the merged record (with all commit signatures) to ALL peers
|
|
852
|
+
// so each peer can independently reach consensus and execute the operations.
|
|
853
|
+
// Without this, only the coordinator's local cluster executes — remote peers
|
|
854
|
+
// never see enough commits to reach consensus on their own.
|
|
855
|
+
const { failures: broadcastFailures, applyOutcomes } = await this.broadcastMergedRecord(record, peerIds);
|
|
856
|
+
mergeApplyOutcomes(record, applyOutcomes);
|
|
857
|
+
if (broadcastFailures.length > 0) {
|
|
858
|
+
this.scheduleCommitRetry(record.messageHash, record, broadcastFailures);
|
|
859
|
+
} else {
|
|
860
|
+
this.clearRetry(record.messageHash);
|
|
861
|
+
}
|
|
862
|
+
} else {
|
|
863
|
+
const missingPeers = commitFailures.map(entry => entry.peerId);
|
|
864
|
+
if (missingPeers.length > 0) {
|
|
865
|
+
this.scheduleCommitRetry(record.messageHash, record, missingPeers);
|
|
866
|
+
} else {
|
|
867
|
+
this.clearRetry(record.messageHash);
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
return record;
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
/**
|
|
874
|
+
* Broadcast the merged commit record to every peer, with `commitBroadcastImmediateRetries`
|
|
875
|
+
* in-line re-attempts per peer before giving up. The libp2p connection used during
|
|
876
|
+
* the prior commit phase is typically still warm, so a single immediate retry recovers
|
|
877
|
+
* most transient stream errors without falling back to the scheduled retry timer.
|
|
878
|
+
* Local cluster is invoked exactly once — local failures are fatal, not transient.
|
|
879
|
+
*/
|
|
880
|
+
private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[]; applyOutcomes?: ClusterRecord['applyOutcomes'] }> {
|
|
881
|
+
const results = await Promise.all(peerIds.map(async peerIdStr => {
|
|
882
|
+
try {
|
|
883
|
+
const response = await this.updateMember(peerIdStr, record, this.commitBroadcastImmediateRetries, 'commit-broadcast');
|
|
884
|
+
return { peerId: peerIdStr, success: true as const, response };
|
|
885
|
+
} catch (err) {
|
|
886
|
+
log('cluster-tx:consensus-broadcast-error', {
|
|
887
|
+
messageHash: record.messageHash,
|
|
888
|
+
peerId: peerIdStr,
|
|
889
|
+
error: err instanceof Error ? err.message : String(err)
|
|
890
|
+
});
|
|
891
|
+
return { peerId: peerIdStr, success: false as const, response: undefined };
|
|
892
|
+
}
|
|
893
|
+
}));
|
|
894
|
+
const failures = results.filter(r => !r.success).map(r => r.peerId);
|
|
895
|
+
// This broadcast is where members actually apply the operations, so their responses carry the
|
|
896
|
+
// only report the coordinator ever gets of what each member's OWN storage said. Collecting it
|
|
897
|
+
// here is what lets a pend refused by a non-coordinating member reach the writer as a conflict
|
|
898
|
+
// instead of the fabricated success that used to fork the block.
|
|
899
|
+
//
|
|
900
|
+
// Each peer's entry is taken from that peer's OWN response and re-keyed under the peer we
|
|
901
|
+
// asked, so a member cannot report an outcome on another member's behalf by echoing a record
|
|
902
|
+
// full of entries. Unsigned and advisory either way — see ClusterRecord.applyOutcomes.
|
|
903
|
+
const applyOutcomes = collectApplyOutcomes(results);
|
|
904
|
+
return { failures, ...(applyOutcomes === undefined ? {} : { applyOutcomes }) };
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
/**
|
|
908
|
+
* Fire-and-forget replay of an abandoned transaction's record to every peer in its cohort.
|
|
909
|
+
*
|
|
910
|
+
* Called only where the record itself proves the transaction is dead (enough signed rejections that
|
|
911
|
+
* super-majority is unreachable). Each member re-derives `TransactionPhase.Rejected` from the votes
|
|
912
|
+
* it verifies and drops the entry from its own reservation table, freeing the blocks immediately
|
|
913
|
+
* instead of after its 2 s staleness window. No new message type and no wire-format change — this is
|
|
914
|
+
* the same `update()` every other phase uses.
|
|
915
|
+
*
|
|
916
|
+
* Never awaited into the caller's throw and never rethrows: an abandonment must not turn into a
|
|
917
|
+
* *different* failure, and the staleness sweep remains the backstop if delivery fails.
|
|
918
|
+
*/
|
|
919
|
+
private broadcastAbandonment(record: ClusterRecord, reason: string): void {
|
|
920
|
+
const peerIds = Object.keys(record.peers);
|
|
921
|
+
log('cluster-tx:abandon-broadcast', { messageHash: record.messageHash, reason, peerIds });
|
|
922
|
+
void Promise.all(peerIds.map(async peerIdStr => {
|
|
923
|
+
try {
|
|
924
|
+
await this.updateMember(peerIdStr, record, 0, 'abandon-broadcast');
|
|
925
|
+
} catch (err) {
|
|
926
|
+
log('cluster-tx:abandon-broadcast-error', {
|
|
927
|
+
messageHash: record.messageHash,
|
|
928
|
+
peerId: peerIdStr,
|
|
929
|
+
error: err instanceof Error ? err.message : String(err)
|
|
930
|
+
});
|
|
931
|
+
}
|
|
932
|
+
}));
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
private updateTransactionRecord(record: ClusterRecord, stage: string): void {
|
|
936
|
+
const state = this.transactions.get(record.messageHash);
|
|
937
|
+
if (!state) {
|
|
938
|
+
log('cluster-tx:transaction-update-miss', { messageHash: record.messageHash, stage });
|
|
939
|
+
return;
|
|
940
|
+
}
|
|
941
|
+
state.record = { ...record };
|
|
942
|
+
state.lastUpdate = this.now();
|
|
943
|
+
log('cluster-tx:transaction-update', {
|
|
944
|
+
messageHash: record.messageHash,
|
|
945
|
+
stage,
|
|
946
|
+
promises: Object.keys(record.promises ?? {}),
|
|
947
|
+
commits: Object.keys(record.commits ?? {})
|
|
948
|
+
});
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
private scheduleCommitRetry(messageHash: string, _record: ClusterRecord, missingPeers: string[]): void {
|
|
952
|
+
const state = this.transactions.get(messageHash);
|
|
953
|
+
if (!state) {
|
|
954
|
+
return;
|
|
955
|
+
}
|
|
956
|
+
const existing = state.retry;
|
|
957
|
+
const nextAttempt = (existing?.attempt ?? 0) + 1;
|
|
958
|
+
if (nextAttempt > this.retryMaxAttempts) {
|
|
959
|
+
log('cluster-tx:retry-abort', { messageHash, missingPeers });
|
|
960
|
+
return;
|
|
961
|
+
}
|
|
962
|
+
if (missingPeers.length === 0) {
|
|
963
|
+
this.clearRetry(messageHash);
|
|
964
|
+
return;
|
|
965
|
+
}
|
|
966
|
+
const pendingPeers = new Set(missingPeers);
|
|
967
|
+
const baseInterval = existing ? Math.min(existing.intervalMs * this.retryBackoffFactor, this.retryMaxIntervalMs) : this.retryInitialIntervalMs;
|
|
968
|
+
existing?.cancel?.();
|
|
969
|
+
const cancel = this.setTimer(() => {
|
|
970
|
+
void this.retryCommits(messageHash);
|
|
971
|
+
}, baseInterval);
|
|
972
|
+
state.retry = {
|
|
973
|
+
pendingPeers,
|
|
974
|
+
attempt: nextAttempt,
|
|
975
|
+
intervalMs: baseInterval,
|
|
976
|
+
cancel
|
|
977
|
+
};
|
|
978
|
+
this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
|
|
979
|
+
pendingPeers: Array.from(pendingPeers),
|
|
980
|
+
attempt: nextAttempt,
|
|
981
|
+
intervalMs: baseInterval
|
|
982
|
+
});
|
|
983
|
+
log('cluster-tx:retry-scheduled', { messageHash, attempt: nextAttempt, missingPeers, delayMs: baseInterval });
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
private async retryCommits(messageHash: string): Promise<void> {
|
|
987
|
+
const state = this.transactions.get(messageHash);
|
|
988
|
+
if (!state?.retry) {
|
|
989
|
+
return;
|
|
990
|
+
}
|
|
991
|
+
const { pendingPeers, attempt } = state.retry;
|
|
992
|
+
if (pendingPeers.size === 0) {
|
|
993
|
+
this.clearRetry(messageHash);
|
|
994
|
+
return;
|
|
995
|
+
}
|
|
996
|
+
const peerIds = Array.from(pendingPeers);
|
|
997
|
+
const record = state.record;
|
|
998
|
+
log('cluster-tx:retry-start', { messageHash, attempt, peerIds });
|
|
999
|
+
const results = await Promise.all(peerIds.map(async peerIdStr => {
|
|
1000
|
+
const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
|
|
1001
|
+
const payload: ClusterRecord = {
|
|
1002
|
+
...record,
|
|
1003
|
+
commits: record.commits
|
|
1004
|
+
};
|
|
1005
|
+
try {
|
|
1006
|
+
const res = isLocal
|
|
1007
|
+
? await this.localCluster!.update(payload)
|
|
1008
|
+
: await this.createClusterClient(peerIdFromString(peerIdStr)).update(payload);
|
|
1009
|
+
state.record.commits = { ...state.record.commits, ...res.commits };
|
|
1010
|
+
return { peerId: peerIdStr, success: true as const };
|
|
1011
|
+
} catch (err) {
|
|
1012
|
+
return {
|
|
1013
|
+
peerId: peerIdStr,
|
|
1014
|
+
success: false as const,
|
|
1015
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1016
|
+
};
|
|
1017
|
+
}
|
|
1018
|
+
}));
|
|
1019
|
+
const successes = results.filter(r => r.success).map(r => r.peerId);
|
|
1020
|
+
const failures = results.filter(r => !r.success);
|
|
1021
|
+
for (const peerId of successes) {
|
|
1022
|
+
pendingPeers.delete(peerId);
|
|
1023
|
+
}
|
|
1024
|
+
log('cluster-tx:retry-complete', { messageHash, attempt, successes, failures });
|
|
1025
|
+
if (pendingPeers.size === 0) {
|
|
1026
|
+
log('cluster-tx:retry-finished', { messageHash });
|
|
1027
|
+
this.clearRetry(messageHash);
|
|
1028
|
+
return;
|
|
1029
|
+
}
|
|
1030
|
+
if (!this.transactions.has(messageHash)) {
|
|
1031
|
+
return;
|
|
1032
|
+
}
|
|
1033
|
+
this.scheduleCommitRetry(messageHash, state.record, Array.from(pendingPeers));
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
private clearRetry(messageHash: string): void {
|
|
1037
|
+
const state = this.transactions.get(messageHash);
|
|
1038
|
+
if (!state?.retry) {
|
|
1039
|
+
return;
|
|
1040
|
+
}
|
|
1041
|
+
state.retry.cancel?.();
|
|
1042
|
+
state.retry = undefined;
|
|
1043
|
+
// Clean up the transaction after retry is complete
|
|
1044
|
+
this.setTimer(() => {
|
|
1045
|
+
this.transactions.delete(messageHash);
|
|
1046
|
+
this.deleteCoordinatorState(messageHash);
|
|
1047
|
+
log('cluster-tx:transaction-remove', {
|
|
1048
|
+
messageHash,
|
|
1049
|
+
remaining: Array.from(this.transactions.keys())
|
|
1050
|
+
});
|
|
1051
|
+
}, 100);
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
/** Fire-and-forget persist — errors are logged, never thrown. */
|
|
1055
|
+
private persistCoordinatorState(
|
|
1056
|
+
messageHash: string,
|
|
1057
|
+
record: ClusterRecord,
|
|
1058
|
+
phase: 'promising' | 'committing' | 'broadcasting',
|
|
1059
|
+
retryState?: { pendingPeers: string[]; attempt: number; intervalMs: number }
|
|
1060
|
+
): void {
|
|
1061
|
+
if (!this.stateStore) return;
|
|
1062
|
+
this.stateStore.saveCoordinatorState(messageHash, {
|
|
1063
|
+
messageHash,
|
|
1064
|
+
record,
|
|
1065
|
+
lastUpdate: this.now(),
|
|
1066
|
+
phase,
|
|
1067
|
+
retryState
|
|
1068
|
+
}).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
/** Fire-and-forget delete — errors are logged, never thrown. */
|
|
1072
|
+
private deleteCoordinatorState(messageHash: string): void {
|
|
1073
|
+
if (!this.stateStore) return;
|
|
1074
|
+
this.stateStore.deleteCoordinatorState(messageHash)
|
|
1075
|
+
.catch(err => log('cluster-tx:persist-delete-error', { messageHash, error: (err as Error).message }));
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
/**
|
|
1079
|
+
* Recover coordinator transactions from persistent store after a restart.
|
|
1080
|
+
* Called during node startup, before accepting new requests.
|
|
1081
|
+
*/
|
|
1082
|
+
async recoverTransactions(): Promise<void> {
|
|
1083
|
+
if (!this.stateStore) return;
|
|
1084
|
+
const states = await this.stateStore.getAllCoordinatorStates();
|
|
1085
|
+
for (const state of states) {
|
|
1086
|
+
const { messageHash } = state;
|
|
1087
|
+
// Expired — clean up
|
|
1088
|
+
if (state.record.message.expiration && state.record.message.expiration < this.now()) {
|
|
1089
|
+
log('cluster-tx:recovery-expired', { messageHash });
|
|
1090
|
+
await this.stateStore.deleteCoordinatorState(messageHash);
|
|
1091
|
+
continue;
|
|
1092
|
+
}
|
|
1093
|
+
// Broadcasting phase with retry state — resume retries
|
|
1094
|
+
if (state.phase === 'broadcasting' && state.retryState) {
|
|
1095
|
+
log('cluster-tx:recovery-resume-broadcast', { messageHash, attempt: state.retryState.attempt });
|
|
1096
|
+
const pending = new Pending(Promise.resolve(state.record));
|
|
1097
|
+
const txState: ClusterTransactionState = {
|
|
1098
|
+
messageHash,
|
|
1099
|
+
record: state.record,
|
|
1100
|
+
pending,
|
|
1101
|
+
lastUpdate: state.lastUpdate
|
|
1102
|
+
};
|
|
1103
|
+
this.transactions.set(messageHash, txState);
|
|
1104
|
+
// Schedule retry from where we left off
|
|
1105
|
+
this.scheduleCommitRetry(messageHash, state.record, state.retryState.pendingPeers);
|
|
1106
|
+
continue;
|
|
1107
|
+
}
|
|
1108
|
+
// Promising or committing — cannot resume (caller context is gone)
|
|
1109
|
+
log('cluster-tx:recovery-stale', { messageHash, phase: state.phase });
|
|
1110
|
+
await this.stateStore.deleteCoordinatorState(messageHash);
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
}
|