@optimystic/db-p2p 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +7 -0
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +52 -4
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +80 -17
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +23 -15
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +28 -2
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +113 -60
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +21 -3
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/testing/mesh-harness.d.ts +7 -0
- package/dist/src/testing/mesh-harness.d.ts.map +1 -1
- package/dist/src/testing/mesh-harness.js +13 -3
- package/dist/src/testing/mesh-harness.js.map +1 -1
- package/package.json +2 -2
- package/src/cluster/cluster-repo.ts +7 -0
- package/src/libp2p-key-network.ts +958 -857
- package/src/libp2p-node-base.ts +23 -14
- package/src/repo/coordinator-repo.ts +138 -62
- package/src/storage/storage-repo.ts +23 -4
- package/src/testing/mesh-harness.ts +19 -3
|
@@ -1,857 +1,958 @@
|
|
|
1
|
-
import type { AbortOptions, Connection, Libp2p, PeerId, Stream } from "@libp2p/interface";
|
|
2
|
-
import { toString as u8ToString } from 'uint8arrays'
|
|
3
|
-
import type { ClusterPeers, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork } from "@optimystic/db-core";
|
|
4
|
-
import { peerIdFromString } from '@libp2p/peer-id'
|
|
5
|
-
import { multiaddr } from '@multiformats/multiaddr'
|
|
6
|
-
import type { FretService, SerializedTable } from 'p2p-fret'
|
|
7
|
-
import { hashKey } from 'p2p-fret'
|
|
8
|
-
import { createLogger, verbose } from './logger.js'
|
|
9
|
-
import type { IPeerReputation } from './reputation/types.js'
|
|
10
|
-
|
|
11
|
-
interface WithFretService { services?: { fret?: FretService } }
|
|
12
|
-
|
|
13
|
-
export type NetworkMode = 'forming' | 'joining';
|
|
14
|
-
|
|
15
|
-
/**
|
|
16
|
-
* Error codes surfaced by {@link Libp2pKeyPeerNetwork.findCoordinator}. Callers
|
|
17
|
-
* (notably the batch-retry logic in `NetworkTransactor`) can inspect `.code`
|
|
18
|
-
* to distinguish between "transient — try again with different excludes" and
|
|
19
|
-
* "terminal — stop retrying".
|
|
20
|
-
*/
|
|
21
|
-
export const FIND_COORDINATOR_ERROR_CODES = {
|
|
22
|
-
/**
|
|
23
|
-
* Last-resort self-coordination was blocked by
|
|
24
|
-
*
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
* -
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
private
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
this.
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
if (
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
const
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
//
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
//
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
const
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
const
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
//
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
//
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
const
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
*
|
|
831
|
-
*
|
|
832
|
-
*
|
|
833
|
-
*
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
}
|
|
1
|
+
import type { AbortOptions, Connection, Libp2p, PeerId, Stream } from "@libp2p/interface";
|
|
2
|
+
import { toString as u8ToString } from 'uint8arrays'
|
|
3
|
+
import type { ClusterPeers, CoordinatorIntent, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork } from "@optimystic/db-core";
|
|
4
|
+
import { peerIdFromString } from '@libp2p/peer-id'
|
|
5
|
+
import { multiaddr } from '@multiformats/multiaddr'
|
|
6
|
+
import type { FretService, SerializedTable } from 'p2p-fret'
|
|
7
|
+
import { hashKey } from 'p2p-fret'
|
|
8
|
+
import { createLogger, verbose } from './logger.js'
|
|
9
|
+
import type { IPeerReputation } from './reputation/types.js'
|
|
10
|
+
|
|
11
|
+
interface WithFretService { services?: { fret?: FretService } }
|
|
12
|
+
|
|
13
|
+
export type NetworkMode = 'forming' | 'joining';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Error codes surfaced by {@link Libp2pKeyPeerNetwork.findCoordinator}. Callers
|
|
17
|
+
* (notably the batch-retry logic in `NetworkTransactor`) can inspect `.code`
|
|
18
|
+
* to distinguish between "transient — try again with different excludes" and
|
|
19
|
+
* "terminal — stop retrying".
|
|
20
|
+
*/
|
|
21
|
+
export const FIND_COORDINATOR_ERROR_CODES = {
|
|
22
|
+
/**
|
|
23
|
+
* Last-resort self-coordination was blocked by a HARD verdict from the
|
|
24
|
+
* self-coordination guard — self-coordination switched off by config, or a detected
|
|
25
|
+
* partition / suspicious shrinkage on a WRITE. Retrying is unlikely to help. A
|
|
26
|
+
* *deferrable* denial (see {@link SelfCoordinationDecision.deferrable}) never produces
|
|
27
|
+
* this code: selection degrades to self with a warning instead.
|
|
28
|
+
*/
|
|
29
|
+
SELF_COORDINATION_BLOCKED: 'SELF_COORDINATION_BLOCKED',
|
|
30
|
+
/**
|
|
31
|
+
* Self-coordination was already attempted and self is now excluded. On a solo
|
|
32
|
+
* or bootstrap node with no other peers, this means retries are exhausted and
|
|
33
|
+
* the original error from the prior attempt should be surfaced instead.
|
|
34
|
+
*/
|
|
35
|
+
SELF_COORDINATION_EXHAUSTED: 'SELF_COORDINATION_EXHAUSTED',
|
|
36
|
+
/** No peer (including self) is an eligible coordinator. */
|
|
37
|
+
NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE',
|
|
38
|
+
/**
|
|
39
|
+
* The candidate set was non-empty but every non-self candidate serves a
|
|
40
|
+
* DIFFERENT network's protocol (or none of this network's). Distinct from
|
|
41
|
+
* NO_COORDINATOR_AVAILABLE so a Sereus-style trace points at the real cause —
|
|
42
|
+
* "peer(s) do not serve this network's protocol" — instead of a generic
|
|
43
|
+
* "all candidates excluded" / super-majority failure.
|
|
44
|
+
*/
|
|
45
|
+
NO_NETWORK_COORDINATOR: 'NO_NETWORK_COORDINATOR'
|
|
46
|
+
} as const;
|
|
47
|
+
|
|
48
|
+
export type FindCoordinatorErrorCode =
|
|
49
|
+
typeof FIND_COORDINATOR_ERROR_CODES[keyof typeof FIND_COORDINATOR_ERROR_CODES];
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Network-membership classification of a peer relative to THIS node's network,
|
|
53
|
+
* derived from the peer's libp2p peerStore protocol list:
|
|
54
|
+
* - `serves` — advertises this network's namespaced `cluster`/`repo` protocol.
|
|
55
|
+
* - `foreign` — has a non-empty protocol list but none for this network → another network.
|
|
56
|
+
* - `unknown` — protocol list empty / peer absent → identify not yet completed. This is
|
|
57
|
+
* both a fresh same-network peer (will flip to `serves`) AND a cross-network
|
|
58
|
+
* peer (whose network-namespaced identify can NEVER complete, so it stays
|
|
59
|
+
* `unknown` forever) — indistinguishable at a single instant, separated over
|
|
60
|
+
* the retry/stabilization window.
|
|
61
|
+
*/
|
|
62
|
+
export type NetworkMembership = 'serves' | 'foreign' | 'unknown';
|
|
63
|
+
|
|
64
|
+
export class FindCoordinatorError extends Error {
|
|
65
|
+
readonly code: FindCoordinatorErrorCode;
|
|
66
|
+
constructor(code: FindCoordinatorErrorCode, message: string) {
|
|
67
|
+
super(message);
|
|
68
|
+
this.name = 'FindCoordinatorError';
|
|
69
|
+
this.code = code;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface PersistedNetworkState {
|
|
74
|
+
version: 1;
|
|
75
|
+
networkHighWaterMark: number;
|
|
76
|
+
lastConnectedTimestamp: number;
|
|
77
|
+
consecutiveIsolatedSessions: number;
|
|
78
|
+
fretTable?: SerializedTable;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export interface NetworkStatePersistence {
|
|
82
|
+
load(): Promise<PersistedNetworkState | undefined>;
|
|
83
|
+
save(state: PersistedNetworkState): Promise<void>;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Configuration options for self-coordination behavior
|
|
88
|
+
*/
|
|
89
|
+
export interface SelfCoordinationConfig {
|
|
90
|
+
/** Time (ms) after last connection before allowing self-coordination. Default: 30000 */
|
|
91
|
+
gracePeriodMs?: number;
|
|
92
|
+
/** Threshold for suspicious network shrinkage (0-1). >50% drop is suspicious. Default: 0.5 */
|
|
93
|
+
shrinkageThreshold?: number;
|
|
94
|
+
/** Allow self-coordination at all. Default: true (for testing). Set false in production. */
|
|
95
|
+
allowSelfCoordination?: boolean;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Decision result from self-coordination guard
|
|
100
|
+
*/
|
|
101
|
+
export interface SelfCoordinationDecision {
|
|
102
|
+
allow: boolean;
|
|
103
|
+
reason: 'bootstrap-node' | 'partition-detected' | 'suspicious-shrinkage' | 'grace-period-not-elapsed' | 'extended-isolation' | 'hwm-decay' | 'disabled';
|
|
104
|
+
warn?: boolean;
|
|
105
|
+
/**
|
|
106
|
+
* Set on a denial. `true` means "self is not the PREFERRED coordinator right now, but
|
|
107
|
+
* nothing says it is unsafe" — the last-resort tier degrades to self with a warning
|
|
108
|
+
* rather than failing the caller. `false` means there is a positive reason to refuse
|
|
109
|
+
* (operator config, or evidence of a partition) and the caller is failed.
|
|
110
|
+
*
|
|
111
|
+
* Hardness by reason, given the caller's {@link CoordinatorIntent}:
|
|
112
|
+
*
|
|
113
|
+
* | reason | write | read |
|
|
114
|
+
* | ------------------------- | ---------- | ---------- |
|
|
115
|
+
* | `disabled` | hard | hard |
|
|
116
|
+
* | `grace-period-not-elapsed`| deferrable | deferrable |
|
|
117
|
+
* | `partition-detected` | hard | deferrable |
|
|
118
|
+
* | `suspicious-shrinkage` | hard | deferrable |
|
|
119
|
+
*
|
|
120
|
+
* `grace-period-not-elapsed` is deferrable for BOTH because it is a timing condition
|
|
121
|
+
* with no evidence behind it: the same node, with the same FRET table and the same zero
|
|
122
|
+
* connections, is allowed to self-coordinate once the clock passes `gracePeriodMs`. It
|
|
123
|
+
* postpones an isolated write rather than preventing it (a self-only cohort commits
|
|
124
|
+
* under `allowClusterDownsize`, the default), so failing the caller buys no safety.
|
|
125
|
+
*
|
|
126
|
+
* The read column is uniformly deferrable because none of these reasons protects a
|
|
127
|
+
* read: self-coordinating a read means "answer from my own replica", which is what an
|
|
128
|
+
* isolated node must accept anyway, and the layers below already report the quality of
|
|
129
|
+
* that answer (`CoordinatorRepo.fetchBlockFromCluster` short-circuits a self-only cohort
|
|
130
|
+
* as conclusive; an unreachable cohort comes back flagged `unavailable`). `disabled` is
|
|
131
|
+
* the exception for both intents — it is an explicit operator switch, not an inference.
|
|
132
|
+
*
|
|
133
|
+
* NOTE: optional, so a NEW denial branch that forgets to set it silently reads as HARD
|
|
134
|
+
* (`findCoordinator` tests `deferrable !== true`) — safe for a write, but it reinstates
|
|
135
|
+
* the original defect for a read: an outright lookup failure where degrading to our own
|
|
136
|
+
* replica would do. Every denial branch today sets it explicitly. If a fifth reason is
|
|
137
|
+
* ever added, either set it there too or split this into a discriminated union
|
|
138
|
+
* (`{ allow: true, … } | { allow: false, deferrable: boolean, … }`) so omission is a
|
|
139
|
+
* compile error.
|
|
140
|
+
*/
|
|
141
|
+
deferrable?: boolean;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
|
|
145
|
+
private readonly selfCoordinationConfig: Required<SelfCoordinationConfig>;
|
|
146
|
+
private networkHighWaterMark = 1;
|
|
147
|
+
private lastConnectedTime = Date.now();
|
|
148
|
+
private consecutiveIsolatedSessions = 0;
|
|
149
|
+
private readonly networkMode: NetworkMode;
|
|
150
|
+
private readonly persistence?: NetworkStatePersistence;
|
|
151
|
+
|
|
152
|
+
constructor(
|
|
153
|
+
private readonly libp2p: Libp2p,
|
|
154
|
+
private readonly clusterSize: number = 16,
|
|
155
|
+
selfCoordinationConfig?: SelfCoordinationConfig,
|
|
156
|
+
networkMode?: NetworkMode,
|
|
157
|
+
persistence?: NetworkStatePersistence,
|
|
158
|
+
private readonly reputation?: IPeerReputation,
|
|
159
|
+
/**
|
|
160
|
+
* Network-namespaced protocol prefix (`/optimystic/<networkName>`). When
|
|
161
|
+
* provided, coordinator/cohort selection is scoped to peers that serve THIS
|
|
162
|
+
* network's `cluster`/`repo` protocol, so a peer that only belongs to another
|
|
163
|
+
* network sharing the same physical nodes/bootstraps is never chosen. When
|
|
164
|
+
* ABSENT, the membership filter is disabled (today's exact behavior) — required
|
|
165
|
+
* for backward compatibility because most call sites don't know the network name.
|
|
166
|
+
*/
|
|
167
|
+
private readonly protocolPrefix?: string
|
|
168
|
+
) {
|
|
169
|
+
// NOTE: no construction site in this repo passes a SelfCoordinationConfig — every one
|
|
170
|
+
// leaves it `undefined` (libp2p-node-base.ts, quereus-plugin-optimystic's
|
|
171
|
+
// collection-factory.ts and key-network.ts, reference-peer's cli.ts), so these
|
|
172
|
+
// defaults are always what is in force and no operator can tune them. If tuning
|
|
173
|
+
// `gracePeriodMs` is ever needed, those four sites have to thread the config through
|
|
174
|
+
// first. Low urgency: a grace-period denial no longer fails the caller, it only costs
|
|
175
|
+
// a write the ~1s findCoordinator retry window before self-coordinating.
|
|
176
|
+
this.selfCoordinationConfig = {
|
|
177
|
+
gracePeriodMs: selfCoordinationConfig?.gracePeriodMs ?? 30_000,
|
|
178
|
+
shrinkageThreshold: selfCoordinationConfig?.shrinkageThreshold ?? 0.5,
|
|
179
|
+
allowSelfCoordination: selfCoordinationConfig?.allowSelfCoordination ?? true
|
|
180
|
+
};
|
|
181
|
+
this.networkMode = networkMode ?? 'forming';
|
|
182
|
+
this.persistence = persistence;
|
|
183
|
+
this.setupConnectionTracking();
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// coordinator cache: key (base64url) -> peerId until expiry (bounded LRU-ish via Map insertion order)
|
|
187
|
+
private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
|
|
188
|
+
private static readonly MAX_CACHE_ENTRIES = 1000
|
|
189
|
+
private readonly log = createLogger('libp2p-key-network')
|
|
190
|
+
|
|
191
|
+
private toCacheKey(key: Uint8Array): string { return u8ToString(key, 'base64url') }
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Set up connection event tracking to update high water mark and last connected time.
|
|
195
|
+
*/
|
|
196
|
+
private setupConnectionTracking(): void {
|
|
197
|
+
this.libp2p.addEventListener('connection:open', () => {
|
|
198
|
+
this.updateNetworkObservations();
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Update network high water mark and last connected time.
|
|
204
|
+
* Called on new connections.
|
|
205
|
+
*/
|
|
206
|
+
private updateNetworkObservations(): void {
|
|
207
|
+
const connections = this.libp2p.getConnections?.() ?? [];
|
|
208
|
+
if (connections.length > 0) {
|
|
209
|
+
this.lastConnectedTime = Date.now();
|
|
210
|
+
this.consecutiveIsolatedSessions = 0;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
try {
|
|
214
|
+
const fret = this.getFret();
|
|
215
|
+
const estimate = fret.getNetworkSizeEstimate();
|
|
216
|
+
if (estimate.size_estimate > this.networkHighWaterMark) {
|
|
217
|
+
this.networkHighWaterMark = estimate.size_estimate;
|
|
218
|
+
this.log('network-hwm-updated mark=%d confidence=%f', this.networkHighWaterMark, estimate.confidence);
|
|
219
|
+
}
|
|
220
|
+
} catch {
|
|
221
|
+
// FRET not available - use connection count as fallback
|
|
222
|
+
const connectionCount = this.libp2p.getConnections?.().length ?? 0;
|
|
223
|
+
const observedSize = connectionCount + 1; // +1 for self
|
|
224
|
+
if (observedSize > this.networkHighWaterMark) {
|
|
225
|
+
this.networkHighWaterMark = observedSize;
|
|
226
|
+
this.log('network-hwm-updated mark=%d (from connections)', this.networkHighWaterMark);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
this.persistState();
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
async initFromPersistedState(): Promise<void> {
|
|
234
|
+
if (!this.persistence) return;
|
|
235
|
+
const state = await this.persistence.load();
|
|
236
|
+
if (!state) return;
|
|
237
|
+
|
|
238
|
+
this.networkHighWaterMark = state.networkHighWaterMark;
|
|
239
|
+
this.lastConnectedTime = state.lastConnectedTimestamp;
|
|
240
|
+
this.consecutiveIsolatedSessions = state.consecutiveIsolatedSessions;
|
|
241
|
+
|
|
242
|
+
if (state.fretTable) {
|
|
243
|
+
try {
|
|
244
|
+
this.getFret().importTable(state.fretTable);
|
|
245
|
+
} catch (err) { this.log('init:fret-import-skipped %o', err); }
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// If HWM > 1 but FRET table is empty/self-only, increment isolated sessions
|
|
249
|
+
if (state.networkHighWaterMark > 1) {
|
|
250
|
+
const fretEntryCount = state.fretTable?.entries?.length ?? 0;
|
|
251
|
+
if (fretEntryCount <= 1) {
|
|
252
|
+
this.consecutiveIsolatedSessions++;
|
|
253
|
+
this.log('init:isolated-session count=%d hwm=%d', this.consecutiveIsolatedSessions, this.networkHighWaterMark);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
private canRetryImprove(fretNeighborIds: string[]): boolean {
|
|
259
|
+
if (this.networkMode !== 'forming') return true;
|
|
260
|
+
if (this.networkHighWaterMark > 1) return true;
|
|
261
|
+
const onlySelf = fretNeighborIds.length <= 1
|
|
262
|
+
&& (fretNeighborIds.length === 0 || fretNeighborIds[0] === this.libp2p.peerId.toString());
|
|
263
|
+
return !onlySelf;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
private persistState(): void {
|
|
267
|
+
if (!this.persistence) return;
|
|
268
|
+
const state: PersistedNetworkState = {
|
|
269
|
+
version: 1,
|
|
270
|
+
networkHighWaterMark: this.networkHighWaterMark,
|
|
271
|
+
lastConnectedTimestamp: this.lastConnectedTime,
|
|
272
|
+
consecutiveIsolatedSessions: this.consecutiveIsolatedSessions,
|
|
273
|
+
};
|
|
274
|
+
try {
|
|
275
|
+
const fret = this.getFret();
|
|
276
|
+
state.fretTable = fret.exportTable();
|
|
277
|
+
} catch { /* FRET not available */ }
|
|
278
|
+
void this.persistence.save(state).catch(err => this.log('persist-state-failed %o', err));
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Determine if self-coordination should be allowed based on network observations.
|
|
283
|
+
*
|
|
284
|
+
* Principle: If we've ever seen a larger network, assume our connectivity is the problem,
|
|
285
|
+
* not the network shrinking.
|
|
286
|
+
*
|
|
287
|
+
* A denial is classified as HARD or DEFERRABLE via {@link SelfCoordinationDecision.deferrable}
|
|
288
|
+
* — see that field for the reason/intent table. A hard denial fails the caller; a deferrable
|
|
289
|
+
* one only means "self is not the preferred coordinator", and the last-resort tier degrades
|
|
290
|
+
* to self with a warning.
|
|
291
|
+
*
|
|
292
|
+
* @param intent What the caller means to do with the coordinator. Defaults to `'write'`,
|
|
293
|
+
* the conservative reading, so callers that don't know are held to the stricter bar.
|
|
294
|
+
*/
|
|
295
|
+
shouldAllowSelfCoordination(intent: CoordinatorIntent = 'write'): SelfCoordinationDecision {
|
|
296
|
+
// A read never coordinates a mutation, so every evidence-based denial below is merely
|
|
297
|
+
// a preference for a better-placed peer — the caller can always be answered from this
|
|
298
|
+
// node's own replica. Only the explicit `disabled` switch is absolute for a read.
|
|
299
|
+
const deferrableOnEvidence = intent === 'read';
|
|
300
|
+
|
|
301
|
+
// Check global disable
|
|
302
|
+
if (!this.selfCoordinationConfig.allowSelfCoordination) {
|
|
303
|
+
return { allow: false, reason: 'disabled', deferrable: false };
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// Case 1: New/bootstrap node (never seen larger network)
|
|
307
|
+
if (this.networkHighWaterMark <= 1) {
|
|
308
|
+
return { allow: true, reason: 'bootstrap-node' };
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// Case 1b: Repeated isolation across sessions — decay HWM to allow eventual self-coordination
|
|
312
|
+
if (this.consecutiveIsolatedSessions >= 3) {
|
|
313
|
+
this.log('self-coord-allowed: hwm-decayed sessions=%d', this.consecutiveIsolatedSessions);
|
|
314
|
+
return { allow: true, reason: 'hwm-decay', warn: true };
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// Case 2: Check for partition via FRET
|
|
318
|
+
try {
|
|
319
|
+
const fret = this.getFret();
|
|
320
|
+
if (fret.detectPartition()) {
|
|
321
|
+
this.log('self-coord-blocked: partition-detected intent=%s', intent);
|
|
322
|
+
return { allow: false, reason: 'partition-detected', deferrable: deferrableOnEvidence };
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
// Case 3: Suspicious network shrinkage (>threshold drop)
|
|
326
|
+
const estimate = fret.getNetworkSizeEstimate();
|
|
327
|
+
const shrinkage = 1 - (estimate.size_estimate / this.networkHighWaterMark);
|
|
328
|
+
if (shrinkage > this.selfCoordinationConfig.shrinkageThreshold) {
|
|
329
|
+
this.log('self-coord-blocked: suspicious-shrinkage current=%d hwm=%d shrinkage=%f intent=%s',
|
|
330
|
+
estimate.size_estimate, this.networkHighWaterMark, shrinkage, intent);
|
|
331
|
+
return { allow: false, reason: 'suspicious-shrinkage', deferrable: deferrableOnEvidence };
|
|
332
|
+
}
|
|
333
|
+
} catch {
|
|
334
|
+
// FRET not available - be conservative
|
|
335
|
+
const connections = this.libp2p.getConnections?.() ?? [];
|
|
336
|
+
if (this.networkHighWaterMark > 1 && connections.length === 0) {
|
|
337
|
+
// We've seen peers before but have none now - suspicious
|
|
338
|
+
const timeSinceConnection = Date.now() - this.lastConnectedTime;
|
|
339
|
+
if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
|
|
340
|
+
this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
|
|
341
|
+
return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// Case 4: Recently connected (grace period not elapsed)
|
|
347
|
+
const timeSinceConnection = Date.now() - this.lastConnectedTime;
|
|
348
|
+
if (timeSinceConnection < this.selfCoordinationConfig.gracePeriodMs) {
|
|
349
|
+
const connections = this.libp2p.getConnections?.() ?? [];
|
|
350
|
+
// Only block if we have no connections but did recently
|
|
351
|
+
if (connections.length === 0) {
|
|
352
|
+
this.log('self-coord-blocked: grace-period-not-elapsed since=%dms', timeSinceConnection);
|
|
353
|
+
// Deferrable for BOTH intents: nothing here is evidence, only a clock. The same
|
|
354
|
+
// node with the same information self-coordinates once gracePeriodMs elapses.
|
|
355
|
+
return { allow: false, reason: 'grace-period-not-elapsed', deferrable: true };
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
// Case 5: Extended isolation with gradual shrinkage - allow with warning
|
|
360
|
+
this.log('self-coord-allowed: extended-isolation (warn)');
|
|
361
|
+
return { allow: true, reason: 'extended-isolation', warn: true };
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/**
|
|
365
|
+
* Memoize the coordinator for a key. A pick of SELF is deliberately ignored — the
|
|
366
|
+
* cache is consulted ahead of every selection tier, so a self entry would keep the
|
|
367
|
+
* key routed at our own (possibly stale) replica for the full TTL long after a
|
|
368
|
+
* better-placed peer became reachable, and would return self without re-consulting
|
|
369
|
+
* {@link shouldAllowSelfCoordination}, letting a partitioned node silently serve its
|
|
370
|
+
* own data. Self needs no memoizing anyway: every tier that can select it re-derives
|
|
371
|
+
* it from a local lookup with no dial and no retry sleep.
|
|
372
|
+
*
|
|
373
|
+
* The gate lives here rather than at each call site because most writers are OUTSIDE
|
|
374
|
+
* this class — `recordCoordinator` is public and is fed self-valued picks by
|
|
375
|
+
* `NetworkTransactor` (it writes back whatever `findCoordinator` returned, including
|
|
376
|
+
* self) and by `RepoClient`/`ClusterClient` on redirect responses.
|
|
377
|
+
*/
|
|
378
|
+
public recordCoordinator(key: Uint8Array, peerId: PeerId, ttlMs = 30 * 60 * 1000): void {
|
|
379
|
+
if (peerId.toString() === this.libp2p.peerId.toString()) {
|
|
380
|
+
this.log('coordinator-cache:self-write-ignored key=%s', this.toCacheKey(key).substring(0, 12))
|
|
381
|
+
return
|
|
382
|
+
}
|
|
383
|
+
const k = this.toCacheKey(key)
|
|
384
|
+
const now = Date.now()
|
|
385
|
+
for (const [ck, entry] of this.coordinatorCache) {
|
|
386
|
+
if (entry.expires <= now) this.coordinatorCache.delete(ck)
|
|
387
|
+
}
|
|
388
|
+
this.coordinatorCache.set(k, { id: peerId, expires: now + ttlMs })
|
|
389
|
+
while (this.coordinatorCache.size > Libp2pKeyPeerNetwork.MAX_CACHE_ENTRIES) {
|
|
390
|
+
const firstKey = this.coordinatorCache.keys().next().value as string | undefined
|
|
391
|
+
if (firstKey == null) break
|
|
392
|
+
this.coordinatorCache.delete(firstKey)
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
private getCachedCoordinator(key: Uint8Array): PeerId | undefined {
|
|
397
|
+
const k = this.toCacheKey(key)
|
|
398
|
+
const hit = this.coordinatorCache.get(k)
|
|
399
|
+
if (hit && hit.expires > Date.now()) return hit.id
|
|
400
|
+
if (hit) this.coordinatorCache.delete(k)
|
|
401
|
+
return undefined
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/**
|
|
405
|
+
* True for a circuit-relay ("limited") connection. libp2p stamps a relayed
|
|
406
|
+
* connection with `limits` (per-circuit data/duration caps); we additionally
|
|
407
|
+
* sniff the multiaddr for `/p2p-circuit` as a fallback for transports/versions
|
|
408
|
+
* that don't populate `limits`.
|
|
409
|
+
*/
|
|
410
|
+
private isLimitedConnection(c: Connection): boolean {
|
|
411
|
+
if ((c as { limits?: unknown }).limits != null) return true
|
|
412
|
+
const addr = c.remoteAddr?.toString?.()
|
|
413
|
+
return addr != null && addr.includes('/p2p-circuit')
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
|
|
417
|
+
const conns = this.libp2p.getConnections?.(peerId) ?? []
|
|
418
|
+
// Filter to only-open connections so a closing/closed entry that libp2p
|
|
419
|
+
// hasn't yet evicted from its index doesn't get picked up here.
|
|
420
|
+
const open = conns.filter(c => c?.status === 'open' && typeof c?.newStream === 'function')
|
|
421
|
+
// Prefer a DIRECT connection over a limited (circuit-relay) one for the RPC.
|
|
422
|
+
// A relayed/limited connection can be reset by the relay once a per-circuit
|
|
423
|
+
// cap or reservation lapses (@libp2p/circuit-relay-v2), surfacing to the
|
|
424
|
+
// coordinator as a StreamResetError that fails consensus. After DCUtR upgrades
|
|
425
|
+
// a relayed link to direct, both connections briefly coexist — picking the
|
|
426
|
+
// direct one avoids riding the soon-to-be-reset circuit. We only fall back to
|
|
427
|
+
// the limited connection (with runOnLimitedConnection) when it is the only open
|
|
428
|
+
// path — the steady state for browsers and NATed peers before any upgrade.
|
|
429
|
+
const chosen = open.find(c => !this.isLimitedConnection(c)) ?? open[0]
|
|
430
|
+
if (chosen) {
|
|
431
|
+
// runOnLimitedConnection: true is required to open a stream over a
|
|
432
|
+
// circuit-relay (limited) connection — the steady-state path for
|
|
433
|
+
// browsers and NATed peers. Without it, the warm relay connection
|
|
434
|
+
// from a prior dialProtocol cannot be reused on subsequent RPCs. It is
|
|
435
|
+
// a harmless no-op on the preferred direct connection.
|
|
436
|
+
return chosen.newStream([protocol], {
|
|
437
|
+
signal: options?.signal,
|
|
438
|
+
runOnLimitedConnection: true,
|
|
439
|
+
negotiateFully: false
|
|
440
|
+
})
|
|
441
|
+
}
|
|
442
|
+
// Forward the caller's AbortSignal so a per-peer dial deadline (enforced
|
|
443
|
+
// upstream by ProtocolClient.processMessage) can actually cancel a stuck
|
|
444
|
+
// dial — without this, libp2p falls back to its built-in dial timeout
|
|
445
|
+
// (default ~30s) and the caller's tighter deadline is decorative.
|
|
446
|
+
const dialOptions = { runOnLimitedConnection: true, negotiateFully: false, signal: options?.signal } as const
|
|
447
|
+
return this.libp2p.dialProtocol(peerId, [protocol], dialOptions)
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
private getFret(): FretService {
|
|
451
|
+
const svc = (this.libp2p as unknown as WithFretService).services?.fret
|
|
452
|
+
if (svc == null) throw new Error('FRET service is not registered on this libp2p node')
|
|
453
|
+
return svc
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
private async getNeighborIdsForKey(key: Uint8Array, wants: number): Promise<string[]> {
|
|
457
|
+
const fret = this.getFret()
|
|
458
|
+
const coord = await hashKey(key)
|
|
459
|
+
const both = fret.getNeighbors(coord, 'both', wants)
|
|
460
|
+
return Array.from(new Set(both)).slice(0, wants)
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
async findCoordinator(key: Uint8Array, _options?: Partial<FindCoordinatorOptions>): Promise<PeerId> {
|
|
464
|
+
const t0 = Date.now();
|
|
465
|
+
const excludedSet = new Set<string>((_options?.excludedPeers ?? []).map(p => p.toString()))
|
|
466
|
+
// Unset means 'write' — the conservative reading, so a caller that doesn't declare an
|
|
467
|
+
// intent is held to the stricter self-coordination bar.
|
|
468
|
+
const intent: CoordinatorIntent = _options?.intent ?? 'write';
|
|
469
|
+
const keyStr = this.toCacheKey(key).substring(0, 12);
|
|
470
|
+
// Tracks whether the network-membership filter excluded an UNCONFIRMED candidate
|
|
471
|
+
// — `foreign` (another network) OR `unknown` (not yet confirmed to serve this
|
|
472
|
+
// network) — during any attempt. If selection ultimately fails with self
|
|
473
|
+
// unavailable, this lets us surface NO_NETWORK_COORDINATOR (the real cause)
|
|
474
|
+
// instead of the generic NO_COORDINATOR_AVAILABLE.
|
|
475
|
+
let droppedUnconfirmedAnyAttempt = false;
|
|
476
|
+
|
|
477
|
+
this.log('findCoordinator:start key=%s excluded=%o', keyStr, Array.from(excludedSet).map(s => s.substring(0, 12)))
|
|
478
|
+
|
|
479
|
+
// honor cache if not excluded
|
|
480
|
+
const cached = this.getCachedCoordinator(key)
|
|
481
|
+
if (cached != null && !excludedSet.has(cached.toString())) {
|
|
482
|
+
this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'cache')
|
|
483
|
+
return cached
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
// Retry logic: connections can be temporarily down, so retry a few times with delay
|
|
487
|
+
const maxRetries = 3;
|
|
488
|
+
const retryDelayMs = 500;
|
|
489
|
+
|
|
490
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
491
|
+
// Get currently connected peers for filtering
|
|
492
|
+
const connected = (this.libp2p.getConnections?.() ?? []).map((c: any) => c.remotePeer) as PeerId[]
|
|
493
|
+
const connectedSet = new Set(connected.map(p => p.toString()))
|
|
494
|
+
this.log('findCoordinator:connected-peers key=%s count=%d peers=%o attempt=%d', keyStr, connected.length, connected.map(p => p.toString().substring(0, 12)), attempt)
|
|
495
|
+
|
|
496
|
+
// prefer FRET neighbors that are also connected, pick first non-excluded
|
|
497
|
+
let ids: string[] = [];
|
|
498
|
+
try {
|
|
499
|
+
ids = await this.getNeighborIdsForKey(key, this.clusterSize)
|
|
500
|
+
this.log('findCoordinator:fret-neighbors key=%s candidates=%d', keyStr, ids.length)
|
|
501
|
+
if (verbose) this.log('findCoordinator:fret-candidates key=%s ids=%o connected=%o', keyStr, ids, Array.from(connectedSet))
|
|
502
|
+
|
|
503
|
+
// Filter to only connected FRET neighbors, excluding banned peers. Self is
|
|
504
|
+
// never "connected" to itself, so it is admitted by the explicit self clause
|
|
505
|
+
// below — but ONLY when the self-coordination guard allows it, otherwise a
|
|
506
|
+
// node whose FRET neighborhood contains self (essentially always on a small or
|
|
507
|
+
// forming network) would bypass the guard and the last-resort tier's
|
|
508
|
+
// SELF_COORDINATION_BLOCKED would never fire. On refusal self is merely DROPPED
|
|
509
|
+
// from the candidate list, so the connected-peer fallback below still gets its
|
|
510
|
+
// chance at a good remote peer; only if that also comes up empty does the
|
|
511
|
+
// last-resort tier raise the accurate error.
|
|
512
|
+
//
|
|
513
|
+
// An ISOLATED READ is the exception: with no connection left there is no better
|
|
514
|
+
// answer to wait for, and a deferrable denial is not evidence that answering
|
|
515
|
+
// from our own replica is wrong — so self is admitted here and the read resolves
|
|
516
|
+
// immediately instead of paying the ~1s retry loop before the last-resort tier
|
|
517
|
+
// degrades to the same answer. A WRITE keeps dropping self exactly as before,
|
|
518
|
+
// so a peer that lands during the retry window still wins the key.
|
|
519
|
+
const selfStr = this.libp2p.peerId.toString()
|
|
520
|
+
let selfAllowedThisAttempt: boolean | undefined
|
|
521
|
+
// Memoized per ATTEMPT, and evaluated lazily so an all-remote neighborhood never
|
|
522
|
+
// pays detectPartition() / getNetworkSizeEstimate(). Re-evaluated on each attempt
|
|
523
|
+
// because a connection can land during the 500ms inter-attempt sleep and
|
|
524
|
+
// legitimately flip the answer — as filterByMembership re-reads the peerStore.
|
|
525
|
+
// NOTE: on a small network self is a neighbor of nearly every key, so this runs
|
|
526
|
+
// per findCoordinator call and self-coordinated keys are never cached to absorb
|
|
527
|
+
// it. Fine while detectPartition()/getNetworkSizeEstimate() stay local FRET
|
|
528
|
+
// table reads; if either ever grows a probe or other network round-trip, cache
|
|
529
|
+
// the decision with a short TTL on the instance instead of per attempt.
|
|
530
|
+
// NOTE: the guard re-reads getConnections() live, while `connectedSet` above was
|
|
531
|
+
// snapshotted at the top of this attempt. A connection landing between the two
|
|
532
|
+
// lifts the guard's grace-period denial while the new peer is still absent from
|
|
533
|
+
// the candidate filter — so self can win an attempt on evidence that attempt
|
|
534
|
+
// cannot yet use. Bounded to one attempt (the next re-snapshots and prefers the
|
|
535
|
+
// peer) and self picks are never cached, so it costs at most one lookup's
|
|
536
|
+
// routing. If that ever matters, pass the snapshot into the guard instead.
|
|
537
|
+
const isSelfAdmissible = (): boolean => {
|
|
538
|
+
if (selfAllowedThisAttempt === undefined) {
|
|
539
|
+
const decision = this.shouldAllowSelfCoordination(intent)
|
|
540
|
+
// Gated on ISOLATION, not just on the read intent. Self carries no reputation
|
|
541
|
+
// record, so it scores 0 and sorts ahead of every remote candidate in the rank
|
|
542
|
+
// below — admitting it while a connection is live would hand the key to a node
|
|
543
|
+
// its own guard just called partitioned, over a reachable FRET neighbour. And
|
|
544
|
+
// waiting costs a connected read nothing: the inter-attempt sleep further down
|
|
545
|
+
// only runs when `connected.length === 0`, so with peers present the remaining
|
|
546
|
+
// attempts and the last-resort degrade run back-to-back with no delay.
|
|
547
|
+
const degradedRead = !decision.allow && decision.deferrable === true
|
|
548
|
+
&& intent === 'read' && connected.length === 0
|
|
549
|
+
selfAllowedThisAttempt = decision.allow || degradedRead
|
|
550
|
+
if (degradedRead) {
|
|
551
|
+
this.log('findCoordinator:fret-self-degraded key=%s reason=%s intent=read attempt=%d', keyStr, decision.reason, attempt)
|
|
552
|
+
} else if (!decision.allow) {
|
|
553
|
+
this.log('findCoordinator:fret-self-dropped key=%s reason=%s intent=%s attempt=%d', keyStr, decision.reason, intent, attempt)
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
return selfAllowedThisAttempt
|
|
557
|
+
}
|
|
558
|
+
const connectedFretIds = ids
|
|
559
|
+
.filter(id => !excludedSet.has(id) && !(this.reputation?.isBanned(id)))
|
|
560
|
+
.filter(id => connectedSet.has(id) || (id === selfStr && isSelfAdmissible()))
|
|
561
|
+
.sort((a, b) => (this.reputation?.getScore(a) ?? 0) - (this.reputation?.getScore(b) ?? 0))
|
|
562
|
+
this.log('findCoordinator:fret-connected key=%s count=%d peers=%o', keyStr, connectedFretIds.length, connectedFretIds.map(s => s.substring(0, 12)))
|
|
563
|
+
|
|
564
|
+
// Network-membership scoping (no-op when protocolPrefix is unset): only a peer
|
|
565
|
+
// CONFIRMED to serve this network ('serves') is eligible — both `foreign`
|
|
566
|
+
// (another network) and `unknown` (not yet identified) peers are excluded
|
|
567
|
+
// from selection. A cross-network peer is permanently 'unknown' (its
|
|
568
|
+
// namespaced identify never completes), so it is never gambled on; over the
|
|
569
|
+
// 3×500ms retry window a genuine same-network peer flips to 'serves' on a
|
|
570
|
+
// re-read of the peerStore and is selected normally on that attempt. Self
|
|
571
|
+
// always classifies as 'serves' and stays eligible.
|
|
572
|
+
const { ranked, droppedUnconfirmed } = await this.filterByMembership(connectedFretIds)
|
|
573
|
+
if (droppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
|
|
574
|
+
const pick = ranked[0]
|
|
575
|
+
if (pick) {
|
|
576
|
+
const pid = peerIdFromString(pick)
|
|
577
|
+
// A self pick is a no-op here — recordCoordinator ignores self-valued
|
|
578
|
+
// writes (see its doc comment), matching the last-resort self tier below.
|
|
579
|
+
this.recordCoordinator(key, pid)
|
|
580
|
+
this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'fret')
|
|
581
|
+
return pid
|
|
582
|
+
}
|
|
583
|
+
} catch (err) {
|
|
584
|
+
this.log('findCoordinator getNeighborIdsForKey failed - %o', err)
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
// fallback: prefer any existing connected peer that's not excluded or banned,
|
|
588
|
+
// scoped to this network's serving peers (a `foreign` or not-yet-confirmed
|
|
589
|
+
// `unknown` peer is never picked). Note this candidate set is built from
|
|
590
|
+
// connected REMOTE peers and never includes self, so when no serving peer is
|
|
591
|
+
// present selection falls through to the last-resort self-coordination block.
|
|
592
|
+
// Being remote-only, this tier needs no self-coordination guard check, unlike the
|
|
593
|
+
// FRET tier above.
|
|
594
|
+
const connectedCandidates = connected
|
|
595
|
+
.filter(p => !excludedSet.has(p.toString()) && !(this.reputation?.isBanned(p.toString())))
|
|
596
|
+
.sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))
|
|
597
|
+
.map(p => p.toString())
|
|
598
|
+
const { ranked: connRanked, droppedUnconfirmed: connDroppedUnconfirmed } = await this.filterByMembership(connectedCandidates)
|
|
599
|
+
if (connDroppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
|
|
600
|
+
const connectedPick = connRanked[0]
|
|
601
|
+
if (connectedPick) {
|
|
602
|
+
const pid = peerIdFromString(connectedPick)
|
|
603
|
+
this.recordCoordinator(key, pid)
|
|
604
|
+
this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'connected-fallback')
|
|
605
|
+
return pid
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
// If no connections and not the last attempt, wait and retry
|
|
609
|
+
if (connected.length === 0 && attempt < maxRetries - 1) {
|
|
610
|
+
if (!this.canRetryImprove(ids)) {
|
|
611
|
+
this.log('findCoordinator:retry-futile key=%s mode=%s hwm=%d',
|
|
612
|
+
keyStr, this.networkMode, this.networkHighWaterMark);
|
|
613
|
+
break;
|
|
614
|
+
}
|
|
615
|
+
this.log('findCoordinator:no-connections-retry key=%s attempt=%d delay=%dms', keyStr, attempt, retryDelayMs)
|
|
616
|
+
await new Promise(resolve => setTimeout(resolve, retryDelayMs))
|
|
617
|
+
continue
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
// last resort: prefer self only if not excluded and guard allows
|
|
622
|
+
const self = this.libp2p.peerId
|
|
623
|
+
if (!excludedSet.has(self.toString())) {
|
|
624
|
+
const decision = this.shouldAllowSelfCoordination(intent);
|
|
625
|
+
// Only a HARD denial fails the caller. A deferrable one (see
|
|
626
|
+
// SelfCoordinationDecision.deferrable) means self is merely not the preferred
|
|
627
|
+
// coordinator — by this point every better tier has already come up empty and the
|
|
628
|
+
// retry window has been spent, so refusing here would just convert "serve from my
|
|
629
|
+
// own replica, degraded" into an outright failure of the whole operation.
|
|
630
|
+
if (!decision.allow && decision.deferrable !== true) {
|
|
631
|
+
this.log('findCoordinator:self-coord-blocked key=%s reason=%s intent=%s', keyStr, decision.reason, intent);
|
|
632
|
+
throw new FindCoordinatorError(
|
|
633
|
+
FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_BLOCKED,
|
|
634
|
+
`Self-coordination blocked: ${decision.reason}. No coordinator available for key.`
|
|
635
|
+
);
|
|
636
|
+
}
|
|
637
|
+
if (!decision.allow) {
|
|
638
|
+
this.log('findCoordinator:self-selected-degraded key=%s coordinator=%s reason=%s intent=%s',
|
|
639
|
+
keyStr, self.toString().substring(0, 12), decision.reason, intent);
|
|
640
|
+
this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self-degraded')
|
|
641
|
+
return self
|
|
642
|
+
}
|
|
643
|
+
if (decision.warn) {
|
|
644
|
+
this.log('findCoordinator:self-selected-warn key=%s coordinator=%s reason=%s',
|
|
645
|
+
keyStr, self.toString().substring(0, 12), decision.reason);
|
|
646
|
+
} else {
|
|
647
|
+
this.log('findCoordinator:self-selected key=%s coordinator=%s reason=%s',
|
|
648
|
+
keyStr, self.toString().substring(0, 12), decision.reason);
|
|
649
|
+
}
|
|
650
|
+
this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'self')
|
|
651
|
+
return self
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
// Self is excluded and selection found no eligible peer. If the membership filter is
|
|
655
|
+
// the reason the candidate set emptied (the only other peers are `foreign` — serving
|
|
656
|
+
// a DIFFERENT network — or `unknown` — not yet confirmed to serve this network),
|
|
657
|
+
// surface a distinct, accurate cause instead of the generic codes below.
|
|
658
|
+
if (droppedUnconfirmedAnyAttempt) {
|
|
659
|
+
this.log('findCoordinator:no-network-coordinator key=%s prefix=%s self=%s',
|
|
660
|
+
keyStr, this.protocolPrefix ?? '?', self.toString().substring(0, 12))
|
|
661
|
+
throw new FindCoordinatorError(
|
|
662
|
+
FIND_COORDINATOR_ERROR_CODES.NO_NETWORK_COORDINATOR,
|
|
663
|
+
`No coordinator available for key on network ${this.protocolPrefix ?? '?'}: ` +
|
|
664
|
+
`the remaining candidate peer(s) are foreign or not-yet-confirmed to serve this network's cluster/repo protocol.`
|
|
665
|
+
);
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
// Self is excluded. On a solo/bootstrap node (HWM<=1 and no other connected/FRET peers),
|
|
669
|
+
// this means the caller already tried self and the retry has nowhere to go — surface a
|
|
670
|
+
// distinct error so retry logic stops and the original first-attempt cause is preserved.
|
|
671
|
+
const isSoloBootstrap = this.networkHighWaterMark <= 1;
|
|
672
|
+
if (isSoloBootstrap) {
|
|
673
|
+
this.log('findCoordinator:self-exhausted-solo key=%s self=%s', keyStr, self.toString().substring(0, 12))
|
|
674
|
+
throw new FindCoordinatorError(
|
|
675
|
+
FIND_COORDINATOR_ERROR_CODES.SELF_COORDINATION_EXHAUSTED,
|
|
676
|
+
'Self-coordination exhausted on solo/bootstrap node (self already attempted). ' +
|
|
677
|
+
'The original first-attempt error describes the actual failure cause.'
|
|
678
|
+
);
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
this.log('findCoordinator:all-excluded key=%s self=%s', keyStr, self.toString().substring(0, 12))
|
|
682
|
+
throw new FindCoordinatorError(
|
|
683
|
+
FIND_COORDINATOR_ERROR_CODES.NO_COORDINATOR_AVAILABLE,
|
|
684
|
+
'No coordinator available for key (all candidates excluded)'
|
|
685
|
+
);
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
private getConnectedAddrsByPeer(): Record<string, string[]> {
|
|
689
|
+
const conns = this.libp2p.getConnections()
|
|
690
|
+
const byPeer: Record<string, string[]> = {}
|
|
691
|
+
for (const c of conns) {
|
|
692
|
+
const id = c.remotePeer.toString()
|
|
693
|
+
const addr = c.remoteAddr?.toString?.()
|
|
694
|
+
if (addr) (byPeer[id] ??= []).push(addr)
|
|
695
|
+
}
|
|
696
|
+
return byPeer
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
private parseMultiaddrs(addrs: string[]): string[] {
|
|
700
|
+
const out: string[] = []
|
|
701
|
+
for (const a of addrs) {
|
|
702
|
+
try { multiaddr(a); out.push(a) } catch (err) { this.log('WARN: invalid multiaddr from connection %s %o', a, err) }
|
|
703
|
+
}
|
|
704
|
+
return out
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
async findCluster(key: Uint8Array): Promise<ClusterPeers> {
|
|
708
|
+
const t0 = Date.now();
|
|
709
|
+
const fret = this.getFret()
|
|
710
|
+
const coord = await hashKey(key)
|
|
711
|
+
// When membership scoping is active, over-fetch a wider proximity band so the
|
|
712
|
+
// nearest peers that SERVE this network are in the candidate pool even if cross-
|
|
713
|
+
// network peers sit nearer the key (see membershipOverfetch).
|
|
714
|
+
const wants = this.protocolPrefix != null ? this.membershipOverfetch() : this.clusterSize
|
|
715
|
+
const cohort = fret.assembleCohort(coord, wants)
|
|
716
|
+
const keyStr = this.toCacheKey(key).substring(0, 12);
|
|
717
|
+
this.log('findCluster:start key=%s', keyStr);
|
|
718
|
+
|
|
719
|
+
// Include self in the cohort
|
|
720
|
+
const selfId = this.libp2p.peerId.toString()
|
|
721
|
+
let ids = Array.from(new Set([...cohort, selfId]))
|
|
722
|
+
|
|
723
|
+
// Network-membership scoping (no-op when protocolPrefix is unset): a cohort
|
|
724
|
+
// member that serves a DIFFERENT network's protocol can never negotiate THIS
|
|
725
|
+
// network's cluster/repo dial, so it guarantees a super-majority failure rather
|
|
726
|
+
// than contributing a promise. Drop such 'foreign' members; build the cohort from
|
|
727
|
+
// positively-'serves' members only and NEVER admit a not-yet-identified ('unknown')
|
|
728
|
+
// member. A permanently cross-network peer and a freshly-discovered same-network
|
|
729
|
+
// peer mid-identify are indistinguishable while 'unknown' (both have an empty
|
|
730
|
+
// peerStore protocol list), so admitting an 'unknown' on the strength of a viability
|
|
731
|
+
// floor risks pulling a cross-network contaminant into the cohort — its repo dial
|
|
732
|
+
// then negotiates a different network's protocol and the whole write fails. A fresh
|
|
733
|
+
// same-network peer is not starved: it flips to 'serves' once identify completes and
|
|
734
|
+
// is re-included on the caller's retry, and in the meantime a self-only cohort still
|
|
735
|
+
// completes the write under allowClusterDownsize (the default).
|
|
736
|
+
// Scoped path only: one peerStore read per cohort member yields both protocols
|
|
737
|
+
// (for membership classification here) and addresses (reused at backfill below),
|
|
738
|
+
// so a finally-selected member isn't fetched from the peerStore twice. Left
|
|
739
|
+
// undefined on the unscoped path, which never classifies membership.
|
|
740
|
+
let peerStoreRecords: Record<string, { protocols: string[]; addrs: string[] }> | undefined
|
|
741
|
+
if (this.protocolPrefix != null) {
|
|
742
|
+
// `cohort` is the over-fetched nearest-first band. Classify each non-self
|
|
743
|
+
// member, preserving proximity order within each tier.
|
|
744
|
+
const nonSelf = cohort.filter(id => id !== selfId)
|
|
745
|
+
peerStoreRecords = await this.getPeerStoreRecordsByPeer(nonSelf)
|
|
746
|
+
const serves: string[] = []
|
|
747
|
+
const unknown: string[] = []
|
|
748
|
+
let foreignDropped = 0
|
|
749
|
+
for (const id of nonSelf) {
|
|
750
|
+
const m = this.membershipOf(id, peerStoreRecords[id]?.protocols)
|
|
751
|
+
if (m === 'serves') serves.push(id)
|
|
752
|
+
else if (m === 'unknown') unknown.push(id)
|
|
753
|
+
else foreignDropped++
|
|
754
|
+
}
|
|
755
|
+
// Take the nearest `clusterSize - 1` SERVING peers. Self is ALWAYS added below and
|
|
756
|
+
// counts toward `clusterSize` (matching the unscoped path, where `assembleCohort`
|
|
757
|
+
// returns the nearest `clusterSize` peers INCLUDING self when self is near the key —
|
|
758
|
+
// the coordinator case), so reserving a slot for self keeps a healthy same-network
|
|
759
|
+
// cohort at exactly `clusterSize` members rather than `clusterSize + 1`. Over-sizing
|
|
760
|
+
// would inflate the super-majority promise count (ceil(peerCount * threshold)) above
|
|
761
|
+
// what the configured `clusterSize` intends and hurt write availability. 'unknown'
|
|
762
|
+
// members are never backfilled: an 'unknown' peer may be a permanently cross-network
|
|
763
|
+
// contaminant whose repo dial cannot negotiate this network's protocol, and a fresh
|
|
764
|
+
// same-network peer mid-identify is indistinguishable from it. We therefore admit
|
|
765
|
+
// only positively-'serves' peers; when self is the sole serving member the cohort is
|
|
766
|
+
// self-only, which completes the write under allowClusterDownsize (the default) and
|
|
767
|
+
// re-includes any legitimate peer as 'serves' on the caller's retry once identify
|
|
768
|
+
// completes. `unknown.length` is still computed above for the diagnostic log line.
|
|
769
|
+
const nonSelfTarget = Math.max(0, this.clusterSize - 1)
|
|
770
|
+
const others = serves.slice(0, nonSelfTarget)
|
|
771
|
+
ids = Array.from(new Set([selfId, ...others]))
|
|
772
|
+
this.log('findCluster:membership key=%s serves=%d unknown=%d foreignDropped=%d kept=%d',
|
|
773
|
+
keyStr, serves.length, unknown.length, foreignDropped, ids.length)
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
const connectedByPeer = this.getConnectedAddrsByPeer()
|
|
777
|
+
const connectedPeerIds = Object.keys(connectedByPeer)
|
|
778
|
+
|
|
779
|
+
// Backfill addresses from the peerStore for cohort members we don't have
|
|
780
|
+
// a live connection to. The cohort is keyspace-determined and can include
|
|
781
|
+
// peers we know-of but haven't dialed yet; without this backfill those
|
|
782
|
+
// would be silently dropped. On the scoped path reuse the addresses already
|
|
783
|
+
// read into `peerStoreRecords` above (no second store.get per member); on the
|
|
784
|
+
// unscoped path (no record map) do the single peerStore read as before.
|
|
785
|
+
const backfillIds = ids.filter(id => id !== selfId)
|
|
786
|
+
const peerStoreAddrs = peerStoreRecords
|
|
787
|
+
? Object.fromEntries(
|
|
788
|
+
backfillIds
|
|
789
|
+
.map(id => [id, peerStoreRecords![id]?.addrs ?? []] as const)
|
|
790
|
+
.filter(([, addrs]) => addrs.length > 0)
|
|
791
|
+
)
|
|
792
|
+
: await this.getPeerStoreAddrsByPeer(backfillIds)
|
|
793
|
+
|
|
794
|
+
this.log('findCluster key=%s fretCohort=%d connected=%d', keyStr, cohort.length, connectedPeerIds.length)
|
|
795
|
+
if (verbose) this.log('findCluster:detail key=%s cohortPeers=%o connectedPeers=%o', keyStr, ids, connectedPeerIds)
|
|
796
|
+
|
|
797
|
+
const peers: ClusterPeers = {}
|
|
798
|
+
|
|
799
|
+
for (const idStr of ids) {
|
|
800
|
+
if (idStr === selfId) {
|
|
801
|
+
const raw = this.libp2p.peerId.publicKey?.raw ?? new Uint8Array()
|
|
802
|
+
peers[idStr] = { multiaddrs: this.libp2p.getMultiaddrs().map(ma => ma.toString()), publicKey: u8ToString(raw, 'base64url') }
|
|
803
|
+
continue
|
|
804
|
+
}
|
|
805
|
+
const connectedStrings = connectedByPeer[idStr] ?? []
|
|
806
|
+
const peerStoreStrings = peerStoreAddrs[idStr] ?? []
|
|
807
|
+
// De-duplicate while preserving connected-first ordering. The
|
|
808
|
+
// connected multiaddr is the one libp2p just used to reach this peer
|
|
809
|
+
// and is the most reliable; peerStore addrs are the fallback for
|
|
810
|
+
// cohort members we know-of but aren't currently connected to.
|
|
811
|
+
const merged = Array.from(new Set([...connectedStrings, ...peerStoreStrings]))
|
|
812
|
+
const parsed = this.parseMultiaddrs(merged)
|
|
813
|
+
const remotePeerId = peerIdFromString(idStr)
|
|
814
|
+
const raw = remotePeerId.publicKey?.raw ?? new Uint8Array()
|
|
815
|
+
// Note: parsed may be empty for a cohort member we have neither a
|
|
816
|
+
// live connection to nor a peerStore entry for. The dial will then
|
|
817
|
+
// surface as `code=none msg="no valid addresses"` and the caller's
|
|
818
|
+
// retry/exclude logic takes over — we intentionally do NOT drop
|
|
819
|
+
// addressless members here, because shrinking the cohort below
|
|
820
|
+
// `clusterSize` puts consensus supermajority out of reach.
|
|
821
|
+
peers[idStr] = { multiaddrs: parsed, publicKey: u8ToString(raw, 'base64url') }
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
this.log('findCluster:done key=%s ms=%d peers=%d',
|
|
825
|
+
keyStr, Date.now() - t0, Object.keys(peers).length)
|
|
826
|
+
return peers
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
/**
|
|
830
|
+
* Look up the libp2p peerStore for known multiaddrs of the given peer ids.
|
|
831
|
+
* Returns a map from peer-id string to multiaddr strings — empty/missing
|
|
832
|
+
* when the peerStore has no entry. Errors are swallowed; we'd rather fail
|
|
833
|
+
* back to the defense-in-depth drop than throw out of findCluster.
|
|
834
|
+
*/
|
|
835
|
+
private async getPeerStoreAddrsByPeer(ids: string[]): Promise<Record<string, string[]>> {
|
|
836
|
+
const out: Record<string, string[]> = {}
|
|
837
|
+
const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
|
|
838
|
+
if (!store?.get) return out
|
|
839
|
+
await Promise.all(ids.map(async (idStr) => {
|
|
840
|
+
try {
|
|
841
|
+
const pid = peerIdFromString(idStr)
|
|
842
|
+
const peer = await store.get!(pid)
|
|
843
|
+
const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
|
|
844
|
+
if (addrs.length > 0) out[idStr] = addrs
|
|
845
|
+
} catch {
|
|
846
|
+
// Unknown peer or peerStore failure — leave out of the map.
|
|
847
|
+
}
|
|
848
|
+
}))
|
|
849
|
+
return out
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
/**
|
|
853
|
+
* Single-pass peerStore read returning BOTH protocols and addresses per peer from one
|
|
854
|
+
* `store.get` call. Used on the membership-scoped `findCluster` hot path, where the
|
|
855
|
+
* cohort needs protocols (to classify membership) AND addresses (to backfill dial
|
|
856
|
+
* targets) for the same peers — reading them together avoids a second `store.get` per
|
|
857
|
+
* finally-selected member. Same error handling as {@link getPeerStoreProtocolsByPeer}
|
|
858
|
+
* and {@link getPeerStoreAddrsByPeer}: a missing peer or peerStore failure is left
|
|
859
|
+
* absent from the map (caller treats absent protocols as 'unknown', absent addrs as none).
|
|
860
|
+
*/
|
|
861
|
+
private async getPeerStoreRecordsByPeer(ids: string[]): Promise<Record<string, { protocols: string[]; addrs: string[] }>> {
|
|
862
|
+
const out: Record<string, { protocols: string[]; addrs: string[] }> = {}
|
|
863
|
+
const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[]; addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
|
|
864
|
+
if (!store?.get) return out
|
|
865
|
+
await Promise.all(ids.map(async (idStr) => {
|
|
866
|
+
try {
|
|
867
|
+
const pid = peerIdFromString(idStr)
|
|
868
|
+
const peer = await store.get!(pid)
|
|
869
|
+
const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
|
|
870
|
+
out[idStr] = { protocols: peer?.protocols ?? [], addrs }
|
|
871
|
+
} catch {
|
|
872
|
+
// Unknown peer or peerStore failure — leave out of the map.
|
|
873
|
+
}
|
|
874
|
+
}))
|
|
875
|
+
return out
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
/**
|
|
879
|
+
* Prefetch each peer's advertised protocol list from the libp2p peerStore.
|
|
880
|
+
* Returns a map from peer-id string to its protocols (empty array when the peer
|
|
881
|
+
* is absent or has not yet been identified). Mirrors {@link getPeerStoreAddrsByPeer};
|
|
882
|
+
* errors are swallowed so a peerStore hiccup degrades to "unknown" rather than throwing.
|
|
883
|
+
*/
|
|
884
|
+
private async getPeerStoreProtocolsByPeer(ids: string[]): Promise<Record<string, string[]>> {
|
|
885
|
+
const out: Record<string, string[]> = {}
|
|
886
|
+
const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[] }> } }).peerStore
|
|
887
|
+
if (!store?.get) return out
|
|
888
|
+
await Promise.all(ids.map(async (idStr) => {
|
|
889
|
+
try {
|
|
890
|
+
const pid = peerIdFromString(idStr)
|
|
891
|
+
const peer = await store.get!(pid)
|
|
892
|
+
out[idStr] = peer?.protocols ?? []
|
|
893
|
+
} catch {
|
|
894
|
+
// Unknown peer or peerStore failure — leave out (treated as 'unknown').
|
|
895
|
+
}
|
|
896
|
+
}))
|
|
897
|
+
return out
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
/**
|
|
901
|
+
* Over-fetch width for network-membership scoping. A cross-network peer can sit
|
|
902
|
+
* NEARER the key than a legitimate same-network peer and displace it from the
|
|
903
|
+
* nearest-`clusterSize` window, so when scoping is active we ask FRET for a wider
|
|
904
|
+
* proximity band and then keep the nearest peers that actually serve this network.
|
|
905
|
+
* (A ring polluted by more cross-network peers than this band is the domain of the
|
|
906
|
+
* separate FRET-side eviction follow-up; this band covers realistic co-location.)
|
|
907
|
+
*/
|
|
908
|
+
private membershipOverfetch(): number {
|
|
909
|
+
return Math.max(this.clusterSize * 4, this.clusterSize + 16)
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
/**
|
|
913
|
+
* Classify a peer's network membership from its advertised protocols. Self always
|
|
914
|
+
* `serves` (it trivially serves its own network). When no `protocolPrefix` is
|
|
915
|
+
* configured the filter is disabled and EVERY peer is reported `serves`, so all
|
|
916
|
+
* callers behave exactly as before this scoping was added.
|
|
917
|
+
*/
|
|
918
|
+
private membershipOf(idStr: string, protocols: string[] | undefined): NetworkMembership {
|
|
919
|
+
if (this.protocolPrefix == null) return 'serves'
|
|
920
|
+
if (idStr === this.libp2p.peerId.toString()) return 'serves'
|
|
921
|
+
if (protocols == null || protocols.length === 0) return 'unknown'
|
|
922
|
+
if (protocols.includes(`${this.protocolPrefix}/cluster/1.0.0`)
|
|
923
|
+
|| protocols.includes(`${this.protocolPrefix}/repo/1.0.0`)) return 'serves'
|
|
924
|
+
return 'foreign'
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
/**
|
|
928
|
+
* Scope a reputation-ordered candidate id list to this network for COORDINATOR
|
|
929
|
+
* selection: keep ONLY peers confirmed to serve this network (`serves`, which always
|
|
930
|
+
* includes self), dropping both `foreign` peers (serving another network) and
|
|
931
|
+
* `unknown` peers (peerStore protocol list empty — not yet confirmed). Incoming
|
|
932
|
+
* (reputation) order is preserved among the surviving `serves` peers. A no-op
|
|
933
|
+
* (returns the input unchanged, no drops) when `protocolPrefix` is unset or the list
|
|
934
|
+
* is empty — the membership-disabled path is therefore untouched.
|
|
935
|
+
*
|
|
936
|
+
* `droppedUnconfirmed` reports whether any candidate was excluded because it was not
|
|
937
|
+
* confirmed to serve this network — `foreign` OR `unknown` under scoping — so the
|
|
938
|
+
* caller can surface a distinct "no network coordinator" failure rather than a generic
|
|
939
|
+
* one. An `unknown` peer is not gambled on as coordinator: a permanent cross-network
|
|
940
|
+
* contaminant and a fresh same-network peer mid-identify are indistinguishable at an
|
|
941
|
+
* instant, but the filter re-reads the peerStore on every retry attempt, so a genuine
|
|
942
|
+
* same-network peer that completes `identify` within the retry window flips to `serves`
|
|
943
|
+
* and is selected normally on that attempt.
|
|
944
|
+
*/
|
|
945
|
+
private async filterByMembership(ids: string[]): Promise<{ ranked: string[]; droppedUnconfirmed: boolean }> {
|
|
946
|
+
if (this.protocolPrefix == null || ids.length === 0) return { ranked: ids, droppedUnconfirmed: false }
|
|
947
|
+
const selfStr = this.libp2p.peerId.toString()
|
|
948
|
+
const protocolsByPeer = await this.getPeerStoreProtocolsByPeer(ids.filter(id => id !== selfStr))
|
|
949
|
+
const serves: string[] = []
|
|
950
|
+
let droppedUnconfirmed = false
|
|
951
|
+
for (const id of ids) {
|
|
952
|
+
const m = this.membershipOf(id, protocolsByPeer[id])
|
|
953
|
+
if (m === 'serves') serves.push(id)
|
|
954
|
+
else droppedUnconfirmed = true
|
|
955
|
+
}
|
|
956
|
+
return { ranked: serves, droppedUnconfirmed }
|
|
957
|
+
}
|
|
958
|
+
}
|