@optimystic/db-p2p 0.26.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/certified-claims.d.ts +17 -3
- package/dist/src/cluster/certified-claims.d.ts.map +1 -1
- package/dist/src/cluster/certified-claims.js +5 -3
- package/dist/src/cluster/certified-claims.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +10 -1
- package/dist/src/cluster/cluster-repo.d.ts.map +1 -1
- package/dist/src/cluster/cluster-repo.js +11 -1
- package/dist/src/cluster/cluster-repo.js.map +1 -1
- package/dist/src/cluster/commit-proof.d.ts +16 -0
- package/dist/src/cluster/commit-proof.d.ts.map +1 -1
- package/dist/src/cluster/commit-proof.js +32 -1
- package/dist/src/cluster/commit-proof.js.map +1 -1
- package/dist/src/cluster/quorum-restore.d.ts +81 -28
- package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
- package/dist/src/cluster/quorum-restore.js +148 -51
- package/dist/src/cluster/quorum-restore.js.map +1 -1
- package/dist/src/cluster/reconcile-block.d.ts +9 -4
- package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
- package/dist/src/cluster/reconcile-block.js +28 -11
- package/dist/src/cluster/reconcile-block.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +6 -0
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/repo/cluster-coordinator.d.ts +9 -0
- package/dist/src/repo/cluster-coordinator.d.ts.map +1 -1
- package/dist/src/repo/cluster-coordinator.js +13 -2
- package/dist/src/repo/cluster-coordinator.js.map +1 -1
- package/dist/src/repo/coordinator-repo.d.ts +34 -2
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +57 -3
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/package.json +2 -2
- package/src/cluster/certified-claims.ts +22 -9
- package/src/cluster/cluster-repo.ts +12 -1
- package/src/cluster/commit-proof.ts +38 -2
- package/src/cluster/quorum-restore.ts +183 -56
- package/src/cluster/reconcile-block.ts +34 -11
- package/src/libp2p-node-base.ts +6 -0
- package/src/repo/cluster-coordinator.ts +1039 -1027
- package/src/repo/coordinator-repo.ts +1937 -1855
|
@@ -1,1855 +1,1937 @@
|
|
|
1
|
-
import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, StaleFailure, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionId, ActionRev, ActionContext, ClusterRecord, BlockUnavailableReason, ActionPending } from "@optimystic/db-core";
|
|
2
|
-
import { LruMap, blockIdsForTransforms, highestStaleAt, isConflictFailure, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
|
|
3
|
-
import { ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError } from "./cluster-coordinator.js";
|
|
4
|
-
import type { PeerId } from "@libp2p/interface";
|
|
5
|
-
import { peerIdFromString } from "@libp2p/peer-id";
|
|
6
|
-
import type { FretService } from "p2p-fret";
|
|
7
|
-
import { createLogger } from '../logger.js';
|
|
8
|
-
import type { IPeerReputation } from "../reputation/types.js";
|
|
9
|
-
import { PenaltyReason } from "../reputation/types.js";
|
|
10
|
-
import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
|
|
11
|
-
import { quorumSize, corroboratorCapacity, selectQuorumRev, certifiedEquivocation, CORROBORATION_FLOOR, type RevClaim, type QuorumRev } from "../cluster/quorum-restore.js";
|
|
12
|
-
import { certifyClaim, isAttributableProofFailure, proofThresholds, type ProofAnchoring } from "../cluster/certified-claims.js";
|
|
13
|
-
import { DEFAULT_CLUSTER_SIZE } from "../cluster/cluster-policy.js";
|
|
14
|
-
import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
|
|
15
|
-
import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON, type IRevisionActionReader } from "../storage/storage-repo.js";
|
|
16
|
-
import type
|
|
17
|
-
import type {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
*
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
corroborated
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
*
|
|
113
|
-
*
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
*
|
|
125
|
-
*
|
|
126
|
-
*
|
|
127
|
-
*
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
*
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
`
|
|
148
|
-
`
|
|
149
|
-
`
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
`
|
|
155
|
-
`
|
|
156
|
-
`
|
|
157
|
-
`
|
|
158
|
-
`
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
167
|
-
*
|
|
168
|
-
*
|
|
169
|
-
*
|
|
170
|
-
*
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
`
|
|
176
|
-
`
|
|
177
|
-
`
|
|
178
|
-
`
|
|
179
|
-
`
|
|
180
|
-
`
|
|
181
|
-
`
|
|
182
|
-
`
|
|
183
|
-
`
|
|
184
|
-
`
|
|
185
|
-
`
|
|
186
|
-
`
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
*
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
*
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
*
|
|
204
|
-
*
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
*
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
*
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
*
|
|
237
|
-
*
|
|
238
|
-
*
|
|
239
|
-
*
|
|
240
|
-
*
|
|
241
|
-
*
|
|
242
|
-
*
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
/**
|
|
256
|
-
* Optional callback
|
|
257
|
-
*
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
*
|
|
264
|
-
*
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
export
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
components.
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
components.
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
cfg?.
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
//
|
|
506
|
-
//
|
|
507
|
-
//
|
|
508
|
-
//
|
|
509
|
-
//
|
|
510
|
-
//
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
//
|
|
552
|
-
//
|
|
553
|
-
//
|
|
554
|
-
//
|
|
555
|
-
//
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
this.
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
//
|
|
584
|
-
//
|
|
585
|
-
//
|
|
586
|
-
//
|
|
587
|
-
//
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
*
|
|
663
|
-
*
|
|
664
|
-
*
|
|
665
|
-
*
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
}
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
//
|
|
811
|
-
//
|
|
812
|
-
//
|
|
813
|
-
//
|
|
814
|
-
//
|
|
815
|
-
//
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
//
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
//
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
//
|
|
842
|
-
|
|
843
|
-
//
|
|
844
|
-
//
|
|
845
|
-
//
|
|
846
|
-
|
|
847
|
-
//
|
|
848
|
-
//
|
|
849
|
-
//
|
|
850
|
-
//
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
*
|
|
966
|
-
*
|
|
967
|
-
*
|
|
968
|
-
*
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
*
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
//
|
|
1022
|
-
//
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
}
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
//
|
|
1054
|
-
//
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
);
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
const
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
});
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
//
|
|
1132
|
-
//
|
|
1133
|
-
//
|
|
1134
|
-
//
|
|
1135
|
-
//
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
//
|
|
1144
|
-
//
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
*
|
|
1196
|
-
*
|
|
1197
|
-
*
|
|
1198
|
-
*
|
|
1199
|
-
*
|
|
1200
|
-
*
|
|
1201
|
-
*
|
|
1202
|
-
*
|
|
1203
|
-
*
|
|
1204
|
-
*
|
|
1205
|
-
*
|
|
1206
|
-
*
|
|
1207
|
-
*
|
|
1208
|
-
*
|
|
1209
|
-
*
|
|
1210
|
-
*
|
|
1211
|
-
*
|
|
1212
|
-
*
|
|
1213
|
-
*
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
const
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
//
|
|
1266
|
-
//
|
|
1267
|
-
|
|
1268
|
-
//
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
this.
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
//
|
|
1285
|
-
//
|
|
1286
|
-
//
|
|
1287
|
-
//
|
|
1288
|
-
//
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
//
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
*
|
|
1339
|
-
*
|
|
1340
|
-
* `cluster/
|
|
1341
|
-
*
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
}
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
//
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
//
|
|
1430
|
-
//
|
|
1431
|
-
//
|
|
1432
|
-
//
|
|
1433
|
-
//
|
|
1434
|
-
//
|
|
1435
|
-
//
|
|
1436
|
-
//
|
|
1437
|
-
//
|
|
1438
|
-
//
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
})
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
}
|
|
1614
|
-
}
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
//
|
|
1675
|
-
//
|
|
1676
|
-
//
|
|
1677
|
-
//
|
|
1678
|
-
//
|
|
1679
|
-
//
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
//
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1
|
+
import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, StaleFailure, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionId, ActionRev, ActionContext, ClusterRecord, BlockUnavailableReason, ActionPending } from "@optimystic/db-core";
|
|
2
|
+
import { LruMap, blockIdsForTransforms, highestStaleAt, isConflictFailure, isOwnRevision, DEFAULT_SUPER_MAJORITY_THRESHOLD } from "@optimystic/db-core";
|
|
3
|
+
import { ClusterCoordinator, ConflictRaceLostError, ValidatorRejectionError } from "./cluster-coordinator.js";
|
|
4
|
+
import type { PeerId } from "@libp2p/interface";
|
|
5
|
+
import { peerIdFromString } from "@libp2p/peer-id";
|
|
6
|
+
import type { FretService } from "p2p-fret";
|
|
7
|
+
import { createLogger } from '../logger.js';
|
|
8
|
+
import type { IPeerReputation } from "../reputation/types.js";
|
|
9
|
+
import { PenaltyReason } from "../reputation/types.js";
|
|
10
|
+
import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
|
|
11
|
+
import { quorumSize, corroboratorCapacity, selectQuorumRev, certifiedEquivocation, CORROBORATION_FLOOR, type RevClaim, type QuorumRev } from "../cluster/quorum-restore.js";
|
|
12
|
+
import { certifyClaim, isAttributableProofFailure, proofThresholds, type ProofAnchoring } from "../cluster/certified-claims.js";
|
|
13
|
+
import { DEFAULT_CLUSTER_SIZE } from "../cluster/cluster-policy.js";
|
|
14
|
+
import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
|
|
15
|
+
import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON, type ICommitProofPersister, type IRevisionActionReader } from "../storage/storage-repo.js";
|
|
16
|
+
import { buildBlockCommitProof, type BlockCommitProof } from "../cluster/commit-proof.js";
|
|
17
|
+
import type { ReconcileBlockCallback } from "../cluster/cluster-repo.js";
|
|
18
|
+
import type { CertifiedActionRev } from "../storage/block-archive.js";
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Acquire a block's content for a cohort-corroborated revision, from the cohort, and persist it.
|
|
22
|
+
*
|
|
23
|
+
* Deliberately the SAME shape as the commit path's {@link ReconcileBlockCallback}, and in the live
|
|
24
|
+
* node the very same instance (`libp2p-node-base` passes its `reconcileBlock` to both): read-driven
|
|
25
|
+
* acquisition needs exactly what reconcile already provides — a per-peer-bounded archive fetch, a
|
|
26
|
+
* quorum vote on the target `(rev, actionId)`, a quorum vote on the *content* at that revision, and a
|
|
27
|
+
* persist through the monotonic, commit-latched `StorageRepo.saveReplicatedBlock` funnel. Reusing it
|
|
28
|
+
* is what keeps read-repair from being a weaker trust path than reconcile.
|
|
29
|
+
*/
|
|
30
|
+
export type AcquireBlockCallback = ReconcileBlockCallback;
|
|
31
|
+
|
|
32
|
+
/** How long one cohort peer gets to answer the latest-revision consult before it counts as silent. */
|
|
33
|
+
const LATEST_QUERY_TIMEOUT_MS = 1000;
|
|
34
|
+
|
|
35
|
+
/** True when a freshly-read local revision is strictly ahead of the baseline the repair started from. */
|
|
36
|
+
function isAdvanceOver(rev: number | undefined, baselineRev: number | undefined): boolean {
|
|
37
|
+
return typeof rev === 'number' && (baselineRev === undefined || rev > baselineRev);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Reject if `promise` has not settled within `ms`. The timer is cleared on either outcome, so no
|
|
42
|
+
* handle outlives the race (hence no `unref`, which does not exist off Node).
|
|
43
|
+
*/
|
|
44
|
+
function withDeadline<T>(promise: Promise<T>, ms: number, label: string): Promise<T> {
|
|
45
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
46
|
+
const deadline = new Promise<never>((_, reject) => {
|
|
47
|
+
timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
|
|
48
|
+
});
|
|
49
|
+
return Promise.race([promise, deadline]).finally(() => {
|
|
50
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* What one round of polling the cohort learned about a block: the revision the OTHER
|
|
56
|
+
* cohort members corroborated, and what this node itself already holds. The two are kept
|
|
57
|
+
* apart on purpose — the local revision is the baseline being repaired, never evidence
|
|
58
|
+
* about the cluster (see {@link CoordinatorRepo.queryClusterForLatest}) — but the caller
|
|
59
|
+
* still needs it to tell whether the corroborated revision is actually an advance.
|
|
60
|
+
*/
|
|
61
|
+
interface ClusterLatestQuery {
|
|
62
|
+
/** Highest `(rev, actionId)` corroborated by peers other than this node, if any. */
|
|
63
|
+
corroborated?: ActionRev;
|
|
64
|
+
/**
|
|
65
|
+
* This node's own latest for the block, as answered by the callback's self short-circuit.
|
|
66
|
+
* Typed as a {@link CertifiedActionRev} because that short-circuit reads the local proof too —
|
|
67
|
+
* of no use to this node (it trusts its own storage), but the type stays honest about what the
|
|
68
|
+
* value carries rather than silently erasing it.
|
|
69
|
+
*/
|
|
70
|
+
local?: CertifiedActionRev;
|
|
71
|
+
/**
|
|
72
|
+
* Cohort peers (self excluded) that never answered the consult — the callback rejected
|
|
73
|
+
* (dial failure, protocol error) or blew the per-peer deadline. Silence, not evidence:
|
|
74
|
+
* these are never counted as claims, but while this is non-empty a caller must not treat
|
|
75
|
+
* "nothing corroborated" as an authoritative absence, because a silent peer could be the
|
|
76
|
+
* sole holder.
|
|
77
|
+
*/
|
|
78
|
+
silent: string[];
|
|
79
|
+
/**
|
|
80
|
+
* Highest revision any cohort peer CLAIMED when no claim met the corroboration quorum
|
|
81
|
+
* (set only alongside an absent `corroborated`). The claim failed quorum, so it must
|
|
82
|
+
* never drive restoration — it exists so `get` can report content it serves below this
|
|
83
|
+
* revision as possibly behind ({@link GetBlockResult.unconfirmedAheadRev}) instead of
|
|
84
|
+
* confirmed. When a quorum DOES corroborate, higher uncorroborated claims are dropped
|
|
85
|
+
* as before: the quorum's affirmative answer outweighs a lone voter (which may simply
|
|
86
|
+
* be ahead on an in-flight commit), and stamping doubt there would mark every read that
|
|
87
|
+
* races a commit broadcast.
|
|
88
|
+
*/
|
|
89
|
+
uncorroboratedRev?: number;
|
|
90
|
+
/**
|
|
91
|
+
* How many cohort peers OTHER than this node answered the consult at all — with a claim
|
|
92
|
+
* or with "I hold nothing". `silent` says who could not be asked; this says how many
|
|
93
|
+
* could. Zero with a non-empty `silent` means this node reached NOBODY, which is a
|
|
94
|
+
* different fact from partial silence: there is no better-informed answer to be had from
|
|
95
|
+
* this node's position (see {@link AbsenceVerdict}).
|
|
96
|
+
*/
|
|
97
|
+
answered: number;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* What earlier repair passes left unresolved for one block. Two independent facts share one entry
|
|
102
|
+
* — and one map — on purpose: both are "what the last repair pass could not finish", both are
|
|
103
|
+
* cleared by the same event (the block converging), and `CoordinatorRepo` already keeps more
|
|
104
|
+
* per-block maps than anyone can hold in their head (backlog
|
|
105
|
+
* `debt-freshness-state-scattered-across-coordinator-repo`).
|
|
106
|
+
*
|
|
107
|
+
* An entry exists exactly while at least one of the two is set; both clear together in
|
|
108
|
+
* {@link CoordinatorRepo.flagUnconfirmedCurrency} once this node reaches the claimed revision.
|
|
109
|
+
*/
|
|
110
|
+
interface AheadClaimState {
|
|
111
|
+
/**
|
|
112
|
+
* The cohort-claimed revision the last freshness consult could not settle — the doubt
|
|
113
|
+
* {@link CoordinatorRepo.flagUnconfirmedCurrency} stamps onto reads served below it. Absent once a
|
|
114
|
+
* consult finds nothing ahead of what this node holds.
|
|
115
|
+
*/
|
|
116
|
+
rev?: number;
|
|
117
|
+
/**
|
|
118
|
+
* Which `cluster-fetch:repair-deadlock` reasons have already been said for this block (see
|
|
119
|
+
* {@link CoordinatorRepo.reportRepairDeadlock}). Neither reason is about any one revision — one is
|
|
120
|
+
* about the cohort's size, the other about how many of its peers hold the block — so both survive
|
|
121
|
+
* {@link CoordinatorRepo.recordAheadClaim} clearing `rev`: without that, a block whose cohort
|
|
122
|
+
* claims nothing *ahead* of the reader would re-announce the same permanent condition on every
|
|
123
|
+
* single pass, the noise this line exists to replace.
|
|
124
|
+
*
|
|
125
|
+
* Tracked per REASON rather than as one flag: the two diagnose different faults and send the
|
|
126
|
+
* operator to different places, so an episode that starts as `cohort-too-small` and becomes
|
|
127
|
+
* `sole-holder` (the operator added machines, which is what that reason told them to do) has to be
|
|
128
|
+
* able to say the second thing. Bounded at two entries by the reason union itself.
|
|
129
|
+
*/
|
|
130
|
+
deadlocksReported?: readonly DeadlockReason[];
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Why a corroboration decline is provably permanent — see {@link CoordinatorRepo.reportRepairDeadlock}
|
|
135
|
+
* for what makes each provable and which remedy each sends the operator to.
|
|
136
|
+
*/
|
|
137
|
+
type DeadlockReason = 'cohort-too-small' | 'sole-holder';
|
|
138
|
+
|
|
139
|
+
/** The `cohort-too-small` wording: the cohort cannot field the quorum however healthy its peers are. */
|
|
140
|
+
function cohortTooSmallMessage(
|
|
141
|
+
cohortPeers: number,
|
|
142
|
+
claimants: number,
|
|
143
|
+
requiredEvenIfAllAnswered: number,
|
|
144
|
+
repairCorroborationClusterSize: number
|
|
145
|
+
): string {
|
|
146
|
+
return `Block repair cannot converge for this block and the condition is PERMANENT, not transient: ` +
|
|
147
|
+
`this node's cohort has ${cohortPeers} peer(s) besides itself, all of them answered ` +
|
|
148
|
+
`(${claimants} hold the block), but accepting a revision would need ${requiredEvenIfAllAnswered} ` +
|
|
149
|
+
`agreeing peers even if every one of those ${cohortPeers} answered and agreed. No later pass can reach ` +
|
|
150
|
+
`that, however healthy every peer is, so this node's copy of the block stays as it is. Repair needs ` +
|
|
151
|
+
`${CORROBORATION_FLOOR} cohort peers BESIDES the reader to answer and agree, relaxed to 1 only for a ` +
|
|
152
|
+
`cohort that DECLARES it is smaller; repairCorroborationClusterSize currently resolves to ` +
|
|
153
|
+
`${repairCorroborationClusterSize}. Two things produce this, and this node cannot tell them ` +
|
|
154
|
+
`apart: (1) the deployment really does run this few machines — set clusterPolicy.assumedClusterSize ` +
|
|
155
|
+
`to the number you actually run (it does not lower clusterSize / the replication factor), or set an ` +
|
|
156
|
+
`honest clusterSize, and run at least ${CORROBORATION_FLOOR + 2} machines for any tolerance of one ` +
|
|
157
|
+
`unreachable peer; or (2) this node's view of the cohort has shrunk below the real deployment — a ` +
|
|
158
|
+
`partition or a routing problem, which configuration will not fix. Check the peer count above ` +
|
|
159
|
+
`against the machines you run before changing anything.`;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* The `sole-holder` wording: the cohort is big enough, but only one of its peers holds the block.
|
|
164
|
+
*
|
|
165
|
+
* Every claim here is scoped to THIS NODE'S COHORT PEERS, which is the whole of what the pass
|
|
166
|
+
* observed. It deliberately does not say "only one machine in the deployment holds this block": this
|
|
167
|
+
* node's own copy is excluded from the claim set (it cannot corroborate the revision it is trying to
|
|
168
|
+
* repair), so a reader that holds the block itself would make that reading false — and a scary
|
|
169
|
+
* all-caps line an operator can disprove by looking at their own disks is worth less than no line.
|
|
170
|
+
* For the same reason the remedy is "another COHORT PEER holding it" rather than "a second copy":
|
|
171
|
+
* with the reader holding one, a second copy already exists and is still not enough.
|
|
172
|
+
*/
|
|
173
|
+
function soleHolderMessage(cohortPeers: number): string {
|
|
174
|
+
return `Block repair cannot converge for this block and the condition is PERMANENT, not transient: ` +
|
|
175
|
+
`ONLY ONE COHORT PEER HOLDS THIS BLOCK. Of this node's ${cohortPeers} cohort peers, 1 reports holding ` +
|
|
176
|
+
`it and the other ${cohortPeers - 1} answered that they hold NOTHING — an answer, not silence, so this ` +
|
|
177
|
+
`is the whole picture and not a slow pass. Repair adopts a revision only when ${CORROBORATION_FLOOR} ` +
|
|
178
|
+
`peers BESIDES this node agree on it, and a lone holder cannot second itself, so every later pass ` +
|
|
179
|
+
`declines identically. This node's own copy, if it has one, is the copy being repaired and does not ` +
|
|
180
|
+
`count toward that number. MORE MACHINES DO NOT FIX THIS, and neither does any cluster-size setting — ` +
|
|
181
|
+
`what is missing is ANOTHER COHORT PEER HOLDING THE BLOCK. The usual cause is data written while the ` +
|
|
182
|
+
`deployment (or this block's cohort) was smaller: a block that had one holder then still has one holder ` +
|
|
183
|
+
`now, because the two paths that would replicate it — read-repair and reconcile — both decline on this ` +
|
|
184
|
+
`same rule. Committing any new revision of the block writes it to the current cohort and clears this. ` +
|
|
185
|
+
`(A lone holder whose answer carries a valid cohort commit proof for its revision IS adopted without a ` +
|
|
186
|
+
`second voter — reaching this message means the one holder attached no such proof, or one that did not ` +
|
|
187
|
+
`verify.)`;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* What one repair pass established about a block that is still MISSING locally after it.
|
|
192
|
+
* Ordered by how firmly the block is ruled out; `get` consults it only on the missing path.
|
|
193
|
+
*/
|
|
194
|
+
type AbsenceVerdict =
|
|
195
|
+
/** Nobody to ask (empty cohort, or solo-self), or every non-self cohort member answered
|
|
196
|
+
* "I hold nothing". As confirmed as an absence gets — stays authoritative, which is what
|
|
197
|
+
* keeps the routine new-collection probe at one round trip. */
|
|
198
|
+
| 'confirmed'
|
|
199
|
+
/** Some of the cohort answered and some could not be asked. (A consult that THROWS produces
|
|
200
|
+
* no verdict at all — `get`'s catch arm reports it directly.) */
|
|
201
|
+
| 'unconfirmed'
|
|
202
|
+
/** No cohort member outside this node could be asked at all. Mutually exclusive with
|
|
203
|
+
* `claimed` in practice: a claim requires a non-self peer to have answered, which is
|
|
204
|
+
* exactly what this verdict rules out. The precedence below still orders the pair, so
|
|
205
|
+
* the mapping stays total, but there is no reachable case to test. */
|
|
206
|
+
| 'isolated'
|
|
207
|
+
/** A peer claimed a revision this pass did not converge onto — quorum declined it, or a
|
|
208
|
+
* quorum corroborated it and acquisition failed. */
|
|
209
|
+
| 'claimed';
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Extended cluster interface that includes the ability to check if a transaction was executed.
|
|
213
|
+
* This is used by CoordinatorRepo to avoid duplicate execution.
|
|
214
|
+
*/
|
|
215
|
+
interface LocalClusterWithExecutionTracking extends ICluster {
|
|
216
|
+
wasTransactionExecuted?(messageHash: string): boolean;
|
|
217
|
+
/** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
|
|
218
|
+
getExecutedPendResult?(messageHash: string): PendResult | undefined;
|
|
219
|
+
/** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
|
|
220
|
+
getExecutedCommitResult?(messageHash: string): CommitResult | undefined;
|
|
221
|
+
/** Self-sign a one-peer commit proof for the solo-cohort commit path; see ClusterMember.mintSoloCommitProof.
|
|
222
|
+
* Optional like its siblings: absent on a bare ICluster double, and then the solo path simply
|
|
223
|
+
* commits proof-less — exactly the pre-mint behavior. */
|
|
224
|
+
mintSoloCommitProof?(message: RepoMessage): Promise<BlockCommitProof>;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* A cohort peer's answer to the latest-revision consult. Defined with the archive shape it is
|
|
229
|
+
* projected from (`storage/block-archive.ts`) and re-exported here so it reads next to the callback
|
|
230
|
+
* that returns it; see {@link CertifiedActionRev} there for what the optional proof does and does
|
|
231
|
+
* not mean.
|
|
232
|
+
*/
|
|
233
|
+
export type { CertifiedActionRev } from "../storage/block-archive.js";
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* Callback to query a cluster peer for their latest revision of a block. Three-way contract:
|
|
237
|
+
* - resolves a `CertifiedActionRev` — the peer answered and holds the block at that revision;
|
|
238
|
+
* - resolves `undefined` — the peer answered and holds NOTHING (an absent claim);
|
|
239
|
+
* - REJECTS — the peer could not be asked at all (dial failure, protocol error).
|
|
240
|
+
*
|
|
241
|
+
* The distinction between the last two is load-bearing: `queryClusterForLatest` counts a
|
|
242
|
+
* rejection as a SILENT peer, which stops `CoordinatorRepo.get` from reporting a locally-missing
|
|
243
|
+
* block as an authoritative absent, while a resolved `undefined` is a real answer that keeps the
|
|
244
|
+
* absent authoritative. Implementations must therefore let transport errors propagate rather than
|
|
245
|
+
* swallowing them into `undefined` (see the implementations in `libp2p-node-base` and the mesh
|
|
246
|
+
* harness's `silentPeers` failure knob). Slowness needs no handling here — the caller deadlines
|
|
247
|
+
* each query and treats expiry as silence.
|
|
248
|
+
*/
|
|
249
|
+
export type ClusterLatestCallback = (peerId: PeerId, blockId: BlockId, context?: ActionContext) => Promise<CertifiedActionRev | undefined>;
|
|
250
|
+
|
|
251
|
+
interface CoordinatorRepoComponents {
|
|
252
|
+
storageRepo: IRepo;
|
|
253
|
+
localCluster?: LocalClusterWithExecutionTracking;
|
|
254
|
+
localPeerId?: PeerId;
|
|
255
|
+
/**
|
|
256
|
+
* Optional callback to query cluster peers for their latest block revision.
|
|
257
|
+
* Used for read-path cluster verification to discover unknown revisions.
|
|
258
|
+
*/
|
|
259
|
+
clusterLatestCallback?: ClusterLatestCallback;
|
|
260
|
+
/**
|
|
261
|
+
* Optional callback that actually moves a block's bytes from the cohort into local storage once
|
|
262
|
+
* {@link clusterLatestCallback} has established a corroborated revision this node lacks. Absent →
|
|
263
|
+
* the read path can still *select* the right revision but converges only when the node already
|
|
264
|
+
* holds the corroborated action as a promotable pending. See {@link AcquireBlockCallback}.
|
|
265
|
+
*/
|
|
266
|
+
acquireBlockFromCohort?: AcquireBlockCallback;
|
|
267
|
+
/**
|
|
268
|
+
* Optional layer-2 anchoring for the cohort commit proofs the latest-revision consult verifies
|
|
269
|
+
* (`cluster/certified-claims.ts`): re-derive the block's cohort and LOG the overlap with the
|
|
270
|
+
* proof's signers, plus surface proofs accepted without that comparison. Purely observational —
|
|
271
|
+
* never a gate — and absent in production wiring today; `certifyClaim` logs unanchored
|
|
272
|
+
* acceptance internally regardless.
|
|
273
|
+
*/
|
|
274
|
+
proofAnchoring?: ProofAnchoring;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Consensus config for the coordinator side, plus the repair yardstick the read-repair path measures
|
|
279
|
+
* a (possibly shrunken) cohort view against. `repairCorroborationClusterSize` is deliberately its own
|
|
280
|
+
* field rather than an overload of {@link ClusterConsensusConfig.assumedClusterSize}: this same object
|
|
281
|
+
* also builds the `ClusterCoordinator`, and a field whose value silently differed from the cluster
|
|
282
|
+
* member's copy of it would be a trap. See `cluster/cluster-policy.ts` for why the two differ.
|
|
283
|
+
*/
|
|
284
|
+
export type CoordinatorRepoConfig = Partial<ClusterConsensusConfig> & {
|
|
285
|
+
clusterSize?: number;
|
|
286
|
+
repairCorroborationClusterSize?: number;
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
export function coordinatorRepo(
|
|
290
|
+
keyNetwork: IKeyNetwork,
|
|
291
|
+
createClusterClient: (peerId: PeerId) => ICluster,
|
|
292
|
+
cfg?: CoordinatorRepoConfig,
|
|
293
|
+
fretService?: FretService,
|
|
294
|
+
reputation?: IPeerReputation,
|
|
295
|
+
stateStore?: ITransactionStateStore
|
|
296
|
+
): (components: CoordinatorRepoComponents) => CoordinatorRepo {
|
|
297
|
+
return (components: CoordinatorRepoComponents) => new CoordinatorRepo(
|
|
298
|
+
keyNetwork,
|
|
299
|
+
createClusterClient,
|
|
300
|
+
components.storageRepo,
|
|
301
|
+
cfg,
|
|
302
|
+
components.localCluster,
|
|
303
|
+
components.localPeerId,
|
|
304
|
+
fretService,
|
|
305
|
+
components.clusterLatestCallback,
|
|
306
|
+
reputation,
|
|
307
|
+
stateStore,
|
|
308
|
+
components.acquireBlockFromCohort,
|
|
309
|
+
components.proofAnchoring
|
|
310
|
+
);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* The slice of {@link ClusterCoordinator} that {@link CoordinatorRepo} actually consumes.
|
|
315
|
+
*
|
|
316
|
+
* It exists so a test double has something COMPLETE to satisfy. The doubles in
|
|
317
|
+
* `test/coordinator-repo-*.spec.ts` replace the private `coordinator` field wholesale, and while
|
|
318
|
+
* that field was typed as the whole class and the doubles were assigned through
|
|
319
|
+
* `as unknown as { coordinator: unknown }`, every method this class newly called on the coordinator
|
|
320
|
+
* type-checked fine and then threw `... is not a function` at runtime in every spec holding a
|
|
321
|
+
* double — `getClusterPeerIds` cost 21 specs exactly that way. Widening this interface now breaks
|
|
322
|
+
* the doubles at COMPILE time, at the point of widening, which is where the cost belongs.
|
|
323
|
+
*
|
|
324
|
+
* NOTE: the doubles still name the private field by string (`{ coordinator: ... }`), so renaming
|
|
325
|
+
* `CoordinatorRepo.coordinator` would make those casts silently stop applying and every double
|
|
326
|
+
* revert to being ignored. If that field is ever renamed, grep the specs for `coordinator:`.
|
|
327
|
+
*/
|
|
328
|
+
export interface ICoordinatorClusterSeam {
|
|
329
|
+
getClusterSize(blockId: BlockId): Promise<number>;
|
|
330
|
+
getClusterPeerIds(blockId: BlockId): Promise<string[]>;
|
|
331
|
+
executeClusterTransaction(blockId: BlockId, message: RepoMessage, options?: MessageOptions): Promise<{
|
|
332
|
+
record: ClusterRecord;
|
|
333
|
+
localExecuted: boolean;
|
|
334
|
+
localPendResult?: PendResult;
|
|
335
|
+
localCommitResult?: CommitResult;
|
|
336
|
+
}>;
|
|
337
|
+
recoverTransactions(): Promise<void>;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/** Cluster coordination repo - uses local store, as well as distributes changes to other nodes using cluster consensus. */
|
|
341
|
+
export class CoordinatorRepo implements IRepo {
|
|
342
|
+
private coordinator: ICoordinatorClusterSeam;
|
|
343
|
+
private readonly DEFAULT_TIMEOUT = 30000; // 30 seconds default timeout
|
|
344
|
+
private readonly localPeerId?: PeerId;
|
|
345
|
+
private readonly responsibilityCache = new LruMap<string, { inCluster: boolean, expires: number }>(1000);
|
|
346
|
+
private static readonly RESPONSIBILITY_TTL_MS = 60_000;
|
|
347
|
+
private readonly lastSeenCommitMs = new LruMap<string, number>(1000);
|
|
348
|
+
/** Per block, what earlier repair passes left unresolved — see {@link AheadClaimState}.
|
|
349
|
+
* Outlives the consult on purpose: the read-repair window skips consults for blocks checked
|
|
350
|
+
* recently, and a doubt dropped there is a stale answer served as confirmed again.
|
|
351
|
+
* NOTE: LRU-bounded like `lastSeenCommitMs`; an eviction under >1000 doubted blocks loses the
|
|
352
|
+
* doubt until the next consult re-derives it (one read-repair window later, at worst) and lets
|
|
353
|
+
* {@link reportRepairDeadlock} say its piece a second time. */
|
|
354
|
+
private readonly unsettledAheadClaims = new LruMap<string, AheadClaimState>(1000);
|
|
355
|
+
private readonly readRepairMode: 'off' | 'lazy' | 'paranoid';
|
|
356
|
+
private readonly readRepairWindowMs: number;
|
|
357
|
+
private readonly readRepairSampleRate: number;
|
|
358
|
+
/** Simple-majority threshold from the consensus policy; drives the read-repair corroboration quorum. */
|
|
359
|
+
private readonly simpleMajorityThreshold: number;
|
|
360
|
+
/**
|
|
361
|
+
* Yardstick the read-repair corroboration floor is measured against; the floor for
|
|
362
|
+
* {@link corroboratorCapacity}. Resolved by `resolveClusterPolicy` for a real node; falls back to
|
|
363
|
+
* `assumedClusterSize` and then `clusterSize` for direct constructors (see the constructor), so a
|
|
364
|
+
* caller that has adopted neither field keeps today's behavior exactly.
|
|
365
|
+
*/
|
|
366
|
+
private readonly repairCorroborationClusterSize: number;
|
|
367
|
+
/** Resolved super-majority threshold the coordinator commits on (mirrors the value handed to ClusterCoordinator). */
|
|
368
|
+
private readonly superMajorityThreshold: number;
|
|
369
|
+
private readonly reputation?: IPeerReputation;
|
|
370
|
+
/** Per-instance logger, namespaced by peer id when `localPeerId` is known (degrades to the un-suffixed namespace when not — the single-node/test construction has always tolerated its absence). */
|
|
371
|
+
private readonly log: ReturnType<typeof createLogger>;
|
|
372
|
+
/** Test seam: overridable clock for window-based read-repair gating. */
|
|
373
|
+
now: () => number = () => Date.now();
|
|
374
|
+
/** Test seam: overridable RNG (0..1) for sample-rate gating. */
|
|
375
|
+
rand: () => number = () => Math.random();
|
|
376
|
+
|
|
377
|
+
constructor(
|
|
378
|
+
readonly keyNetwork: IKeyNetwork,
|
|
379
|
+
readonly createClusterClient: (peerId: PeerId) => ICluster,
|
|
380
|
+
private readonly storageRepo: IRepo,
|
|
381
|
+
cfg?: CoordinatorRepoConfig,
|
|
382
|
+
private readonly localCluster?: LocalClusterWithExecutionTracking,
|
|
383
|
+
localPeerId?: PeerId,
|
|
384
|
+
fretService?: FretService,
|
|
385
|
+
private readonly clusterLatestCallback?: ClusterLatestCallback,
|
|
386
|
+
reputation?: IPeerReputation,
|
|
387
|
+
stateStore?: ITransactionStateStore,
|
|
388
|
+
private readonly acquireBlockFromCohort?: AcquireBlockCallback,
|
|
389
|
+
private readonly proofAnchoring?: ProofAnchoring
|
|
390
|
+
) {
|
|
391
|
+
this.localPeerId = localPeerId;
|
|
392
|
+
this.log = createLogger('coordinator-repo', localPeerId?.toString());
|
|
393
|
+
const policy: ClusterConsensusConfig & { clusterSize: number } = {
|
|
394
|
+
// Same constant `resolveClusterPolicy` gives a node that declares no clusterSize, not a
|
|
395
|
+
// second literal: a direct constructor (the readme's manual-wiring path) and the node
|
|
396
|
+
// assembly must land on the same width or the two disagree about the same key's cohort.
|
|
397
|
+
clusterSize: cfg?.clusterSize ?? DEFAULT_CLUSTER_SIZE,
|
|
398
|
+
assumedClusterSize: cfg?.assumedClusterSize,
|
|
399
|
+
superMajorityThreshold: cfg?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
|
|
400
|
+
simpleMajorityThreshold: cfg?.simpleMajorityThreshold ?? 0.51,
|
|
401
|
+
minAbsoluteClusterSize: cfg?.minAbsoluteClusterSize ?? 3,
|
|
402
|
+
allowClusterDownsize: cfg?.allowClusterDownsize ?? true,
|
|
403
|
+
clusterSizeTolerance: cfg?.clusterSizeTolerance ?? 0.5,
|
|
404
|
+
partitionDetectionWindow: cfg?.partitionDetectionWindow ?? 60000,
|
|
405
|
+
commitBroadcastRetryInitialMs: cfg?.commitBroadcastRetryInitialMs ?? 250,
|
|
406
|
+
commitBroadcastRetryBackoffFactor: cfg?.commitBroadcastRetryBackoffFactor ?? 2,
|
|
407
|
+
commitBroadcastRetryMaxIntervalMs: cfg?.commitBroadcastRetryMaxIntervalMs ?? 8000,
|
|
408
|
+
commitBroadcastRetryMaxAttempts: cfg?.commitBroadcastRetryMaxAttempts ?? 5,
|
|
409
|
+
commitBroadcastImmediateRetries: cfg?.commitBroadcastImmediateRetries ?? 1,
|
|
410
|
+
promiseImmediateRetries: cfg?.promiseImmediateRetries ?? 1,
|
|
411
|
+
readRepairMode: cfg?.readRepairMode ?? 'lazy',
|
|
412
|
+
readRepairWindowMs: cfg?.readRepairWindowMs ?? 10000,
|
|
413
|
+
readRepairSampleRate: cfg?.readRepairSampleRate ?? 0,
|
|
414
|
+
// Default false: an undersized cluster with no confident network-size estimate
|
|
415
|
+
// is REJECTED (fail closed). Callers only opt in for single-node/local/test meshes.
|
|
416
|
+
allowUnvalidatedSmallCluster: cfg?.allowUnvalidatedSmallCluster ?? false
|
|
417
|
+
};
|
|
418
|
+
this.readRepairMode = policy.readRepairMode!;
|
|
419
|
+
this.readRepairWindowMs = policy.readRepairWindowMs!;
|
|
420
|
+
this.readRepairSampleRate = policy.readRepairSampleRate!;
|
|
421
|
+
this.simpleMajorityThreshold = policy.simpleMajorityThreshold;
|
|
422
|
+
this.superMajorityThreshold = policy.superMajorityThreshold;
|
|
423
|
+
// Unlike the membership admission gate (which treats an absent assumedClusterSize as "unknown"
|
|
424
|
+
// and admits — refusing writes outright is unacceptable), this falls back to the replication
|
|
425
|
+
// factor and stays strict: the failure mode of getting this wrong is a block that goes
|
|
426
|
+
// unrepaired, degraded rather than dead, so there is no reason to relax it for a caller that
|
|
427
|
+
// has not adopted the new field. A real node is handed an explicit
|
|
428
|
+
// `repairCorroborationClusterSize` by `resolveClusterPolicy`; the `assumedClusterSize` middle
|
|
429
|
+
// term keeps direct constructors (embedders, existing tests) behaving as before.
|
|
430
|
+
this.repairCorroborationClusterSize =
|
|
431
|
+
cfg?.repairCorroborationClusterSize ?? policy.assumedClusterSize ?? policy.clusterSize;
|
|
432
|
+
this.reputation = reputation;
|
|
433
|
+
const localClusterRef = localCluster && localPeerId ? {
|
|
434
|
+
update: localCluster.update.bind(localCluster),
|
|
435
|
+
peerId: localPeerId,
|
|
436
|
+
wasTransactionExecuted: localCluster.wasTransactionExecuted?.bind(localCluster),
|
|
437
|
+
getExecutedPendResult: localCluster.getExecutedPendResult?.bind(localCluster),
|
|
438
|
+
getExecutedCommitResult: localCluster.getExecutedCommitResult?.bind(localCluster)
|
|
439
|
+
} : undefined;
|
|
440
|
+
this.coordinator = new ClusterCoordinator(keyNetwork, createClusterClient, policy, localClusterRef, fretService, reputation, stateStore);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
/**
|
|
444
|
+
* The resolved super-majority threshold this coordinator commits on. Exposed so the composition root
|
|
445
|
+
* can fail-fast if the coordinator and the cluster member would run different thresholds (see the
|
|
446
|
+
* coupling assertion in `libp2p-node-base.ts`).
|
|
447
|
+
*/
|
|
448
|
+
get effectiveSuperMajorityThreshold(): number {
|
|
449
|
+
return this.superMajorityThreshold;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/** Recover coordinator transactions from persistent store after a restart. */
|
|
453
|
+
async recoverTransactions(): Promise<void> {
|
|
454
|
+
await this.coordinator.recoverTransactions();
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/**
|
|
458
|
+
* Check if this node is in the cluster for a given block.
|
|
459
|
+
* Uses findCluster membership — in the real network layer, self is always
|
|
460
|
+
* included in the cohort when this node is responsible. This serves as a
|
|
461
|
+
* defense-in-depth guard for requests that arrive at the wrong node.
|
|
462
|
+
* Returns true if localPeerId is not set (backward compat for single-node/test setups).
|
|
463
|
+
*/
|
|
464
|
+
private async isResponsibleForBlock(blockId: BlockId): Promise<boolean> {
|
|
465
|
+
if (!this.localPeerId) return true;
|
|
466
|
+
|
|
467
|
+
const cached = this.responsibilityCache.get(blockId);
|
|
468
|
+
if (cached && cached.expires > Date.now()) {
|
|
469
|
+
return cached.inCluster;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
const blockIdBytes = new TextEncoder().encode(blockId);
|
|
473
|
+
let inCluster: boolean;
|
|
474
|
+
try {
|
|
475
|
+
const peers = await this.keyNetwork.findCluster(blockIdBytes);
|
|
476
|
+
inCluster = this.localPeerId.toString() in peers;
|
|
477
|
+
} catch (err) {
|
|
478
|
+
this.log('proximity:check-error', { blockId, error: (err as Error).message });
|
|
479
|
+
// On failure, assume responsible to avoid false rejections
|
|
480
|
+
return true;
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
this.responsibilityCache.set(blockId, { inCluster, expires: Date.now() + CoordinatorRepo.RESPONSIBILITY_TTL_MS });
|
|
484
|
+
this.log('proximity:checked', { blockId, inCluster });
|
|
485
|
+
return inCluster;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
/**
|
|
489
|
+
* Verify this node is responsible for all given block IDs. Throws if not.
|
|
490
|
+
*/
|
|
491
|
+
private async verifyResponsibility(blockIds: BlockId[]): Promise<void> {
|
|
492
|
+
const notResponsible: BlockId[] = [];
|
|
493
|
+
for (const blockId of blockIds) {
|
|
494
|
+
if (!await this.isResponsibleForBlock(blockId)) {
|
|
495
|
+
notResponsible.push(blockId);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
if (notResponsible.length > 0) {
|
|
499
|
+
this.log('proximity:rejected', { blockIds: notResponsible });
|
|
500
|
+
throw new Error(`Not responsible for block(s): ${notResponsible.join(', ')}`);
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
async get(blockGets: BlockGets, options?: MessageOptions): Promise<GetBlockResults> {
|
|
505
|
+
// Soft proximity check — warn but still serve reads for graceful degradation
|
|
506
|
+
// NOTE: a soft-served read now also *acquires* the block durably (see restoreCorroborated), where
|
|
507
|
+
// before it could at most promote a pending this node already held. So a soft serve leaves behind
|
|
508
|
+
// a replica of a block this node is not responsible for, and nothing sweeps those: ring-shift
|
|
509
|
+
// sheds a keyspace RANGE, not "blocks outside my cohort". Fine while soft serves are what they
|
|
510
|
+
// are meant to be — a rare degradation during routing churn — since routing already placed this
|
|
511
|
+
// node near the block. If they ever become routine, gate acquisition (not the serve itself) on
|
|
512
|
+
// isResponsibleForBlock.
|
|
513
|
+
for (const blockId of blockGets.blockIds) {
|
|
514
|
+
if (!await this.isResponsibleForBlock(blockId)) {
|
|
515
|
+
this.log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
// First try local storage
|
|
520
|
+
const localResult = await this.storageRepo.get(blockGets, options);
|
|
521
|
+
|
|
522
|
+
// Decide per-block whether to consult cluster peers. Two triggers:
|
|
523
|
+
// (a) Missing — block isn't present locally at all (legacy behavior).
|
|
524
|
+
// (b) Stale-by-policy — block is present but read-repair policy says verify.
|
|
525
|
+
// Skip cluster fetch if this is already a sync request (to prevent recursive queries).
|
|
526
|
+
// A sync read is also never marked `unavailable` here — the consult it skips is the
|
|
527
|
+
// one whose failure the flag reports, and flagging would feed the recursion this
|
|
528
|
+
// bypass exists to prevent. (Storage-level 'unmaterializable' flags still pass
|
|
529
|
+
// through untouched; they report local state, not the consult.)
|
|
530
|
+
const skipClusterFetch = (options as any)?.skipClusterFetch;
|
|
531
|
+
// NOTE: NetworkTransactor.get treats an authoritative "absent" ({ state: {} })
|
|
532
|
+
// as final and no longer retries it (ticket txn-perf-authoritative-notfound),
|
|
533
|
+
// relying on this cluster reconciliation to have already run. When the consult
|
|
534
|
+
// FAILS outright — or runs without ruling the block out and the block stays
|
|
535
|
+
// missing — the entry is flagged `unavailable` below with a reason naming what
|
|
536
|
+
// the consult established (see AbsenceVerdict and the mapping in the loop body),
|
|
537
|
+
// which re-enables the transactor-level retry against a different peer. If a
|
|
538
|
+
// coordinator is configured WITHOUT clusterLatestCallback, there is no cohort to
|
|
539
|
+
// consult and the local answer IS the whole truth — it stays authoritative, with
|
|
540
|
+
// no flag and no transactor-level retry to compensate. That is fine (such a
|
|
541
|
+
// coordinator has no cluster to reconcile against), but keep this coupling in
|
|
542
|
+
// mind if a partial-cluster read path is added.
|
|
543
|
+
if (this.clusterLatestCallback && !skipClusterFetch) {
|
|
544
|
+
for (const blockId of blockGets.blockIds) {
|
|
545
|
+
const localEntry = localResult[blockId];
|
|
546
|
+
const localRev = localEntry?.state?.latest?.rev;
|
|
547
|
+
const isMissing = !localEntry?.state?.latest;
|
|
548
|
+
const isStale = !isMissing && this.shouldReadRepair(blockId);
|
|
549
|
+
if (!isMissing && !isStale) {
|
|
550
|
+
// No consult this pass — the read-repair window says this block was checked
|
|
551
|
+
// recently. An unsettled claim an earlier pass recorded still applies: the doubt
|
|
552
|
+
// is a property of what this node HOLDS, not of whether a consult just ran.
|
|
553
|
+
// Without this, every read inside the window after a failed convergence would
|
|
554
|
+
// serve the same content as confirmed — the exact silent lie this marker exists
|
|
555
|
+
// to end, re-opened for `readRepairWindowMs` at a time.
|
|
556
|
+
this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
|
|
557
|
+
continue;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
if (isStale) {
|
|
561
|
+
this.log('cluster-tx:read-repair-triggered', {
|
|
562
|
+
blockId,
|
|
563
|
+
mode: this.readRepairMode,
|
|
564
|
+
ageMs: this.ageMs(blockId),
|
|
565
|
+
localRev
|
|
566
|
+
});
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
try {
|
|
570
|
+
const { absence, claimedAheadRev } = await this.fetchBlockFromCluster(blockId, blockGets.context, localRev);
|
|
571
|
+
const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
|
|
572
|
+
const newRev = refreshed[blockId]?.state?.latest?.rev;
|
|
573
|
+
if (refreshed[blockId]) {
|
|
574
|
+
localResult[blockId] = refreshed[blockId];
|
|
575
|
+
}
|
|
576
|
+
if (isStale) {
|
|
577
|
+
if (typeof newRev === 'number' && typeof localRev === 'number' && newRev > localRev) {
|
|
578
|
+
this.log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
|
|
579
|
+
} else {
|
|
580
|
+
this.log('cluster-tx:read-repair-noop', { blockId });
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
// The consult ran but could not rule the block out, and the verdict names the
|
|
584
|
+
// evidence (see AbsenceVerdict): part of the cohort was silent (`unconfirmed`
|
|
585
|
+
// → 'peers-unreachable' — another coordinator may know better), no cohort
|
|
586
|
+
// member outside this node could be asked at all (`isolated` →
|
|
587
|
+
// 'cohort-unreachable' — there is no better-connected coordinator to re-ask),
|
|
588
|
+
// or a peer positively claimed a revision this pass could neither corroborate
|
|
589
|
+
// nor acquire (`claimed` → 'claimed-elsewhere' — the block is known to exist
|
|
590
|
+
// somewhere). Either way a still-missing block must not pose as an
|
|
591
|
+
// authoritative absent. When the whole cohort answers "holds nothing" the
|
|
592
|
+
// absent stays authoritative (`confirmed`) — the new-collection probe against
|
|
593
|
+
// a healthy cohort stays one round-trip.
|
|
594
|
+
if (isMissing && absence !== 'confirmed') {
|
|
595
|
+
this.flagUnconfirmedAbsence(localResult, blockId,
|
|
596
|
+
absence === 'claimed' ? 'claimed-elsewhere'
|
|
597
|
+
: absence === 'isolated' ? 'cohort-unreachable'
|
|
598
|
+
: 'peers-unreachable');
|
|
599
|
+
}
|
|
600
|
+
// A PRESENT block served below a cohort claim the repair could not settle is
|
|
601
|
+
// the mirror lie: real content posing as confirmed-current. This consult is the
|
|
602
|
+
// authority on that claim, so it replaces whatever an earlier one recorded —
|
|
603
|
+
// including clearing it when nobody claims anything any more. The missing case
|
|
604
|
+
// is excluded — it is the absence path above, and a bare absent below a claim
|
|
605
|
+
// already reads as either authoritative (cohort answered, nothing corroborated)
|
|
606
|
+
// or flagged.
|
|
607
|
+
if (!isMissing) {
|
|
608
|
+
this.recordAheadClaim(blockId, claimedAheadRev);
|
|
609
|
+
this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
|
|
610
|
+
}
|
|
611
|
+
} catch (err) {
|
|
612
|
+
this.log('cluster-fetch:error', { blockId, error: (err as Error).message });
|
|
613
|
+
// The consult that was supposed to make this answer trustworthy did not run.
|
|
614
|
+
// NOTE: a consult that THROWS (e.g. `findCluster` itself rejected) is reported
|
|
615
|
+
// 'peers-unreachable' even on an isolated node: a failed cohort lookup is a
|
|
616
|
+
// routing failure and says nothing about how many cohort members were
|
|
617
|
+
// reachable. If `findCluster` on an isolated node turns out to throw routinely
|
|
618
|
+
// rather than return a stale cohort view, revisit — that would put the
|
|
619
|
+
// isolated case back under this vaguer reason.
|
|
620
|
+
if (isMissing) {
|
|
621
|
+
this.flagUnconfirmedAbsence(localResult, blockId, 'peers-unreachable');
|
|
622
|
+
} else {
|
|
623
|
+
// It told us nothing, so it refutes nothing: an earlier pass's unsettled
|
|
624
|
+
// claim stands.
|
|
625
|
+
this.flagUnconfirmedCurrency(localResult, blockId, blockGets.context);
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
return localResult;
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
/**
|
|
635
|
+
* Downgrade an absence the coordinator could not confirm to the given `unavailable` reason —
|
|
636
|
+
* a flag `NetworkTransactor.get` retries against another peer instead of taking as final.
|
|
637
|
+
* The reason names the evidence (see {@link AbsenceVerdict} for the mapping in `get`); this
|
|
638
|
+
* method only decides WHETHER the entry may carry a flag at all.
|
|
639
|
+
*
|
|
640
|
+
* No-op once the entry carries a real answer (the consult restored the block) or a sharper flag
|
|
641
|
+
* (storage's `'unmaterializable'`), so callers only need to establish that the answer is a guess.
|
|
642
|
+
*
|
|
643
|
+
* "Carries a real answer" is tested as `entry.block !== undefined`, NOT as `state.latest` being
|
|
644
|
+
* set. The two used to move together, so `state.latest` read as a serviceable proxy — but a
|
|
645
|
+
* pending-only insert (pended, not yet committed) served through the pending overlay has real
|
|
646
|
+
* CONTENT and no committed revision at all, so its `state.latest` is undefined. Flagging that
|
|
647
|
+
* entry would mark a block this node is positively holding as an unconfirmed absence, and
|
|
648
|
+
* `NetworkTransactor`'s `isAuthoritative` keys off the flag alone — the read would burn its
|
|
649
|
+
* retry budget re-asking other peers for content it already has. `state.latest` stays in the
|
|
650
|
+
* test as well so a stale-but-real committed answer is likewise never downgraded.
|
|
651
|
+
*/
|
|
652
|
+
private flagUnconfirmedAbsence(results: GetBlockResults, blockId: BlockId, reason: BlockUnavailableReason): void {
|
|
653
|
+
const entry = results[blockId];
|
|
654
|
+
if (!entry) {
|
|
655
|
+
results[blockId] = { state: {}, unavailable: reason };
|
|
656
|
+
} else if (entry.block === undefined && !entry.state?.latest && entry.unavailable === undefined) {
|
|
657
|
+
entry.unavailable = reason;
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
/**
|
|
662
|
+
* Remember (or forget) the cohort claim a freshness consult could not settle for a block.
|
|
663
|
+
* Only a consult that actually RAN may call this: it is the authority, so `undefined` clears
|
|
664
|
+
* a claim an earlier pass recorded. Entries are also dropped once this node reaches the
|
|
665
|
+
* claimed revision (see {@link flagUnconfirmedCurrency}), which is what bounds the map.
|
|
666
|
+
*/
|
|
667
|
+
private recordAheadClaim(blockId: BlockId, claimedRev: number | undefined): void {
|
|
668
|
+
const prior = this.unsettledAheadClaims.get(blockId);
|
|
669
|
+
if (claimedRev === undefined) {
|
|
670
|
+
// The consult is the authority on the CLAIM, and only on the claim. A recorded deadlock is
|
|
671
|
+
// not about any revision — it is about how many machines this deployment can field, or how
|
|
672
|
+
// many of them hold the block — so it outlives the claim that first exposed it and is
|
|
673
|
+
// dropped only when the block converges (see {@link flagUnconfirmedCurrency}).
|
|
674
|
+
if (prior?.deadlocksReported) this.unsettledAheadClaims.set(blockId, { deadlocksReported: prior.deadlocksReported });
|
|
675
|
+
else this.unsettledAheadClaims.delete(blockId);
|
|
676
|
+
return;
|
|
677
|
+
}
|
|
678
|
+
this.unsettledAheadClaims.set(blockId, {
|
|
679
|
+
rev: claimedRev,
|
|
680
|
+
...(prior?.deadlocksReported ? { deadlocksReported: prior.deadlocksReported } : {})
|
|
681
|
+
});
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
/**
|
|
685
|
+
* Stamp {@link GetBlockResult.unconfirmedAheadRev} on an entry sitting behind an unsettled
|
|
686
|
+
* cohort claim — served committed content the coordinator cannot confirm is current.
|
|
687
|
+
* Deliberately narrow; ALL of these must hold:
|
|
688
|
+
* - a consult (this read's or an earlier one's, see {@link recordAheadClaim}) left a claim
|
|
689
|
+
* unsettled for this block;
|
|
690
|
+
* - the entry carries a committed revision (a present block, or a committed tombstone) —
|
|
691
|
+
* never a plain absent, which is the absence path's business;
|
|
692
|
+
* - that served revision is still strictly BELOW the claim: the repair did not converge, and
|
|
693
|
+
* nothing committed past the claim in the meantime (if it did, the claim is settled and the
|
|
694
|
+
* memo is dropped here);
|
|
695
|
+
* - the caller asked for a view that should contain the claim: an unpinned "latest" read, or
|
|
696
|
+
* a pin at/above the claimed revision. A read pinned BELOW the claim is being served
|
|
697
|
+
* correctly and stays unstamped — this keeps a collection's context-pinned data reads
|
|
698
|
+
* quiet while its unpinned tail read (the one seam where fresher truth could arrive —
|
|
699
|
+
* Collection.bootstrapContext) speaks up.
|
|
700
|
+
* NOT covered, on purpose: a cohort that is merely silent and claims nothing (pinned as
|
|
701
|
+
* authoritative by the merely-STALE spec in coordinator-repo-unavailable.spec.ts) — silence
|
|
702
|
+
* carries no revision to be behind of.
|
|
703
|
+
*
|
|
704
|
+
* Pin comparability: `ActionContext.rev` and a block's `state.latest.rev` count the same
|
|
705
|
+
* per-collection revision sequence — `Collection.bootstrapContext` seeds the context straight
|
|
706
|
+
* from the tail block's `latest.rev`, and `syncInternal` commits every block of an action at
|
|
707
|
+
* `context.rev + 1` — so `context.rev >= claimedRev` is a well-defined comparison. `state.latest`
|
|
708
|
+
* is this node's newest revision for the block even on a pinned read (StorageRepo reports the
|
|
709
|
+
* content's own revision separately as `materialized`), which is exactly the number "is this
|
|
710
|
+
* node behind the claim?" asks about.
|
|
711
|
+
*/
|
|
712
|
+
private flagUnconfirmedCurrency(results: GetBlockResults, blockId: BlockId, context?: ActionContext): void {
|
|
713
|
+
const claimedRev = this.unsettledAheadClaims.get(blockId)?.rev;
|
|
714
|
+
if (claimedRev === undefined) return;
|
|
715
|
+
const entry = results[blockId];
|
|
716
|
+
if (!entry || entry.unavailable !== undefined) return;
|
|
717
|
+
const servedRev = entry.state?.latest?.rev;
|
|
718
|
+
if (typeof servedRev !== 'number') return;
|
|
719
|
+
if (servedRev >= claimedRev) {
|
|
720
|
+
// Caught up — by this pass's repair or by a commit that landed since. Nothing to doubt, and
|
|
721
|
+
// nothing deadlocked either: repair demonstrably converged for this block, so a later
|
|
722
|
+
// non-convergence is a new episode and gets to say so again.
|
|
723
|
+
this.unsettledAheadClaims.delete(blockId);
|
|
724
|
+
return;
|
|
725
|
+
}
|
|
726
|
+
if (context !== undefined && context.rev < claimedRev) return;
|
|
727
|
+
entry.unconfirmedAheadRev = claimedRev;
|
|
728
|
+
this.log('cluster-tx:read-unconfirmed', { blockId, servedRev, claimedAheadRev: claimedRev });
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
/** Decide whether the read-repair policy wants us to consult the cluster for a present-but-possibly-stale block. */
|
|
732
|
+
private shouldReadRepair(blockId: BlockId): boolean {
|
|
733
|
+
switch (this.readRepairMode) {
|
|
734
|
+
case 'off': return false;
|
|
735
|
+
case 'paranoid': return true;
|
|
736
|
+
case 'lazy': {
|
|
737
|
+
const lastSeen = this.lastSeenCommitMs.get(blockId);
|
|
738
|
+
if (lastSeen == null) return true;
|
|
739
|
+
if (this.now() - lastSeen > this.readRepairWindowMs) return true;
|
|
740
|
+
if (this.readRepairSampleRate > 0 && this.rand() < this.readRepairSampleRate) return true;
|
|
741
|
+
return false;
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
/** Milliseconds since we last marked this block fresh, or undefined if never. */
|
|
747
|
+
private ageMs(blockId: BlockId): number | undefined {
|
|
748
|
+
const lastSeen = this.lastSeenCommitMs.get(blockId);
|
|
749
|
+
return lastSeen == null ? undefined : this.now() - lastSeen;
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
/** Mark blocks as freshly observed from cluster authority (post-commit or post-fetch). */
|
|
753
|
+
private markBlocksSeen(blockIds: BlockId[]): void {
|
|
754
|
+
const now = this.now();
|
|
755
|
+
for (const id of blockIds) {
|
|
756
|
+
this.lastSeenCommitMs.set(id, now);
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
/**
|
|
761
|
+
* Test seam: directly set the last-seen timestamp for a block. Used by read-repair
|
|
762
|
+
* specs to simulate "the local commit happened at time T" without needing to drive
|
|
763
|
+
* a full pend/commit cycle through the cluster coordinator.
|
|
764
|
+
*/
|
|
765
|
+
setLastSeenForTest(blockId: BlockId, ts: number): void {
|
|
766
|
+
this.lastSeenCommitMs.set(blockId, ts);
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
/**
|
|
770
|
+
* One repair pass for a block: ask the cohort what it holds, and converge onto that if it is
|
|
771
|
+
* ahead of `localRev` — the revision the caller's read already loaded, and the baseline every
|
|
772
|
+
* decision below is measured against.
|
|
773
|
+
*
|
|
774
|
+
* Returns the two things `get` needs beyond the storage side effects:
|
|
775
|
+
* - `absence` — the verdict on this node's local absence of the block (see
|
|
776
|
+
* {@link AbsenceVerdict}): whether the pass may rule the block out, and on what evidence.
|
|
777
|
+
* Only `'confirmed'` lets a still-missing block be reported as an authoritative absent.
|
|
778
|
+
* Paths that consult nobody (no cohort, solo-self) are `'confirmed'`: there, the local
|
|
779
|
+
* answer genuinely is the whole truth. When several verdicts apply at once the sharpest
|
|
780
|
+
* evidence wins: `claimed` > `isolated` > `unconfirmed` > `confirmed` — a peer positively
|
|
781
|
+
* saying "it exists" outranks any amount of silence.
|
|
782
|
+
* - `claimedAheadRev` — a cohort peer claimed a revision strictly ahead of what this node
|
|
783
|
+
* holds and the pass did NOT converge onto it: the claim failed the corroboration quorum,
|
|
784
|
+
* or was corroborated but could not be acquired. Content `get` serves below this revision
|
|
785
|
+
* cannot be confirmed current (see {@link GetBlockResult.unconfirmedAheadRev}); the claim
|
|
786
|
+
* itself must never drive restoration.
|
|
787
|
+
*/
|
|
788
|
+
private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext, localRev?: number): Promise<{ absence: AbsenceVerdict; claimedAheadRev?: number }> {
|
|
789
|
+
if (!this.clusterLatestCallback) return { absence: 'confirmed' };
|
|
790
|
+
|
|
791
|
+
const blockIdBytes = new TextEncoder().encode(blockId);
|
|
792
|
+
const peers = await this.keyNetwork.findCluster(blockIdBytes);
|
|
793
|
+
const peerIds = peers ? Object.keys(peers) : [];
|
|
794
|
+
if (peerIds.length === 0) return { absence: 'confirmed' };
|
|
795
|
+
|
|
796
|
+
// Solo-cluster short-circuit: the only responsible peer is us. There is no
|
|
797
|
+
// remote to sync from, so skip the callback entirely. Querying ourselves
|
|
798
|
+
// would dial self via SyncClient — pointless at best, and on nodes without
|
|
799
|
+
// listen addresses (e.g. solo WebSocket-only) the dial can hang.
|
|
800
|
+
if (
|
|
801
|
+
peerIds.length === 1
|
|
802
|
+
&& this.localPeerId
|
|
803
|
+
&& peerIds[0] === this.localPeerId.toString()
|
|
804
|
+
) {
|
|
805
|
+
this.log('cluster-fetch:solo-self-skip', { blockId });
|
|
806
|
+
return { absence: 'confirmed' };
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
const { corroborated, local, silent, answered, uncorroboratedRev } = await this.queryClusterForLatest(peerIds, blockId, context);
|
|
810
|
+
// Any silence taints the WHOLE consult, not a fraction of it (fail-closed): one silent
|
|
811
|
+
// peer could be the sole holder, and the cost — an extra transactor-level retry against
|
|
812
|
+
// another coordinator — is paid only while a peer is actually unreachable. Silence with
|
|
813
|
+
// NOBODY else reached at all is its own verdict: partial silence says "ask a better-
|
|
814
|
+
// connected coordinator", total silence says there is no better-informed answer to be
|
|
815
|
+
// had from this node.
|
|
816
|
+
const silenceVerdict: AbsenceVerdict =
|
|
817
|
+
silent.length > 0 ? (answered === 0 ? 'isolated' : 'unconfirmed') : 'confirmed';
|
|
818
|
+
// Nothing corroborated: keep local data AND stay eligible for repair — marking the
|
|
819
|
+
// block seen here would suppress the next attempt for the whole read-repair window.
|
|
820
|
+
// An uncorroborated claim strictly ahead of what this node holds still travels up as
|
|
821
|
+
// doubt: the answer about to be served may be behind it, and only the caller knows
|
|
822
|
+
// whether that matters for the view it was asked for.
|
|
823
|
+
if (!corroborated) {
|
|
824
|
+
const uncorroboratedBaseline = local?.rev ?? localRev;
|
|
825
|
+
const claimIsAhead = uncorroboratedRev !== undefined
|
|
826
|
+
&& (uncorroboratedBaseline === undefined || uncorroboratedRev > uncorroboratedBaseline);
|
|
827
|
+
// A claim — even one the quorum declined — is a peer positively attesting the block
|
|
828
|
+
// exists, the sharpest fact this pass can surface. It outranks silence.
|
|
829
|
+
const absence: AbsenceVerdict = uncorroboratedRev !== undefined ? 'claimed' : silenceVerdict;
|
|
830
|
+
return { absence, ...(claimIsAhead ? { claimedAheadRev: uncorroboratedRev } : {}) };
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
// The self answer is the sharper baseline (same storage, same context, read alongside the
|
|
834
|
+
// cohort's), but it exists only when `findCluster` returned this node. A soft serve for a
|
|
835
|
+
// block this node is no longer responsible for is absent from its own cohort view, so fall
|
|
836
|
+
// back to the revision the caller's read already loaded. Without the fallback both decisions
|
|
837
|
+
// below degrade to "any local revision is an advance", which restores backwards and reports
|
|
838
|
+
// a sync at the revision the pass started from.
|
|
839
|
+
const baselineRev = local?.rev ?? localRev;
|
|
840
|
+
|
|
841
|
+
// Never restore backwards. With this node's own claim excluded from the quorum, a
|
|
842
|
+
// cohort that lags behind the reader corroborates an OLDER revision; adopting it
|
|
843
|
+
// would be a regression, and logging it as a sync would be a lie. The cohort did
|
|
844
|
+
// answer, so the block is verified fresh — mark it seen.
|
|
845
|
+
// NOTE: in a cohort of two, that sole peer is the only corroborator, so a lying one can park
|
|
846
|
+
// the reader here — corroborating the revision it already holds — and re-arm the lazy window
|
|
847
|
+
// on every pass, hiding a real divergence. Bounded by `readRepairWindowMs` (10s default) and
|
|
848
|
+
// no worse than the peer simply staying silent. If two-member cohorts become a supported
|
|
849
|
+
// production topology rather than a dev convenience, stop re-arming the window on a
|
|
850
|
+
// corroboration that came from a single voter.
|
|
851
|
+
if (baselineRev !== undefined && corroborated.rev <= baselineRev) {
|
|
852
|
+
this.log('cluster-fetch:local-current', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
|
|
853
|
+
this.markBlocksSeen([blockId]);
|
|
854
|
+
// Only reachable when this node HOLDS a revision (the baseline), so `get` never
|
|
855
|
+
// consults this verdict — computed consistently rather than hard-coded.
|
|
856
|
+
return { absence: silenceVerdict };
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
// Corroborated revision is ahead of ours — converge onto it.
|
|
860
|
+
const rev = await this.restoreCorroborated(blockId, corroborated, baselineRev, peerIds);
|
|
861
|
+
|
|
862
|
+
// Log the OUTCOME, not the attempt. Logging `synced` unconditionally reported hundreds of
|
|
863
|
+
// phantom convergences per run and made a real replication defect invisible for two debugging
|
|
864
|
+
// sessions.
|
|
865
|
+
if (rev !== undefined) {
|
|
866
|
+
this.log('cluster-fetch:synced', { blockId, rev });
|
|
867
|
+
} else {
|
|
868
|
+
this.log('cluster-fetch:not-restored', { blockId, localRev: baselineRev, clusterRev: corroborated.rev });
|
|
869
|
+
}
|
|
870
|
+
// A corroborated revision this node failed to converge onto rules nothing out, even with
|
|
871
|
+
// the whole cohort answering: the reader has just been TOLD the block exists, so reporting
|
|
872
|
+
// it absent would be a lie regardless of silence — that is the `claimed` verdict, and it
|
|
873
|
+
// outranks whatever the silence-based mapping would have said.
|
|
874
|
+
const absence: AbsenceVerdict = rev === undefined ? 'claimed' : silenceVerdict;
|
|
875
|
+
// Converged means REACHED the corroborated revision, not merely advanced: a promotion that
|
|
876
|
+
// landed short of it (possible in principle — restoreCorroborated only requires an advance
|
|
877
|
+
// over the baseline) still leaves the served answer behind a revision the cohort attested.
|
|
878
|
+
const converged = rev !== undefined && rev >= corroborated.rev;
|
|
879
|
+
// The block is marked seen either way — the cohort DID answer, so its freshness was checked,
|
|
880
|
+
// which is what the read-repair window tracks. A failed convergence therefore waits out the
|
|
881
|
+
// window before retrying. The DOUBT it produced does not wait: `get` remembers the
|
|
882
|
+
// unsettled claim (`recordAheadClaim`) and keeps stamping reads served below it while the
|
|
883
|
+
// window suppresses the retry — the window damps repair effort, not honesty.
|
|
884
|
+
// NOTE: that damping covers only a block this node holds at an OLDER revision. A block entirely
|
|
885
|
+
// missing locally never consults the window (`get` triggers on `isMissing` before
|
|
886
|
+
// `shouldReadRepair`), so a persistently failing acquisition — e.g. a two-node deployment that
|
|
887
|
+
// never set `assumedClusterSize`, where the content quorum can never be met — re-fetches an
|
|
888
|
+
// archive on every read of that block. Correct, and self-limiting once the cohort can agree; if
|
|
889
|
+
// it ever shows as read amplification, gate the acquisition step (not the latest-query) on the
|
|
890
|
+
// same window rather than widening `isMissing`.
|
|
891
|
+
this.markBlocksSeen([blockId]);
|
|
892
|
+
return { absence, ...(converged ? {} : { claimedAheadRev: corroborated.rev }) };
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
/**
|
|
896
|
+
* Bring this node up to the cohort-corroborated `corroborated`, returning the revision it holds
|
|
897
|
+
* afterwards when that is an advance over `baselineRev`, else `undefined`.
|
|
898
|
+
*
|
|
899
|
+
* Two mechanisms, cheapest first:
|
|
900
|
+
* 1. **Promote a local pending** — free, no network, and the only mechanism that existed before
|
|
901
|
+
* block acquisition. Covers the node that saw the pend and missed the commit broadcast.
|
|
902
|
+
* 2. **Acquire the bytes from the cohort** ({@link AcquireBlockCallback}) — covers everything else,
|
|
903
|
+
* including a block this node has never seen at all.
|
|
904
|
+
*
|
|
905
|
+
* **Why acquisition is gated here and not on a plain local miss.** `BlockStorage.getBlock` returns
|
|
906
|
+
* `undefined` for a block with no local metadata *without* consulting its restore callback, so that
|
|
907
|
+
* an insert probing a fresh random block id for a collision does not cost a network fetch. That
|
|
908
|
+
* remains true: this method runs only after {@link queryClusterForLatest} produced a quorum-
|
|
909
|
+
* corroborated `(rev, actionId)`, which a genuinely non-existent block can never produce (no peer
|
|
910
|
+
* claims it, so `selectQuorumRev` declines and `fetchBlockFromCluster` returns before reaching
|
|
911
|
+
* here). The cost of a genuine absence is unchanged — the latest-query round trip that already
|
|
912
|
+
* happened — while a block the cohort demonstrably holds is no longer thrown away.
|
|
913
|
+
*
|
|
914
|
+
* Cohort peer ids are passed straight through: the callback filters self out and caps its own
|
|
915
|
+
* corroboration quorum by how many peers could answer at all.
|
|
916
|
+
*/
|
|
917
|
+
private async restoreCorroborated(
|
|
918
|
+
blockId: BlockId,
|
|
919
|
+
corroborated: ActionRev,
|
|
920
|
+
baselineRev: number | undefined,
|
|
921
|
+
cohortPeerIds: string[]
|
|
922
|
+
): Promise<number | undefined> {
|
|
923
|
+
const promoted = await this.promoteCorroborated(blockId, corroborated);
|
|
924
|
+
if (isAdvanceOver(promoted, baselineRev)) {
|
|
925
|
+
return promoted;
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
if (!this.acquireBlockFromCohort) {
|
|
929
|
+
return undefined;
|
|
930
|
+
}
|
|
931
|
+
try {
|
|
932
|
+
// Bounded: a stalled cohort peer must not hold up the caller's read. Persisting happens
|
|
933
|
+
// inside the callback via `saveReplicatedBlock`, which takes the block write latch —
|
|
934
|
+
// safe to call from here because the read path holds no latch of its own (`StorageRepo.get`
|
|
935
|
+
// acquires and releases it around the promotion above, and nothing wraps this method).
|
|
936
|
+
// NOTE: `get` walks its block ids sequentially, so the bound is per block, not per call — a
|
|
937
|
+
// multi-block read that is missing N blocks against a wholly stalled cohort waits N × this.
|
|
938
|
+
// Acceptable today (the underlying per-peer archive fetch is itself 1s-bounded and runs the
|
|
939
|
+
// cohort in parallel, so the 5s is a stall ceiling, not a typical cost). If a cold reader
|
|
940
|
+
// batching a wide read ever times out above this layer, repair the block ids concurrently
|
|
941
|
+
// rather than shortening the bound.
|
|
942
|
+
await withDeadline(
|
|
943
|
+
this.acquireBlockFromCohort(blockId, corroborated, cohortPeerIds),
|
|
944
|
+
RECONCILE_TIMEOUT_MS,
|
|
945
|
+
`block acquisition for ${blockId}`
|
|
946
|
+
);
|
|
947
|
+
} catch (err) {
|
|
948
|
+
// Declines are cheap and retryable — nothing was persisted. Report and leave the block behind.
|
|
949
|
+
this.log('cluster-fetch:acquire-error', { blockId, rev: corroborated.rev, error: (err as Error).message });
|
|
950
|
+
return undefined;
|
|
951
|
+
}
|
|
952
|
+
const acquired = await this.readLocalRev(blockId);
|
|
953
|
+
return isAdvanceOver(acquired, baselineRev) ? acquired : undefined;
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
/**
|
|
957
|
+
* Promote a corroborated action this node already holds as a local pending — the no-network half of
|
|
958
|
+
* the repair. Returns the local revision afterwards.
|
|
959
|
+
*
|
|
960
|
+
* A pending-only block (metadata seeded by `savePendingTransaction`, no committed revision) asked
|
|
961
|
+
* for a forward revision no promotion can reach used to throw out of the restore step (now
|
|
962
|
+
* `BlockStorage.restoreRevision`, driven by `StorageRepo.get`'s healing helper).
|
|
963
|
+
* It no longer does: "no committed base here" is an absence, so that read comes back as a plain
|
|
964
|
+
* unflagged `{ state: {} }` and this method simply returns `undefined` — acquisition then supplies
|
|
965
|
+
* the revision. The `unavailable` arm below still fires for the shapes that ARE a guess (a `latest`
|
|
966
|
+
* this node cannot materialize, a missing-base promotion refusal); on THIS path those are an
|
|
967
|
+
* absence too rather than a read failure, so they are logged as `promote-unavailable` and stepped
|
|
968
|
+
* over rather than short-circuiting the caller. The catch stays for any other fault, same reason.
|
|
969
|
+
*/
|
|
970
|
+
private async promoteCorroborated(blockId: BlockId, corroborated: ActionRev): Promise<number | undefined> {
|
|
971
|
+
try {
|
|
972
|
+
const entry = await this.readLocalEntry(blockId, { committed: [corroborated], rev: corroborated.rev });
|
|
973
|
+
if (entry?.unavailable !== undefined) {
|
|
974
|
+
this.log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: entry.unavailable });
|
|
975
|
+
return undefined;
|
|
976
|
+
}
|
|
977
|
+
return entry?.state?.latest?.rev;
|
|
978
|
+
} catch (err) {
|
|
979
|
+
this.log('cluster-fetch:promote-unavailable', { blockId, rev: corroborated.rev, error: (err as Error).message });
|
|
980
|
+
return undefined;
|
|
981
|
+
}
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
/** This node's own answer for a block, optionally driving a promotion context through the read.
|
|
985
|
+
* Callers that care whether the answer is authoritative inspect `entry.unavailable`. */
|
|
986
|
+
private async readLocalEntry(blockId: BlockId, context?: ActionContext) {
|
|
987
|
+
const result = await this.storageRepo.get({ blockIds: [blockId], context });
|
|
988
|
+
return result[blockId];
|
|
989
|
+
}
|
|
990
|
+
|
|
991
|
+
/** This node's own `latest.rev` for a block, optionally driving a promotion context through the read. */
|
|
992
|
+
private async readLocalRev(blockId: BlockId, context?: ActionContext): Promise<number | undefined> {
|
|
993
|
+
return (await this.readLocalEntry(blockId, context))?.state?.latest?.rev;
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
/**
|
|
997
|
+
* Query cluster peers for their latest revision and return the highest revision
|
|
998
|
+
* corroborated by a quorum of distinct peers, alongside this node's own latest.
|
|
999
|
+
*
|
|
1000
|
+
* Replaces the old "max rev any single peer reports" — which let one lying
|
|
1001
|
+
* peer over-reporting its revision steer restoration — with quorum
|
|
1002
|
+
* corroboration on the exact `(rev, actionId)` pair (see {@link selectQuorumRev}).
|
|
1003
|
+
*
|
|
1004
|
+
* This node's own answer is split out of the claim set rather than counted in it:
|
|
1005
|
+
* `clusterLatestCallback` short-circuits self to local storage, so including it let a
|
|
1006
|
+
* reader whose only peer timed out "corroborate" the very revision it was trying to
|
|
1007
|
+
* repair. It is returned separately so the caller can compare, not vote.
|
|
1008
|
+
*
|
|
1009
|
+
* NOTE: the quorum is corroboration-of-a-claim, NOT Sybil-resistant cohort
|
|
1010
|
+
* membership — a peer minting fresh keypairs still casts a vote. A claim that arrives
|
|
1011
|
+
* with a cohort commit proof is additionally VERIFIED here (`certifyClaim`,
|
|
1012
|
+
* `cluster/certified-claims.ts`); when the proof holds, the claim is certified and
|
|
1013
|
+
* {@link selectQuorumRev} accepts it without a second voter — the cohort's signature set
|
|
1014
|
+
* is its corroboration. What a passing proof does NOT prove is that its signers are the
|
|
1015
|
+
* block's responsible cohort (anyone controlling N keys can sign their own N-peer
|
|
1016
|
+
* proof); anchoring the signer set to topology is the optional, observational-only
|
|
1017
|
+
* {@link ProofAnchoring} layer, unwired in production today.
|
|
1018
|
+
*/
|
|
1019
|
+
private async queryClusterForLatest(peerIds: string[], blockId: BlockId, context?: ActionContext): Promise<ClusterLatestQuery> {
|
|
1020
|
+
// Query peers in parallel for their latest revision. Each query is DEADLINED (rejects), not
|
|
1021
|
+
// raced-to-undefined: a peer that blows the deadline lands in the silent set below exactly
|
|
1022
|
+
// like a dial failure, because a slow peer and a peer claiming "I hold nothing" must produce
|
|
1023
|
+
// different answers (ticket cluster-read-consult-cannot-report-unreachable).
|
|
1024
|
+
// NOTE: LATEST_QUERY_TIMEOUT_MS is a LAN-shaped budget. A cohort whose round trip honestly
|
|
1025
|
+
// exceeds it now reads as permanently silent, which is safe (the read is flagged, not
|
|
1026
|
+
// mis-reported) but makes every miss cost a transactor-level retry. If a WAN deployment shows
|
|
1027
|
+
// steady `cluster-fetch:peers-silent` against healthy peers, raise this rather than softening
|
|
1028
|
+
// the deadline back into an absent claim.
|
|
1029
|
+
const latestResults = await Promise.allSettled(
|
|
1030
|
+
peerIds.map(async peerIdStr => {
|
|
1031
|
+
const peerId = peerIdFromString(peerIdStr);
|
|
1032
|
+
return await withDeadline(
|
|
1033
|
+
this.clusterLatestCallback!(peerId, blockId, context),
|
|
1034
|
+
LATEST_QUERY_TIMEOUT_MS,
|
|
1035
|
+
`latest query to ${peerIdStr}`
|
|
1036
|
+
);
|
|
1037
|
+
})
|
|
1038
|
+
);
|
|
1039
|
+
|
|
1040
|
+
// NOTE: self-exclusion is keyed on `localPeerId`, which is optional for the single-node/test
|
|
1041
|
+
// construction this class has always tolerated. Left unset, this node's own answer is counted
|
|
1042
|
+
// as a peer claim again. Harmless today — the self answer can only ever corroborate the
|
|
1043
|
+
// revision already held, so the pass declines as `local-current` — but if a future caller can
|
|
1044
|
+
// make self report something the reader does not hold, make `localPeerId` required instead.
|
|
1045
|
+
// The same unset-`localPeerId` tolerance also lets self count toward `answered` below, and
|
|
1046
|
+
// lets a self read that REJECTS land in `silent`: a solo repo whose own storage throws then
|
|
1047
|
+
// reads as `answered === 0` and reports isolation ('cohort-unreachable') rather than a local
|
|
1048
|
+
// fault. Same fix if it ever matters — require `localPeerId`.
|
|
1049
|
+
const selfId = this.localPeerId?.toString();
|
|
1050
|
+
let local: CertifiedActionRev | undefined;
|
|
1051
|
+
const claims: RevClaim[] = [];
|
|
1052
|
+
const silent: string[] = [];
|
|
1053
|
+
// `allSettled` preserves input order, so results correlate to `peerIds` by index — a
|
|
1054
|
+
// rejected entry carries no payload of its own, and its peer id is what `silent` records.
|
|
1055
|
+
for (let i = 0; i < latestResults.length; i++) {
|
|
1056
|
+
const result = latestResults[i]!;
|
|
1057
|
+
const peerIdStr = peerIds[i]!;
|
|
1058
|
+
if (result.status !== 'fulfilled') {
|
|
1059
|
+
// Silence: the callback rejected or the deadline expired. Never a claim. Self is
|
|
1060
|
+
// excluded — its short-circuit reads local storage, and a local read error is not a
|
|
1061
|
+
// cohort peer being unreachable.
|
|
1062
|
+
if (peerIdStr !== selfId) silent.push(peerIdStr);
|
|
1063
|
+
continue;
|
|
1064
|
+
}
|
|
1065
|
+
const value = result.value;
|
|
1066
|
+
if (peerIdStr === selfId) {
|
|
1067
|
+
local = value;
|
|
1068
|
+
continue;
|
|
1069
|
+
}
|
|
1070
|
+
if (!value) continue; // responded, holds nothing — an absent claim, not silence
|
|
1071
|
+
// The proof rides along here and is verified BELOW (certifyClaim) before selection reads
|
|
1072
|
+
// the claim set: presence proves nothing — the peer chose what to attach — but a proof
|
|
1073
|
+
// that verifies certifies the claim, and a certified claim needs no second voter.
|
|
1074
|
+
claims.push({
|
|
1075
|
+
peerId: peerIdStr, rev: value.rev, actionId: value.actionId,
|
|
1076
|
+
...(value.proof ? { proof: value.proof } : {})
|
|
1077
|
+
});
|
|
1078
|
+
}
|
|
1079
|
+
if (silent.length > 0) {
|
|
1080
|
+
this.log('cluster-fetch:peers-silent', { blockId, silent: silent.length, consulted: peerIds.length });
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
// Verify every attached proof, in parallel, BEFORE selection — and penalize provable proof
|
|
1084
|
+
// misbehavior HERE, at verification time, independent of what selection later does with the
|
|
1085
|
+
// claim. Only attributable failures (isAttributableProofFailure) are penalized: a failure
|
|
1086
|
+
// whose signer identities were never proven — unknown/non-ed25519 signer, malformed
|
|
1087
|
+
// signature or proof, a legacy record, the oversized-cohort cap — could have been authored
|
|
1088
|
+
// by anyone in the chain, and penalizing on it would let an attacker frame a peer (the same
|
|
1089
|
+
// discipline as VerifyOutcome.penalize in cluster-repo.ts). A claim whose proof fails stays
|
|
1090
|
+
// in the claim set UNCERTIFIED: it still corroborates by distinct-peer count exactly as a
|
|
1091
|
+
// proof-less claim does — a peer that could fabricate a bad proof could equally have sent no
|
|
1092
|
+
// proof, so dropping the vote would buy nothing.
|
|
1093
|
+
// NOTE: cost is one verification pass per proof-carrying answer per consult, each bounded by
|
|
1094
|
+
// MAX_PROOF_SIGNERS (256) signature checks. In `lazy` mode consults are rate-limited by the
|
|
1095
|
+
// read-repair window; in `paranoid` mode every read of every block pays cohort-width
|
|
1096
|
+
// verifications. Fine at deployment cohort sizes (~10) — if paranoid readers ever show CPU
|
|
1097
|
+
// time in `certifyClaim`, cache verdicts per (blockId, rev, actionId, proof hash) rather than
|
|
1098
|
+
// skipping verification.
|
|
1099
|
+
await Promise.all(claims.map(async claim => {
|
|
1100
|
+
if (!claim.proof) return;
|
|
1101
|
+
const verdict = await certifyClaim(
|
|
1102
|
+
claim.proof,
|
|
1103
|
+
{ blockId, rev: claim.rev, actionId: claim.actionId },
|
|
1104
|
+
// Shared with the reconcile path, so the two cannot drift on what the members actually
|
|
1105
|
+
// enforced — see `proofThresholds` for why the simple-majority term is not
|
|
1106
|
+
// this.simpleMajorityThreshold.
|
|
1107
|
+
proofThresholds(this.superMajorityThreshold),
|
|
1108
|
+
this.proofAnchoring
|
|
1109
|
+
);
|
|
1110
|
+
if (verdict.certified) {
|
|
1111
|
+
claim.certified = true;
|
|
1112
|
+
// The verdict's signer count, never proof.peerIds read here — selection weighs a
|
|
1113
|
+
// single-signer certification below multi-peer corroboration (quorum-restore.ts).
|
|
1114
|
+
claim.certifiedSignerCount = verdict.signerCount;
|
|
1115
|
+
return;
|
|
1116
|
+
}
|
|
1117
|
+
this.log('cluster-fetch:proof-uncertified', {
|
|
1118
|
+
blockId, peerId: claim.peerId, rev: claim.rev, failure: verdict.failure
|
|
1119
|
+
});
|
|
1120
|
+
if (isAttributableProofFailure(verdict.failure)) {
|
|
1121
|
+
this.penalizeProofService(claim.peerId, blockId);
|
|
1122
|
+
}
|
|
1123
|
+
}));
|
|
1124
|
+
|
|
1125
|
+
const nonSelfCount = peerIds.filter(id => id !== selfId).length;
|
|
1126
|
+
const answered = nonSelfCount - silent.length;
|
|
1127
|
+
const capacity = corroboratorCapacity(nonSelfCount, this.repairCorroborationClusterSize);
|
|
1128
|
+
const required = quorumSize(claims.length, this.simpleMajorityThreshold, capacity);
|
|
1129
|
+
const selected = selectQuorumRev(claims, this.simpleMajorityThreshold, capacity);
|
|
1130
|
+
if (!selected) {
|
|
1131
|
+
// A decline can be the certified path REFUSING to pick a side: two distinct actions each
|
|
1132
|
+
// carrying a verified cohort proof for the same top revision. Name that apart from the
|
|
1133
|
+
// routine no-quorum — the cohort (or whoever holds its keys) provably signed both sides,
|
|
1134
|
+
// an incident rather than a shortage of answers. Neither claimant is penalized: both
|
|
1135
|
+
// proofs verified, so which side is "wrong" is exactly what this node cannot know.
|
|
1136
|
+
const equivocation = certifiedEquivocation(claims);
|
|
1137
|
+
if (equivocation) {
|
|
1138
|
+
this.log('cluster-fetch:certified-equivocation', {
|
|
1139
|
+
blockId, rev: equivocation.rev, actionIds: equivocation.actionIds
|
|
1140
|
+
});
|
|
1141
|
+
}
|
|
1142
|
+
// The three populations are reported SEPARATELY, never rolled into one "responders" count:
|
|
1143
|
+
// "1 of 2 responded" and "1 holder, 1 confirmed non-holder, 0 silent" call for completely
|
|
1144
|
+
// different operator actions — the first says wait or fix reachability, the second says the
|
|
1145
|
+
// block has only one copy and no amount of waiting produces a second.
|
|
1146
|
+
this.log('cluster-fetch:no-quorum', {
|
|
1147
|
+
blockId,
|
|
1148
|
+
cohortPeers: nonSelfCount,
|
|
1149
|
+
holders: claims.length,
|
|
1150
|
+
absent: answered - claims.length,
|
|
1151
|
+
silent: silent.length,
|
|
1152
|
+
required,
|
|
1153
|
+
repairCorroborationClusterSize: this.repairCorroborationClusterSize
|
|
1154
|
+
});
|
|
1155
|
+
// ...and, when this decline is provably permanent rather than transient, say THAT once,
|
|
1156
|
+
// in words. The `no-quorum` line above fires on every pass and cannot tell the two apart.
|
|
1157
|
+
this.reportRepairDeadlock({
|
|
1158
|
+
blockId, claims, silentCount: silent.length, cohortPeers: nonSelfCount, answered, required, capacity
|
|
1159
|
+
});
|
|
1160
|
+
// The claims themselves must not drive restoration — but their existence is
|
|
1161
|
+
// evidence the caller needs: an answer served below the highest claim cannot be
|
|
1162
|
+
// confirmed current (see ClusterLatestQuery.uncorroboratedRev).
|
|
1163
|
+
// NOTE: ONE claim is enough to raise that doubt, and the claims reaching this branch
|
|
1164
|
+
// are unverified assertions — a certified claim converges above instead of declining
|
|
1165
|
+
// (the only certified shape that lands here is the equivocation decline). So a single
|
|
1166
|
+
// lying cohort peer can deny unpinned reads of a block by claiming a revision nobody
|
|
1167
|
+
// else holds — an availability lever it did not have while uncorroborated claims were
|
|
1168
|
+
// discarded. Deliberate for now: the alternative is the silent stale serve this marker
|
|
1169
|
+
// exists to end, and the same liar can already force a silent-treated absence by
|
|
1170
|
+
// staying quiet. If the lever is ever exercised, gate the stamp on a certified claim
|
|
1171
|
+
// (the verification machinery now exists) rather than on the bare assertion — at the
|
|
1172
|
+
// cost of re-opening the stale-serve window for the proof-less honest majority.
|
|
1173
|
+
const uncorroboratedRev = claims.length > 0 ? Math.max(...claims.map(c => c.rev)) : undefined;
|
|
1174
|
+
return { local, silent, answered, ...(uncorroboratedRev !== undefined ? { uncorroboratedRev } : {}) };
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
if (selected.certified) {
|
|
1178
|
+
// Which rule won matters when reading a repair log: a certified selection may rest on a
|
|
1179
|
+
// SINGLE claimant whose corroboration is the cohort's signature set, not other voters.
|
|
1180
|
+
this.log('cluster-fetch:certified-selected', {
|
|
1181
|
+
blockId, rev: selected.rev, claimants: selected.supporters.length
|
|
1182
|
+
});
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
// Best-effort: penalize peers whose claim contradicts a CORROBORATED selection — a different
|
|
1186
|
+
// action at the very same revision. A higher rev may be honest leadership and a lower rev is
|
|
1187
|
+
// just lag; neither is penalized, nor is anything contradicting a certified-only selection
|
|
1188
|
+
// (an unanchored proof must not be able to convict the honest cohort). Never let this throw.
|
|
1189
|
+
this.penalizeContradictingRevClaims(claims, selected, blockId);
|
|
1190
|
+
|
|
1191
|
+
return { corroborated: { actionId: selected.actionId, rev: selected.rev }, local, silent, answered };
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
/**
|
|
1195
|
+
* Say ONCE per block, in words, when a corroboration decline is provably PERMANENT rather than a
|
|
1196
|
+
* transient shortage of answers. There are exactly TWO permanent shapes, and they send the operator
|
|
1197
|
+
* to different places, so each gets its own `reason` and its own wording:
|
|
1198
|
+
*
|
|
1199
|
+
* - `cohort-too-small` — this node's cohort has fewer peers than the quorum would demand even if
|
|
1200
|
+
* every one of them answered and agreed. The remedy is machines or an honest declared size.
|
|
1201
|
+
* - `sole-holder` — the cohort is big enough, but exactly ONE of its peers holds the block at all
|
|
1202
|
+
* and every other peer answered that it holds nothing. The remedy is another cohort peer
|
|
1203
|
+
* holding the block; machines and configuration are both irrelevant. Note the scope: this node's
|
|
1204
|
+
* own copy is excluded from the claim set, so a reader that holds the block itself still sees
|
|
1205
|
+
* `sole-holder` — the message says "cohort peer", never "machine in the deployment".
|
|
1206
|
+
*
|
|
1207
|
+
* **What makes `cohort-too-small` provable.** Not "this pass fell short" — a pass falls short
|
|
1208
|
+
* whenever some peer simply does not hold the block *yet*. The decisive question is whether the
|
|
1209
|
+
* cohort could supply the quorum AT ALL: ask what would be required if every cohort peer answered
|
|
1210
|
+
* and agreed — the best case any later pass can reach without new machines — and compare it to how
|
|
1211
|
+
* many peers the cohort has. Short of that best case the shortfall is not the machine count, and
|
|
1212
|
+
* saying PERMANENT would send the operator to change a number that was never the problem. Twelve
|
|
1213
|
+
* days of log archaeology went into re-deriving the real condition from a thousand identical
|
|
1214
|
+
* `cluster-fetch:no-quorum` lines; the node knows it at the moment of each decline.
|
|
1215
|
+
*
|
|
1216
|
+
* **What makes `sole-holder` provable.** Note it is only reachable for a lone UNCERTIFIED holder:
|
|
1217
|
+
* a lone holder whose cohort commit proof verified is selected by the certified path and converges
|
|
1218
|
+
* before any decline — so the wording's "a lone holder cannot second itself" stays accurate for
|
|
1219
|
+
* every claim that gets here. "That peer will hold it later" is an assumption, and for a
|
|
1220
|
+
* peer that ANSWERED "I hold nothing" it is false: the only two mechanisms that would turn a
|
|
1221
|
+
* non-holder into a holder — `queryClusterForLatest` (read-repair) and `createReconcileBlock`
|
|
1222
|
+
* (reconcile) — consume this very decision, so they decline for exactly the same reason on that
|
|
1223
|
+
* peer. Every peer answered, one holds the block, the rest hold nothing, and no later pass changes
|
|
1224
|
+
* any of that. What DOES change it is a new copy: a commit that writes the block again pushes it to
|
|
1225
|
+
* the current cohort. (Sibling work `replicate-owned-blocks-when-the-cohort-grows` makes that
|
|
1226
|
+
* automatic; until it lands the operator has to cause the write.)
|
|
1227
|
+
*
|
|
1228
|
+
* **What is deliberately NOT reported.** A cohort that answers unanimously "I hold nothing" — an
|
|
1229
|
+
* agreed absence is an answer, not a failed repair. A pass with any silent peer: silence cannot
|
|
1230
|
+
* change the arithmetic (`cohortPeers` counts silent peers too), but it does mean this node saw less
|
|
1231
|
+
* than the whole picture, and the next clean pass says the same thing at no cost. Note there is
|
|
1232
|
+
* deliberately NO "the claims disagreed" exemption for `cohort-too-small`: a cohort too small to
|
|
1233
|
+
* reach quorum stays too small whether its peers agree or not, so disagreement would suppress a line
|
|
1234
|
+
* that is still true. Two or more disagreeing holders DO suppress `sole-holder`, because that is a
|
|
1235
|
+
* cohort with two copies whose peers have not settled yet — a later pass can settle it.
|
|
1236
|
+
*
|
|
1237
|
+
* **Never a lever.** This only classifies and logs; it never relaxes a floor. Which is also why the
|
|
1238
|
+
* `cohort-too-small` message names *two* readings of the same numbers — a deployment that genuinely
|
|
1239
|
+
* runs this few machines, or a cohort view shrunk below the real deployment by a partition or by an
|
|
1240
|
+
* attacker with routing influence. `corroboratorCapacity` keeps the shrunken view out of the relaxed
|
|
1241
|
+
* branch, but this node cannot tell the two apart from the inside, and an operator sent to fix the
|
|
1242
|
+
* wrong one is the failure this line exists to end.
|
|
1243
|
+
*
|
|
1244
|
+
* NOTE: the reader is still told only "this may be stale" — `BlockPossiblyStaleError` implies a
|
|
1245
|
+
* retry might help, which is wrong advice for a block whose repair is deadlocked as configured.
|
|
1246
|
+
* Carrying this condition into the error needs a new field on `GetBlockResult` plus a change to
|
|
1247
|
+
* that error's documented contract; deliberately out of scope here (see the ticket
|
|
1248
|
+
* `repair-deadlock-is-never-named`, *Not this ticket*).
|
|
1249
|
+
*/
|
|
1250
|
+
private reportRepairDeadlock(pass: {
|
|
1251
|
+
blockId: BlockId;
|
|
1252
|
+
claims: RevClaim[];
|
|
1253
|
+
silentCount: number;
|
|
1254
|
+
/** Cohort peers besides this node, from the cohort view — whether they answered or not. */
|
|
1255
|
+
cohortPeers: number;
|
|
1256
|
+
answered: number;
|
|
1257
|
+
/** The quorum THIS pass demanded, computed from the peers that actually claimed. */
|
|
1258
|
+
required: number;
|
|
1259
|
+
/** `corroboratorCapacity` for this pass — a function of the view and the resolved size, not of who answered. */
|
|
1260
|
+
capacity: number;
|
|
1261
|
+
}): void {
|
|
1262
|
+
const { blockId, claims, silentCount, cohortPeers, answered, required, capacity } = pass;
|
|
1263
|
+
// An incomplete picture proves nothing about the deployment; the next clean pass says it.
|
|
1264
|
+
if (silentCount > 0) return;
|
|
1265
|
+
// Nobody claimed anything: the cohort agrees the block is absent, which is an answer, not a
|
|
1266
|
+
// deadlock.
|
|
1267
|
+
if (claims.length === 0) return;
|
|
1268
|
+
// The decisive test for the first shape. `requiredEvenIfAllAnswered` is the quorum this cohort
|
|
1269
|
+
// would face with every one of its peers answering and agreeing — the best case reachable
|
|
1270
|
+
// without adding machines. A cohort that can meet it is not too small.
|
|
1271
|
+
const requiredEvenIfAllAnswered = quorumSize(cohortPeers, this.simpleMajorityThreshold, capacity);
|
|
1272
|
+
const cohortTooSmall = cohortPeers < requiredEvenIfAllAnswered;
|
|
1273
|
+
// The second shape: exactly one cohort peer holds the block AT ALL, and — since a claim is one
|
|
1274
|
+
// peer's latest, so a single claim is a single distinct (rev, actionId) group with a single
|
|
1275
|
+
// supporter — every other cohort peer answered that it holds nothing. `answered === cohortPeers`
|
|
1276
|
+
// is already implied by the silence guard above; it is stated because the two counts arrive as
|
|
1277
|
+
// independent parameters and "everybody answered" is half of what makes this provable.
|
|
1278
|
+
//
|
|
1279
|
+
// NOTE: there is a narrow window where `sole-holder` is true of the instant but not of the
|
|
1280
|
+
// deployment — a commit that has landed on one cohort member and has not yet been pushed to the
|
|
1281
|
+
// rest presents exactly this shape. Calling it PERMANENT is defensible even there (repair
|
|
1282
|
+
// genuinely cannot converge until the push lands, and the once-per-episode flag clears the
|
|
1283
|
+
// moment the block converges, so the line does not repeat), and widening the window is what the
|
|
1284
|
+
// push path's own threat model decides — see
|
|
1285
|
+
// `tickets/blocked/repair-floor-defends-a-door-the-push-path-leaves-open`. If commit-to-push
|
|
1286
|
+
// latency ever grows enough that operators see `sole-holder` on blocks that heal moments later,
|
|
1287
|
+
// gate the line on the block having been quiet for longer than that latency rather than
|
|
1288
|
+
// softening the wording.
|
|
1289
|
+
const soleHolder = claims.length === 1 && answered === cohortPeers;
|
|
1290
|
+
if (!cohortTooSmall && !soleHolder) return;
|
|
1291
|
+
|
|
1292
|
+
// Both shapes can hold at once (an undeclared two-machine deployment whose single peer holds the
|
|
1293
|
+
// block is both). `cohort-too-small` is reported in preference because its remedy is the one
|
|
1294
|
+
// that actually works there: declaring the real size makes the floor reachable, after which the
|
|
1295
|
+
// lone peer's claim IS adopted — so calling it a sole-holder problem would send the operator
|
|
1296
|
+
// looking for a copy they do not need.
|
|
1297
|
+
const reason: DeadlockReason = cohortTooSmall ? 'cohort-too-small' : 'sole-holder';
|
|
1298
|
+
const state = this.unsettledAheadClaims.get(blockId);
|
|
1299
|
+
const alreadySaid = state?.deadlocksReported ?? [];
|
|
1300
|
+
// Suppressed per REASON, not once outright: an episode that starts as `cohort-too-small` and
|
|
1301
|
+
// becomes `sole-holder` — the operator added the machines that reason asked for, and the block
|
|
1302
|
+
// is still stuck — has a second thing to say, and a silent log there is the failure this line
|
|
1303
|
+
// exists to end. Neither reason repeats within an episode.
|
|
1304
|
+
if (alreadySaid.includes(reason)) return;
|
|
1305
|
+
|
|
1306
|
+
this.log('cluster-fetch:repair-deadlock', {
|
|
1307
|
+
blockId,
|
|
1308
|
+
reason,
|
|
1309
|
+
cohortPeers,
|
|
1310
|
+
answered,
|
|
1311
|
+
claimants: claims.length,
|
|
1312
|
+
required,
|
|
1313
|
+
requiredEvenIfAllAnswered,
|
|
1314
|
+
repairCorroborationClusterSize: this.repairCorroborationClusterSize,
|
|
1315
|
+
message: cohortTooSmall
|
|
1316
|
+
? cohortTooSmallMessage(cohortPeers, claims.length, requiredEvenIfAllAnswered, this.repairCorroborationClusterSize)
|
|
1317
|
+
: soleHolderMessage(cohortPeers)
|
|
1318
|
+
});
|
|
1319
|
+
// Hung off the existing per-block freshness entry rather than a fourth per-block map. The entry
|
|
1320
|
+
// survives `recordAheadClaim` clearing its `rev`, and is dropped wholesale once the block
|
|
1321
|
+
// converges — so each reason is said once per non-convergence episode, not once per pass.
|
|
1322
|
+
// NOTE: per BLOCK, though the condition is a property of the cohort, not of any block — so a node
|
|
1323
|
+
// in this state that reads N distinct blocks emits N lines. Deliberate: the operator wants to
|
|
1324
|
+
// know which blocks are stuck, and N is bounded by blocks actually read (1821 lines for a single
|
|
1325
|
+
// block was the defect). If a deployment in this state ever makes this the noisy line again, add
|
|
1326
|
+
// a node-level once-flag keyed on (cohortPeers, requiredEvenIfAllAnswered) and let the per-block
|
|
1327
|
+
// entry only suppress repeats.
|
|
1328
|
+
this.unsettledAheadClaims.set(blockId, { ...(state ?? {}), deadlocksReported: [...alreadySaid, reason] });
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
/**
|
|
1332
|
+
* Report peers whose reported latest PROVABLY contradicts a CORROBORATED selection: the same
|
|
1333
|
+
* revision under a different actionId. Two actions cannot both be the commit at one revision,
|
|
1334
|
+
* and the pair a quorum of distinct peers agreed on is the one this node can stand behind, so
|
|
1335
|
+
* the disagreeing claimant is wrong. Best-effort.
|
|
1336
|
+
*
|
|
1337
|
+
* A CERTIFIED selection is deliberately excluded — no claim is penalized against it. A passing
|
|
1338
|
+
* proof shows the cohort it names signed the commit, never that those signers are the block's
|
|
1339
|
+
* responsible cohort: anyone holding N keys can mint a proof that verifies (see caller
|
|
1340
|
+
* obligation #1 in `cluster/commit-proof.ts`, and the unwired {@link ProofAnchoring} layer).
|
|
1341
|
+
* Penalizing here would therefore hand one forged proof a lever it must not have — every honest
|
|
1342
|
+
* peer holding the real action at that revision reported for InvalidRestoration (weight 30,
|
|
1343
|
+
* above the deprioritize threshold of 20), on every consult. Losing the selection to the proof
|
|
1344
|
+
* is already the accepted cost of the certified path; deprioritizing the honest cohort on top of
|
|
1345
|
+
* it is not. Revisit when certification is anchored to the block's derived cohort
|
|
1346
|
+
* (`feat-cluster-membership-threshold-cert-anchoring`): a gated proof makes the contradiction
|
|
1347
|
+
* provable again.
|
|
1348
|
+
*
|
|
1349
|
+
* A claim at a HIGHER rev than the selection is deliberately NOT penalized: a peer can honestly
|
|
1350
|
+
* be ahead of the sampled quorum — an in-flight commit it durably stored before the rest of the
|
|
1351
|
+
* cohort, or other honest holders dropped from the sample by the 1s per-peer consult deadline —
|
|
1352
|
+
* and the InvalidRestoration weight (30) sits above the deprioritize threshold (20), so a single
|
|
1353
|
+
* false hit used to deprioritize an honest, up-to-date peer. Declining to RESTORE from the
|
|
1354
|
+
* uncorroborated higher claim already happens in selection; the affirmative penalty on that
|
|
1355
|
+
* ambiguous evidence is what this method no longer applies. Provably-bad proof SERVICE is
|
|
1356
|
+
* penalized at verification time instead (the certifyClaim pass in
|
|
1357
|
+
* {@link queryClusterForLatest}).
|
|
1358
|
+
*/
|
|
1359
|
+
private penalizeContradictingRevClaims(claims: RevClaim[], selected: QuorumRev, blockId: BlockId): void {
|
|
1360
|
+
if (!this.reputation || selected.certified) return;
|
|
1361
|
+
try {
|
|
1362
|
+
for (const c of claims) {
|
|
1363
|
+
// A CERTIFIED disagreeing claim is exempt: its proof verified, so the peer honestly
|
|
1364
|
+
// served a commit that really happened — when multi-peer corroboration outweighs its
|
|
1365
|
+
// single-signer proof at the same rev (quorum-restore.ts), that peer is a partition
|
|
1366
|
+
// casualty on the losing side of a fork, not a liar. Provably-bad proof SERVICE was
|
|
1367
|
+
// already penalized at verification time above.
|
|
1368
|
+
if (c.certified === true) continue;
|
|
1369
|
+
if (c.rev === selected.rev && c.actionId !== selected.actionId) {
|
|
1370
|
+
this.reputation.reportPeer(c.peerId, PenaltyReason.InvalidRestoration, `read-repair:${blockId}`);
|
|
1371
|
+
}
|
|
1372
|
+
}
|
|
1373
|
+
} catch (err) {
|
|
1374
|
+
this.log('cluster-fetch:penalize-error', { blockId, error: (err as Error).message });
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
/**
|
|
1379
|
+
* Best-effort penalty for a peer whose SERVED PROOF provably lies or provably does not cover the
|
|
1380
|
+
* claim it was attached to (see the attributability classification in
|
|
1381
|
+
* `cluster/certified-claims.ts`). Never throws — mirrors
|
|
1382
|
+
* {@link penalizeContradictingRevClaims}.
|
|
1383
|
+
*/
|
|
1384
|
+
private penalizeProofService(peerId: string, blockId: BlockId): void {
|
|
1385
|
+
if (!this.reputation) return;
|
|
1386
|
+
try {
|
|
1387
|
+
this.reputation.reportPeer(peerId, PenaltyReason.InvalidRestoration, `read-repair:${blockId}`);
|
|
1388
|
+
} catch (err) {
|
|
1389
|
+
this.log('cluster-fetch:penalize-error', { blockId, error: (err as Error).message });
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
async pend(request: PendRequest, options?: MessageOptions): Promise<PendResult> {
|
|
1394
|
+
const allBlockIds = blockIdsForTransforms(request.transforms);
|
|
1395
|
+
await this.verifyResponsibility(allBlockIds);
|
|
1396
|
+
const coordinatingBlockIds = options?.coordinatingBlockIds ?? allBlockIds;
|
|
1397
|
+
|
|
1398
|
+
const peerCount = await this.coordinator.getClusterSize(coordinatingBlockIds[0]!);
|
|
1399
|
+
if (peerCount <= 1) {
|
|
1400
|
+
return await this.storageRepo.pend(request, options);
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1403
|
+
const message: RepoMessage = {
|
|
1404
|
+
operations: [{ pend: request }],
|
|
1405
|
+
expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT,
|
|
1406
|
+
coordinatingBlockIds
|
|
1407
|
+
};
|
|
1408
|
+
|
|
1409
|
+
try {
|
|
1410
|
+
const { localExecuted, localPendResult } = await this.coordinator.executeClusterTransaction(coordinatingBlockIds[0]!, message, options);
|
|
1411
|
+
this.log('coordinator-repo:pend-cluster-complete', {
|
|
1412
|
+
actionId: request.actionId,
|
|
1413
|
+
localExecuted,
|
|
1414
|
+
localVerdict: localPendResult === undefined ? 'none'
|
|
1415
|
+
: localPendResult.success ? 'success'
|
|
1416
|
+
: isConflictFailure(localPendResult) ? 'conflict' : 'fault'
|
|
1417
|
+
});
|
|
1418
|
+
// Only call storageRepo if local cluster didn't already execute during consensus
|
|
1419
|
+
if (!localExecuted) {
|
|
1420
|
+
const result = await this.storageRepo.pend(request, options);
|
|
1421
|
+
this.log('coordinator-repo:pend-fallback-result', {
|
|
1422
|
+
actionId: request.actionId,
|
|
1423
|
+
success: result.success,
|
|
1424
|
+
hasMissing: !!(result as any).missing?.length,
|
|
1425
|
+
hasPending: !!(result as any).pending?.length
|
|
1426
|
+
});
|
|
1427
|
+
return result;
|
|
1428
|
+
}
|
|
1429
|
+
// Local cluster already executed during consensus — return storage's own verdict rather
|
|
1430
|
+
// than fabricating a success (the peerCount <= 1 path above returns storage's real result
|
|
1431
|
+
// verbatim; the cluster path must never answer differently). Pend-consensus confers no
|
|
1432
|
+
// durability: a refusal carrying `pending` (a rival's unresolved action holds the blocks)
|
|
1433
|
+
// or `missing` (the requested revision is already committed) is the optimistic-concurrency
|
|
1434
|
+
// verdict — the same scan every member runs, not a local fault — and must reach the writer
|
|
1435
|
+
// as a retryable conflict so NetworkTransactor.pendPhase cancels the partial pend and the
|
|
1436
|
+
// writer rebases. This deliberately differs from `commit`'s divergence split below: a
|
|
1437
|
+
// commit that reached commit-consensus IS the authoritative commit (Theorem 9), whereas a
|
|
1438
|
+
// pend that reached pend-consensus may still have been stored by nobody.
|
|
1439
|
+
if (localPendResult !== undefined) {
|
|
1440
|
+
if (localPendResult.success || isConflictFailure(localPendResult)) {
|
|
1441
|
+
return localPendResult;
|
|
1442
|
+
}
|
|
1443
|
+
// A bare-reason refusal (no pending/missing — e.g. a local validation-hook fault)
|
|
1444
|
+
// stays tolerated local divergence: consensus is authoritative and the pend may well
|
|
1445
|
+
// have landed on the rest of the cohort.
|
|
1446
|
+
this.log('coordinator-repo:pend-local-fault-tolerated', {
|
|
1447
|
+
actionId: request.actionId,
|
|
1448
|
+
reason: localPendResult.reason
|
|
1449
|
+
});
|
|
1450
|
+
}
|
|
1451
|
+
// No verdict retained (member predates retention, restart, or TTL): the prior shape.
|
|
1452
|
+
return {
|
|
1453
|
+
success: true,
|
|
1454
|
+
pending: [],
|
|
1455
|
+
blockIds: allBlockIds
|
|
1456
|
+
};
|
|
1457
|
+
} catch (error) {
|
|
1458
|
+
this.log('coordinator-repo:pend-error', { actionId: request.actionId, error: (error as Error).message });
|
|
1459
|
+
// A lost conflict race is an optimistic-concurrency loss, not a fault: surface it as the
|
|
1460
|
+
// StaleFailure shape the retry machinery already understands (`Collection.sync` and the
|
|
1461
|
+
// multi-collection pendPhase retry it via `isConflictFailure`), exactly as a confirmed
|
|
1462
|
+
// stale revision is. `staleAt` stays absent deliberately — it is confirmed-only, and a
|
|
1463
|
+
// lost race is a rival *pend* holding the blocks, not a revision claim.
|
|
1464
|
+
//
|
|
1465
|
+
// NOTE: `error.conflicts` (peerId → winning messageHash) is dropped here — `StaleFailure`
|
|
1466
|
+
// has no field for it and the retry loop only needs "retryable". If a caller ever needs to
|
|
1467
|
+
// know WHICH transaction won (e.g. to wait on it rather than re-race it), add a typed field
|
|
1468
|
+
// for it; never recover it by parsing `reason`.
|
|
1469
|
+
//
|
|
1470
|
+
// NOTE: with three or more contenders the members can split so that EVERY contender is
|
|
1471
|
+
// told it lost the race — an all-lose round where nobody wins and each writer retries.
|
|
1472
|
+
// The cause is `ClusterMember.resolveRace`'s approvals-first rule, not its tie-break:
|
|
1473
|
+
// each member compares the rivals as IT holds them, so a member that already approved X
|
|
1474
|
+
// keeps X while a member that approved Y first keeps Y, and no rival reaches a promise
|
|
1475
|
+
// supermajority. (The hash tie-break is already symmetric — it cannot be the fix.)
|
|
1476
|
+
// Fine as it stands: since the torn-action fixes landed, an all-lose round costs one
|
|
1477
|
+
// retry cycle rather than wedging, and the contenders are separated next round by the
|
|
1478
|
+
// jittered backoff plus the aged retry priority carried on the re-pend
|
|
1479
|
+
// (`clampPriority(consecutiveFailures)` in `Collection.syncInternal`), which out-ranks
|
|
1480
|
+
// fresh priority-0 rivals at EQUAL approval counts — priority sits below the approval
|
|
1481
|
+
// count in `resolveRace`, so it does not displace a more-progressed rival. If a
|
|
1482
|
+
// high-contention workload ever shows syncs exhausting `maxAttempts` on repeated
|
|
1483
|
+
// all-lose rounds, the fix is reserve/defer at pend time (backlog
|
|
1484
|
+
// `feat-occ-priority-reservation`, which `resolveRace`'s own residual-fairness NOTE
|
|
1485
|
+
// already points at) rather than raising maxAttempts.
|
|
1486
|
+
if (error instanceof ConflictRaceLostError) {
|
|
1487
|
+
return { success: false, conflict: true, reason: error.message };
|
|
1488
|
+
}
|
|
1489
|
+
const stale = await this.classifyStaleRejection(error, request, allBlockIds)
|
|
1490
|
+
?? await this.classifyPendingConflictRejection(error, request, allBlockIds);
|
|
1491
|
+
if (stale) return stale;
|
|
1492
|
+
throw error;
|
|
1493
|
+
}
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
/**
|
|
1497
|
+
* Decide whether a cluster validator rejection was an optimistic-concurrency loss — the block
|
|
1498
|
+
* already advanced past the requested revision — rather than a genuine validation fault.
|
|
1499
|
+
* A confirmed loss returns a {@link StaleFailure} carrying `conflict: true` so the caller
|
|
1500
|
+
* receives a non-success *response* that says plainly it is a lost race: network-transactor's
|
|
1501
|
+
* pend then takes its stale branch and both writers (`Collection.sync`, and the coordinator's
|
|
1502
|
+
* multi-collection pendPhase via `isConflictFailure`) retry, instead of a thrown error escaping
|
|
1503
|
+
* mid-batch (which splits multi-tree commits — see PartialCommitError).
|
|
1504
|
+
*
|
|
1505
|
+
* The failure carries no `missing` list: confirmation is a local re-read that reveals the
|
|
1506
|
+
* revision is taken but not which actions took it, and no consumer rebases from `missing`
|
|
1507
|
+
* anyway (it is only counted or logged). `conflict` conveys retryability directly instead.
|
|
1508
|
+
*
|
|
1509
|
+
* Confirmation is purely local: re-read the affected blocks from our own storage and require
|
|
1510
|
+
* `latest.rev >= request.rev`. The signed reject-reason text is never consulted — it is
|
|
1511
|
+
* free-form wire-visible prose and must not become control flow. Anything unconfirmed
|
|
1512
|
+
* (including read errors during confirmation) stays a throw, preserving fail-fast for
|
|
1513
|
+
* genuine validation faults.
|
|
1514
|
+
*/
|
|
1515
|
+
private async classifyStaleRejection(error: unknown, request: PendRequest, blockIds: BlockId[]): Promise<StaleFailure | undefined> {
|
|
1516
|
+
const requestedRev = request.rev;
|
|
1517
|
+
if (!(error instanceof ValidatorRejectionError) || requestedRev === undefined) return undefined;
|
|
1518
|
+
let results: GetBlockResults;
|
|
1519
|
+
try {
|
|
1520
|
+
results = await this.storageRepo.get({ blockIds });
|
|
1521
|
+
} catch (readError) {
|
|
1522
|
+
this.log('coordinator-repo:pend-stale-classify-read-error', {
|
|
1523
|
+
actionId: request.actionId,
|
|
1524
|
+
error: (readError as Error).message
|
|
1525
|
+
});
|
|
1526
|
+
return undefined;
|
|
1527
|
+
}
|
|
1528
|
+
// Scan EVERY block rather than stopping at the first confirmation: several of the request's
|
|
1529
|
+
// blocks can be past the requested revision at different revisions, and it is the highest
|
|
1530
|
+
// that the loser's next request has to clear (see `highestStaleAt`). Both the reported
|
|
1531
|
+
// number and the reason prose name that block, so they never disagree.
|
|
1532
|
+
const staleAt = highestStaleAt(blockIds.map(blockId => {
|
|
1533
|
+
const latest = results[blockId]?.state.latest;
|
|
1534
|
+
if (!latest || latest.rev < requestedRev) return undefined;
|
|
1535
|
+
// Per-block self-exclusion (see {@link isOwnRevision}): our own durable half of a torn
|
|
1536
|
+
// action is not a confirmed loss. Deliberately per-block, NOT the bail-entirely
|
|
1537
|
+
// 'own-durable' shape of confirmCommitRivalAgainstLocal — a confirmed rival on ANOTHER
|
|
1538
|
+
// block still confirms, and when none is confirmed anywhere the rejection stays a throw
|
|
1539
|
+
// exactly as before. With the two pend-tier sites upstream fixed (StorageRepo.pend and
|
|
1540
|
+
// ClusterMember.validatePendOperations) this shape should not reach here; mirrored so
|
|
1541
|
+
// all three pend-tier checks agree.
|
|
1542
|
+
if (isOwnRevision(latest, requestedRev, request.actionId)) return undefined;
|
|
1543
|
+
return { blockId, rev: latest.rev };
|
|
1544
|
+
}));
|
|
1545
|
+
if (staleAt) {
|
|
1546
|
+
this.log('coordinator-repo:pend-stale-classified', {
|
|
1547
|
+
actionId: request.actionId,
|
|
1548
|
+
blockId: staleAt.blockId,
|
|
1549
|
+
latestRev: staleAt.rev,
|
|
1550
|
+
requestedRev
|
|
1551
|
+
});
|
|
1552
|
+
return {
|
|
1553
|
+
success: false,
|
|
1554
|
+
conflict: true,
|
|
1555
|
+
reason: `stale revision: block ${staleAt.blockId} at rev ${staleAt.rev}, requested rev ${requestedRev}`,
|
|
1556
|
+
// The same fact as the reason prose, but as data. This is the ONLY place a losing
|
|
1557
|
+
// writer can learn the revision it lost to, since this failure deliberately carries
|
|
1558
|
+
// no `missing`. Confirmed-local: read out of our own storage just above.
|
|
1559
|
+
staleAt
|
|
1560
|
+
};
|
|
1561
|
+
}
|
|
1562
|
+
// NOTE: conservative — when only remote members saw the newer revision (local storage still
|
|
1563
|
+
// behind), staleness can't be confirmed locally and the rejection stays a throw. If that
|
|
1564
|
+
// shows up in practice, extend confirmation with a quorum read; never trust the reject text.
|
|
1565
|
+
// `staleAt` is absent on this path for the same reason, and deliberately so — there is no
|
|
1566
|
+
// confirmed number to report, and the field's contract forbids inferring one from that text.
|
|
1567
|
+
return undefined;
|
|
1568
|
+
}
|
|
1569
|
+
|
|
1570
|
+
/**
|
|
1571
|
+
* Sibling of {@link classifyStaleRejection} for the OTHER optimistic-concurrency refusal shape:
|
|
1572
|
+
* the promise-phase pending-conflict vote (`validatePendOperations` rejecting a pend whose
|
|
1573
|
+
* blocks are held by a different unresolved pending action). That vote surfaces here as a
|
|
1574
|
+
* {@link ValidatorRejectionError}, and without classification it would escape as a throw —
|
|
1575
|
+
* splitting multi-tree pends mid-batch instead of taking the retry path a lost race deserves.
|
|
1576
|
+
*
|
|
1577
|
+
* Same confirmation discipline as the stale classifier: purely local. Re-read the affected
|
|
1578
|
+
* blocks from our own storage and require some block's `state.pendings` to carry a rival
|
|
1579
|
+
* actionId; the signed reject text is never consulted. A confirmed rival returns a
|
|
1580
|
+
* {@link StaleFailure} with `conflict: true` and the rivals as `pending` (`ActionPending`
|
|
1581
|
+
* without `transform` — the type allows it, and no consumer rebases from it). Unconfirmed —
|
|
1582
|
+
* including read errors during confirmation — stays a throw, preserving fail-fast for genuine
|
|
1583
|
+
* validation faults. Checked after `classifyStaleRejection` so a confirmed committed loss
|
|
1584
|
+
* (which carries the sharper `staleAt`) wins when both hold.
|
|
1585
|
+
*/
|
|
1586
|
+
private async classifyPendingConflictRejection(error: unknown, request: PendRequest, blockIds: BlockId[]): Promise<StaleFailure | undefined> {
|
|
1587
|
+
if (!(error instanceof ValidatorRejectionError)) return undefined;
|
|
1588
|
+
let results: GetBlockResults;
|
|
1589
|
+
try {
|
|
1590
|
+
results = await this.storageRepo.get({ blockIds });
|
|
1591
|
+
} catch (readError) {
|
|
1592
|
+
this.log('coordinator-repo:pend-conflict-classify-read-error', {
|
|
1593
|
+
actionId: request.actionId,
|
|
1594
|
+
error: (readError as Error).message
|
|
1595
|
+
});
|
|
1596
|
+
return undefined;
|
|
1597
|
+
}
|
|
1598
|
+
const pending: ActionPending[] = [];
|
|
1599
|
+
for (const blockId of blockIds) {
|
|
1600
|
+
for (const actionId of results[blockId]?.state?.pendings ?? []) {
|
|
1601
|
+
if (actionId !== request.actionId) pending.push({ blockId, actionId });
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
if (pending.length === 0) return undefined;
|
|
1605
|
+
this.log('coordinator-repo:pend-conflict-classified', {
|
|
1606
|
+
actionId: request.actionId,
|
|
1607
|
+
rivals: pending.map(p => `${p.blockId}:${p.actionId}`)
|
|
1608
|
+
});
|
|
1609
|
+
return {
|
|
1610
|
+
success: false,
|
|
1611
|
+
conflict: true,
|
|
1612
|
+
pending,
|
|
1613
|
+
reason: `pending conflict: block(s) held by unresolved rival action(s) ${[...new Set(pending.map(p => p.actionId))].join(', ')}`
|
|
1614
|
+
};
|
|
1615
|
+
}
|
|
1616
|
+
|
|
1617
|
+
async cancel(actionRef: ActionBlocks, options?: MessageOptions): Promise<void> {
|
|
1618
|
+
const blockIds = actionRef.blockIds;
|
|
1619
|
+
await this.verifyResponsibility(blockIds);
|
|
1620
|
+
|
|
1621
|
+
// Create a message for this cancel operation with timeout
|
|
1622
|
+
const message: RepoMessage = {
|
|
1623
|
+
operations: [{ cancel: { actionRef } }],
|
|
1624
|
+
expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
|
|
1625
|
+
};
|
|
1626
|
+
|
|
1627
|
+
try {
|
|
1628
|
+
// One cluster transaction per block ID — but a block whose cohort is just this node
|
|
1629
|
+
// short-circuits to local storage, exactly as `pend` and `commit` do above. Without the
|
|
1630
|
+
// short-circuit a solo cohort enters `executeTransaction`, fails `minAbsoluteClusterSize`
|
|
1631
|
+
// (2), and throws `Cluster size 1 below minimum 2 and not validated` — so a single-peer
|
|
1632
|
+
// deployment could pend and commit but never cancel, unless the operator had opened the
|
|
1633
|
+
// `allowUnvalidatedSmallCluster` hatch. Decided per block rather than once for
|
|
1634
|
+
// `blockIds[0]`, because a multi-block cancel can span cohorts of different sizes.
|
|
1635
|
+
//
|
|
1636
|
+
// NOTE: `getClusterSize` is a second `findCluster` for the same key that
|
|
1637
|
+
// `executeClusterTransaction` is about to look up again, so a cancel over N blocks now
|
|
1638
|
+
// costs 2N cohort lookups instead of N. Same shape `pend` and `commit` already pay, but
|
|
1639
|
+
// they pay it once (they only ever consult `blockIds[0]`) where this scales with N. Fine
|
|
1640
|
+
// while cancels span a handful of blocks; if wide multi-block cancels ever show up hot,
|
|
1641
|
+
// have `executeClusterTransaction` return the cohort it already fetched (or own the
|
|
1642
|
+
// short-circuit itself) rather than adding a cache here.
|
|
1643
|
+
const results = await Promise.all(blockIds.map(async blockId => {
|
|
1644
|
+
const peerCount = await this.coordinator.getClusterSize(blockId);
|
|
1645
|
+
if (peerCount <= 1) return false;
|
|
1646
|
+
const { localExecuted } = await this.coordinator.executeClusterTransaction(blockId, message, options);
|
|
1647
|
+
return localExecuted;
|
|
1648
|
+
}));
|
|
1649
|
+
|
|
1650
|
+
// Only call storageRepo if local cluster didn't already execute during consensus
|
|
1651
|
+
const anyLocalExecuted = results.some(Boolean);
|
|
1652
|
+
if (!anyLocalExecuted) {
|
|
1653
|
+
await this.storageRepo.cancel(actionRef, options);
|
|
1654
|
+
}
|
|
1655
|
+
} catch (error) {
|
|
1656
|
+
this.log('coordinator-repo:cancel-error', { actionId: actionRef.actionId, error: (error as Error).message });
|
|
1657
|
+
throw error;
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
async commit(request: CommitRequest, options?: MessageOptions): Promise<CommitResult> {
|
|
1662
|
+
const blockIds = request.blockIds;
|
|
1663
|
+
await this.verifyResponsibility(blockIds);
|
|
1664
|
+
|
|
1665
|
+
const cohortPeerIds = await this.coordinator.getClusterPeerIds(blockIds[0]!);
|
|
1666
|
+
const peerCount = cohortPeerIds.length;
|
|
1667
|
+
if (peerCount <= 1) {
|
|
1668
|
+
// Solo cohort: consensus never runs, so no ClusterRecord exists to project a proof from —
|
|
1669
|
+
// the lone member self-signs a one-peer proof instead (mintSoloCommitProof), which is what
|
|
1670
|
+
// lets a block born on a cohort of one ever gain a second holder under the certified-push
|
|
1671
|
+
// default (handlePush refuses a proof-less block). Minted even when peerCount is 0 or the
|
|
1672
|
+
// sole peer is not self — findCluster failing (getClusterPeerIds returns []) puts the
|
|
1673
|
+
// DEGRADED-ROUTING case in this same branch, and self genuinely committed these bytes
|
|
1674
|
+
// either way; a proof's peer list is already not evidence of cohort membership by design
|
|
1675
|
+
// (caller obligation #1 on verifyBlockCommitProofClaim), so gating the mint on cohort
|
|
1676
|
+
// composition would buy no safety while opening a silent no-proof hole exactly when
|
|
1677
|
+
// routing is degraded. The log line is how an operator tells a real cohort of one
|
|
1678
|
+
// (cohortSize 1, soleIsSelf true) from a routing failure (cohortSize 0, or a sole peer
|
|
1679
|
+
// that is not this node).
|
|
1680
|
+
this.log('commit:solo-cohort', {
|
|
1681
|
+
blockId: blockIds[0],
|
|
1682
|
+
cohortSize: peerCount,
|
|
1683
|
+
soleIsSelf: peerCount === 1 && this.localPeerId !== undefined
|
|
1684
|
+
&& cohortPeerIds[0] === this.localPeerId.toString()
|
|
1685
|
+
});
|
|
1686
|
+
// Same message shape the multi-peer path produces — executeClusterTransaction stamps
|
|
1687
|
+
// coordinatingBlockIds at its choke point, so the solo artifact must carry it too or a
|
|
1688
|
+
// solo proof's message is distinguishable from every other proof's.
|
|
1689
|
+
const message: RepoMessage = {
|
|
1690
|
+
operations: [{ commit: request }],
|
|
1691
|
+
coordinatingBlockIds: [blockIds[0]!],
|
|
1692
|
+
expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
|
|
1693
|
+
};
|
|
1694
|
+
// `undefined` when no local cluster is wired (direct constructors, unit-test doubles) —
|
|
1695
|
+
// then the commit lands proof-less, exactly the pre-mint behavior. The cast is the named
|
|
1696
|
+
// ICommitProofPersister contract; a plain IRepo double ignores the extra argument.
|
|
1697
|
+
const proof = await this.localCluster?.mintSoloCommitProof?.(message);
|
|
1698
|
+
const result = await (this.storageRepo as IRepo & ICommitProofPersister).commit(request, options, proof);
|
|
1699
|
+
if (result.success) this.markBlocksSeen(blockIds);
|
|
1700
|
+
return result;
|
|
1701
|
+
}
|
|
1702
|
+
|
|
1703
|
+
const message: RepoMessage = {
|
|
1704
|
+
operations: [{ commit: request }],
|
|
1705
|
+
expiration: options?.expiration ?? Date.now() + this.DEFAULT_TIMEOUT
|
|
1706
|
+
};
|
|
1707
|
+
|
|
1708
|
+
try {
|
|
1709
|
+
const { record, localExecuted, localCommitResult } = await this.coordinator.executeClusterTransaction(blockIds[0]!, message, options);
|
|
1710
|
+
if (localExecuted) {
|
|
1711
|
+
// Our own member applied this commit during consensus. Its retained storage verdict is
|
|
1712
|
+
// the one honest signal we have about durability: the member-side apply tolerates an
|
|
1713
|
+
// "ahead" refusal as divergence (see the NOTE in ClusterMember.applyConsensusOperation),
|
|
1714
|
+
// which is correct for a redelivered or lagging commit — but when the refusal's real
|
|
1715
|
+
// cause is a RIVAL action holding the requested revision, that tolerance turns a commit
|
|
1716
|
+
// no member durably stored into a fabricated success. This is the
|
|
1717
|
+
// signed-but-not-yet-applied window: two commits for one revision can BOTH assemble
|
|
1718
|
+
// consensus when every member signs the second after signing (but before applying) the
|
|
1719
|
+
// first, because signing drops the member's reservation. Confirm the rival against local
|
|
1720
|
+
// storage (never the verdict's prose) and answer the writer with a retryable conflict so
|
|
1721
|
+
// it re-drives at a fresh revision. Own-action or unconfirmed refusals keep the
|
|
1722
|
+
// prior fabricated-success shape: consensus is authoritative and this member converges
|
|
1723
|
+
// via replication.
|
|
1724
|
+
//
|
|
1725
|
+
// NOTE: a CONFIRMED rival is trusted over the consensus outcome here. That is right in
|
|
1726
|
+
// the window this closes (the cohort refused the loser too), but it inverts if the two
|
|
1727
|
+
// ever disagree — a local rival at the requested revision while a super-majority
|
|
1728
|
+
// approved OUR commit means this node is on a forked lineage, and refusing then tells a
|
|
1729
|
+
// writer whose write did land to re-drive it (a duplicate entry). Members holding the
|
|
1730
|
+
// rival reject at the promise round, so consensus and a local rival can only disagree
|
|
1731
|
+
// after a fork; that is partition-healing scope (docs/partition-healing.md). If forks
|
|
1732
|
+
// are ever observed here, weigh the retained verdict against the cohort's votes instead
|
|
1733
|
+
// of trusting the local re-read alone.
|
|
1734
|
+
if (localCommitResult !== undefined && !localCommitResult.success) {
|
|
1735
|
+
const rival = await this.confirmCommitRivalAgainstLocal(request);
|
|
1736
|
+
if (typeof rival === 'object') return rival;
|
|
1737
|
+
this.log('coordinator-repo:commit-local-refusal-tolerated', {
|
|
1738
|
+
actionId: request.actionId,
|
|
1739
|
+
confirmation: rival ?? 'unconfirmed',
|
|
1740
|
+
reason: localCommitResult.reason
|
|
1741
|
+
});
|
|
1742
|
+
}
|
|
1743
|
+
this.markBlocksSeen(blockIds);
|
|
1744
|
+
return { success: true };
|
|
1745
|
+
}
|
|
1746
|
+
// Local cluster didn't execute during consensus. Attempt a local commit, but tolerate
|
|
1747
|
+
// local divergence when the cluster already reached consensus — this coordinator was
|
|
1748
|
+
// likely picked for commit after missing the pend phase (unreachable during pend, fresh
|
|
1749
|
+
// join, etc.). The cluster's majority is authoritative; this peer catches up via sync.
|
|
1750
|
+
//
|
|
1751
|
+
// Divergence reaches us in BOTH shapes and both must be tolerated identically:
|
|
1752
|
+
// - a THROW ("Pending action … not found"), when we never saw the pend;
|
|
1753
|
+
// - a RETURNED `success:false` carrying `missing-base-revision`, when we saw the pend
|
|
1754
|
+
// but not the revision that created the block (see StorageRepo.internalCommit).
|
|
1755
|
+
// Only the throw was tolerated before the refusal existed. Reporting the refusal to the
|
|
1756
|
+
// caller instead would surface a committed transaction as a stale loss: db-core's
|
|
1757
|
+
// commitPhase treats any returned `success:false` as a permanent stale failure, so the
|
|
1758
|
+
// client would retry an action the cluster already landed until it exhausted its budget.
|
|
1759
|
+
//
|
|
1760
|
+
// Deliberately NOT self-signed here (unlike the solo short-circuit above): consensus for
|
|
1761
|
+
// this commit ran on the cohort, so a one-peer minted proof would be a FALSE statement
|
|
1762
|
+
// about the committing cohort. Thread the record's REAL proof instead —
|
|
1763
|
+
// executeClusterTransaction resolves only after the record's promise/commit votes are
|
|
1764
|
+
// populated, so the projection is genuine. Passed unconditionally, threshold or not:
|
|
1765
|
+
// persistProofIfContentMatches retains it under the digest-match rule, and a record that
|
|
1766
|
+
// never reached threshold simply yields a proof no verifier accepts — the same posture
|
|
1767
|
+
// ClusterMember.applyConsensusOperation takes with its own projection.
|
|
1768
|
+
const consensusProof = buildBlockCommitProof(record);
|
|
1769
|
+
try {
|
|
1770
|
+
const result = await (this.storageRepo as IRepo & ICommitProofPersister).commit(request, options, consensusProof);
|
|
1771
|
+
if (result.success) {
|
|
1772
|
+
this.markBlocksSeen(blockIds);
|
|
1773
|
+
return result;
|
|
1774
|
+
}
|
|
1775
|
+
if (isMissingBaseRevisionFailure(result) && clusterReachedCommitConsensus(record)) {
|
|
1776
|
+
return this.tolerateLocalCommitDivergence(request, blockIds, result.reason ?? MISSING_BASE_REVISION_REASON);
|
|
1777
|
+
}
|
|
1778
|
+
return result;
|
|
1779
|
+
} catch (err) {
|
|
1780
|
+
if (clusterReachedCommitConsensus(record)) {
|
|
1781
|
+
return this.tolerateLocalCommitDivergence(request, blockIds, (err as Error).message);
|
|
1782
|
+
}
|
|
1783
|
+
throw err;
|
|
1784
|
+
}
|
|
1785
|
+
} catch (error) {
|
|
1786
|
+
this.log('coordinator-repo:commit-error', { actionId: request.actionId, error: (error as Error).message });
|
|
1787
|
+
// A lost commit-consensus race is an optimistic-concurrency loss, not a fault — mirror
|
|
1788
|
+
// `pend`'s conversion above. At the moment this is thrown, zero members approved and the
|
|
1789
|
+
// members hold the winner: nothing of the loser landed, so a retryable-conflict answer is
|
|
1790
|
+
// truthful. Returning it (rather than rethrowing) matters more here than on the pend path:
|
|
1791
|
+
// db-core's `commitCollection` retries a THROWN commit error verbatim up to 3 times, and by
|
|
1792
|
+
// the retry the members have applied the winner and cleared its reservation — the re-driven
|
|
1793
|
+
// commit can then assemble a consensus no member will durably store (the writer's append
|
|
1794
|
+
// fulfills, the entry exists on no node). A RETURNED `success:false` is instead surfaced
|
|
1795
|
+
// immediately as a stale loss; the writer cancels the pend, re-reads, and re-drives the
|
|
1796
|
+
// whole pend+commit at a fresh revision. `staleAt` stays absent for the same reason as
|
|
1797
|
+
// pend's: it is confirmed-only, and a lost race is a rival commit racing the same revision,
|
|
1798
|
+
// not a locally-confirmed revision claim.
|
|
1799
|
+
if (error instanceof ConflictRaceLostError) {
|
|
1800
|
+
return { success: false, conflict: true, reason: error.message };
|
|
1801
|
+
}
|
|
1802
|
+
// A promise-phase stale-commit reject (`ClusterMember.validateCommitRevisions` — a member
|
|
1803
|
+
// holds the requested revision under a different action) surfaces here as a
|
|
1804
|
+
// ValidatorRejectionError; classify it against local storage the way `pend` does, so the
|
|
1805
|
+
// writer gets a clean retryable conflict instead of three verbatim re-drives and a hard
|
|
1806
|
+
// failure.
|
|
1807
|
+
const stale = await this.classifyCommitStaleRejection(error, request);
|
|
1808
|
+
if (stale) return stale;
|
|
1809
|
+
throw error;
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
/**
|
|
1814
|
+
* Commit-shaped sibling of {@link classifyStaleRejection}: decide whether a cluster validator
|
|
1815
|
+
* rejection of a COMMIT was an optimistic-concurrency loss — the requested revision is already
|
|
1816
|
+
* committed under a different action — rather than a genuine validation fault. A confirmed loss
|
|
1817
|
+
* returns a {@link StaleFailure} with `conflict: true` so db-core's `commitCollection` surfaces
|
|
1818
|
+
* it immediately as a stale loss (no verbatim retry) and the writer re-drives at a fresh
|
|
1819
|
+
* revision.
|
|
1820
|
+
*
|
|
1821
|
+
* Same confirmation discipline as the pend classifiers: purely local re-read; the signed reject
|
|
1822
|
+
* text is never consulted. One commit-specific delta — confirmation must EXCLUDE the
|
|
1823
|
+
* own-action-at-rev case: a block whose requested revision is held by THIS action is already
|
|
1824
|
+
* durable, and answering `conflict` for it would make the writer rebase and re-append an
|
|
1825
|
+
* already-committed action at a new revision — a duplicate entry. So:
|
|
1826
|
+
* - `latest.rev === request.rev` → compare `latest.actionId`: ours ⇒ bail (stays a throw),
|
|
1827
|
+
* a rival's ⇒ confirmed loss;
|
|
1828
|
+
* - `latest.rev > request.rev` → ask the {@link IRevisionActionReader} capability who holds
|
|
1829
|
+
* `request.rev`: ours ⇒ bail, a rival's ⇒ confirmed loss, unknown/absent/fault ⇒ unconfirmed;
|
|
1830
|
+
* - anything unconfirmed (including read errors) stays a throw — fail-fast for genuine faults.
|
|
1831
|
+
*/
|
|
1832
|
+
private async classifyCommitStaleRejection(error: unknown, request: CommitRequest): Promise<StaleFailure | undefined> {
|
|
1833
|
+
if (!(error instanceof ValidatorRejectionError)) return undefined;
|
|
1834
|
+
const rival = await this.confirmCommitRivalAgainstLocal(request);
|
|
1835
|
+
// 'own-durable' and unconfirmed both stay a throw here: fail-fast for genuine faults, and a
|
|
1836
|
+
// commit already durable under this action must never be answered `conflict` (the writer
|
|
1837
|
+
// would rebase and re-append it — a duplicate entry).
|
|
1838
|
+
return typeof rival === 'object' ? rival : undefined;
|
|
1839
|
+
}
|
|
1840
|
+
|
|
1841
|
+
/**
|
|
1842
|
+
* Shared confirmation core for the two commit-tier conversion sites ({@link classifyCommitStaleRejection}
|
|
1843
|
+
* and the locally-executed refusal check in {@link commit}): decide, from LOCAL storage only, who
|
|
1844
|
+
* holds the requested revision.
|
|
1845
|
+
* - a confirmed RIVAL → the {@link StaleFailure} conflict answer (with `staleAt` = highest
|
|
1846
|
+
* confirmed holder);
|
|
1847
|
+
* - our OWN action durable at the requested revision → `'own-durable'` (callers must not answer
|
|
1848
|
+
* `conflict` — the writer would rebase an already-landed action into a duplicate entry);
|
|
1849
|
+
* - anything else (behind, truncated history, read faults, capability absent) → `undefined`,
|
|
1850
|
+
* unconfirmed.
|
|
1851
|
+
* The signed reject text / retained verdict prose is never consulted.
|
|
1852
|
+
*/
|
|
1853
|
+
private async confirmCommitRivalAgainstLocal(request: CommitRequest): Promise<StaleFailure | 'own-durable' | undefined> {
|
|
1854
|
+
const blockIds = request.blockIds;
|
|
1855
|
+
let results: GetBlockResults;
|
|
1856
|
+
try {
|
|
1857
|
+
results = await this.storageRepo.get({ blockIds });
|
|
1858
|
+
} catch (readError) {
|
|
1859
|
+
this.log('coordinator-repo:commit-stale-classify-read-error', {
|
|
1860
|
+
actionId: request.actionId,
|
|
1861
|
+
error: (readError as Error).message
|
|
1862
|
+
});
|
|
1863
|
+
return undefined;
|
|
1864
|
+
}
|
|
1865
|
+
const reader = this.storageRepo as IRepo & Partial<IRevisionActionReader>;
|
|
1866
|
+
// Scan EVERY block (same rule as the pend classifier): report the highest confirmed rival
|
|
1867
|
+
// revision, but bail the moment any block shows OUR action durable at the requested revision.
|
|
1868
|
+
const rivalStales: ({ blockId: BlockId; rev: number } | undefined)[] = [];
|
|
1869
|
+
for (const blockId of blockIds) {
|
|
1870
|
+
const latest = results[blockId]?.state?.latest;
|
|
1871
|
+
if (!latest || latest.rev < request.rev) continue;
|
|
1872
|
+
if (latest.rev === request.rev) {
|
|
1873
|
+
if (latest.actionId === request.actionId) {
|
|
1874
|
+
this.log('coordinator-repo:commit-stale-classify-own-action', {
|
|
1875
|
+
actionId: request.actionId, blockId, rev: request.rev
|
|
1876
|
+
});
|
|
1877
|
+
return 'own-durable';
|
|
1878
|
+
}
|
|
1879
|
+
rivalStales.push({ blockId, rev: latest.rev });
|
|
1880
|
+
continue;
|
|
1881
|
+
}
|
|
1882
|
+
// latest.rev > request.rev — latest can no longer name who took request.rev.
|
|
1883
|
+
if (typeof reader.getRevisionAction !== 'function') continue;
|
|
1884
|
+
let takenBy: ActionId | undefined;
|
|
1885
|
+
try {
|
|
1886
|
+
takenBy = await reader.getRevisionAction(blockId, request.rev);
|
|
1887
|
+
} catch (readError) {
|
|
1888
|
+
this.log('coordinator-repo:commit-stale-classify-revision-read-error', {
|
|
1889
|
+
actionId: request.actionId, blockId, rev: request.rev,
|
|
1890
|
+
error: (readError as Error).message
|
|
1891
|
+
});
|
|
1892
|
+
continue;
|
|
1893
|
+
}
|
|
1894
|
+
if (takenBy === request.actionId) {
|
|
1895
|
+
this.log('coordinator-repo:commit-stale-classify-own-action', {
|
|
1896
|
+
actionId: request.actionId, blockId, rev: request.rev, latestRev: latest.rev
|
|
1897
|
+
});
|
|
1898
|
+
return 'own-durable';
|
|
1899
|
+
}
|
|
1900
|
+
if (takenBy !== undefined) rivalStales.push({ blockId, rev: latest.rev });
|
|
1901
|
+
// takenBy undefined (truncated history): unconfirmed for this block.
|
|
1902
|
+
}
|
|
1903
|
+
const staleAt = highestStaleAt(rivalStales);
|
|
1904
|
+
if (!staleAt) return undefined;
|
|
1905
|
+
this.log('coordinator-repo:commit-stale-classified', {
|
|
1906
|
+
actionId: request.actionId,
|
|
1907
|
+
blockId: staleAt.blockId,
|
|
1908
|
+
latestRev: staleAt.rev,
|
|
1909
|
+
requestedRev: request.rev
|
|
1910
|
+
});
|
|
1911
|
+
return {
|
|
1912
|
+
success: false,
|
|
1913
|
+
conflict: true,
|
|
1914
|
+
reason: `stale commit: block ${staleAt.blockId} at rev ${staleAt.rev}, requested rev ${request.rev}`,
|
|
1915
|
+
staleAt
|
|
1916
|
+
};
|
|
1917
|
+
}
|
|
1918
|
+
|
|
1919
|
+
/**
|
|
1920
|
+
* Report success for a commit the cluster carried but this peer could not apply locally. The
|
|
1921
|
+
* blocks are marked seen so the read path treats them as freshness-checked; convergence comes
|
|
1922
|
+
* from replication (cohort reconcile, or read-driven acquisition), not from replay here.
|
|
1923
|
+
*/
|
|
1924
|
+
private tolerateLocalCommitDivergence(request: CommitRequest, blockIds: BlockId[], detail: string): CommitResult {
|
|
1925
|
+
this.log('coordinator-repo:commit-local-failed-cluster-succeeded', { actionId: request.actionId, error: detail });
|
|
1926
|
+
this.markBlocksSeen(blockIds);
|
|
1927
|
+
return { success: true };
|
|
1928
|
+
}
|
|
1929
|
+
}
|
|
1930
|
+
|
|
1931
|
+
/** True if a simple majority of cluster peers signed an approving commit. */
|
|
1932
|
+
function clusterReachedCommitConsensus(record: ClusterRecord): boolean {
|
|
1933
|
+
const peerCount = Object.keys(record.peers).length;
|
|
1934
|
+
if (peerCount === 0) return false;
|
|
1935
|
+
const approvedCommits = Object.values(record.commits).filter(s => s.type === 'approve').length;
|
|
1936
|
+
return approvedCommits > peerCount / 2;
|
|
1937
|
+
}
|