@peerbit/shared-log 13.2.12 → 13.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark/receive-prune.js +31 -29
- package/dist/benchmark/receive-prune.js.map +1 -1
- package/dist/src/checked-prune.d.ts +43 -10
- package/dist/src/checked-prune.d.ts.map +1 -1
- package/dist/src/checked-prune.js +257 -27
- package/dist/src/checked-prune.js.map +1 -1
- package/dist/src/exchange-heads.d.ts +26 -1
- package/dist/src/exchange-heads.d.ts.map +1 -1
- package/dist/src/exchange-heads.js +114 -36
- package/dist/src/exchange-heads.js.map +1 -1
- package/dist/src/index.d.ts +71 -14
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3492 -1493
- package/dist/src/index.js.map +1 -1
- package/dist/src/sync/index.d.ts +2 -0
- package/dist/src/sync/index.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.d.ts +96 -20
- package/dist/src/sync/rateless-iblt.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.js +1616 -428
- package/dist/src/sync/rateless-iblt.js.map +1 -1
- package/dist/src/sync/simple.d.ts +176 -1
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +2919 -525
- package/dist/src/sync/simple.js.map +1 -1
- package/package.json +11 -11
- package/src/checked-prune.ts +338 -29
- package/src/exchange-heads.ts +74 -37
- package/src/index.ts +6370 -2792
- package/src/sync/index.ts +2 -1
- package/src/sync/rateless-iblt.ts +2036 -498
- package/src/sync/simple.ts +3779 -634
|
@@ -42,6 +42,8 @@ import {
|
|
|
42
42
|
syncProfileStart,
|
|
43
43
|
} from "./profile.js";
|
|
44
44
|
import {
|
|
45
|
+
ResponseMaybeSync,
|
|
46
|
+
ResponseMaybeSyncCapabilities,
|
|
45
47
|
SYNC_MESSAGE_PRIORITY,
|
|
46
48
|
SimpleSyncronizer,
|
|
47
49
|
} from "./simple.js";
|
|
@@ -79,6 +81,7 @@ class SymbolSerialized implements SSymbol {
|
|
|
79
81
|
const CODED_SYMBOL_WORDS = 3;
|
|
80
82
|
const CODED_SYMBOL_WORD_BYTES = 8;
|
|
81
83
|
const CODED_SYMBOL_BYTES = CODED_SYMBOL_WORDS * CODED_SYMBOL_WORD_BYTES;
|
|
84
|
+
const MAX_CODED_SYMBOL_BATCH_SIZE = 1_024;
|
|
82
85
|
const BIG_UINT64_ARRAY_IS_LITTLE_ENDIAN =
|
|
83
86
|
typeof BigUint64Array !== "undefined" &&
|
|
84
87
|
new Uint8Array(new BigUint64Array([1n]).buffer)[0] === 1;
|
|
@@ -224,6 +227,9 @@ const codedSymbolBatchField = {
|
|
|
224
227
|
},
|
|
225
228
|
deserialize: (reader: BinaryReader): CodedSymbolBatch => {
|
|
226
229
|
const length = reader.u32();
|
|
230
|
+
if (length > MAX_CODED_SYMBOL_BATCH_SIZE) {
|
|
231
|
+
throw new Error("RIBLT coded symbol batch exceeds the receiver limit");
|
|
232
|
+
}
|
|
227
233
|
const wordLength = length * CODED_SYMBOL_WORDS;
|
|
228
234
|
const byteLength = length * CODED_SYMBOL_BYTES;
|
|
229
235
|
|
|
@@ -466,7 +472,47 @@ const DEFAULT_CONVERGENT_REPAIR_TIMEOUT_MS = 30_000;
|
|
|
466
472
|
const DEFAULT_CONVERGENT_RETRY_INTERVALS_MS = [0, 1_000, 3_000, 7_000];
|
|
467
473
|
const DEFAULT_MAX_CONVERGENT_TRACKED_HASHES = 4_096;
|
|
468
474
|
const MIN_MORE_SYMBOLS_BATCH_SIZE = 64;
|
|
469
|
-
const MAX_MORE_SYMBOLS_BATCH_SIZE =
|
|
475
|
+
const MAX_MORE_SYMBOLS_BATCH_SIZE = MAX_CODED_SYMBOL_BATCH_SIZE;
|
|
476
|
+
const MAX_INCOMING_RATELESS_PROCESSES = 32;
|
|
477
|
+
const MAX_INCOMING_RATELESS_PROCESSES_PER_SENDER = 4;
|
|
478
|
+
const MAX_INCOMING_RATELESS_QUEUED_BATCHES = 8;
|
|
479
|
+
const MAX_INCOMING_RATELESS_SEQUENCE_GAP = BigInt(
|
|
480
|
+
MAX_INCOMING_RATELESS_QUEUED_BATCHES,
|
|
481
|
+
);
|
|
482
|
+
const MIN_RATELESS_SYMBOL_BUDGET = 4_096;
|
|
483
|
+
const MAX_RATELESS_SYMBOL_BUDGET = 262_144;
|
|
484
|
+
const RATELESS_SYMBOL_BUDGET_MULTIPLIER = 4;
|
|
485
|
+
const INCOMING_RATELESS_IDLE_TIMEOUT_MS = 20_000;
|
|
486
|
+
const OUTGOING_RATELESS_IDLE_TIMEOUT_MS = 10_000;
|
|
487
|
+
const RATELESS_PROCESS_ABSOLUTE_TIMEOUT_MS = 120_000;
|
|
488
|
+
const INCOMING_RATELESS_FALLBACK_GRACE_MS = 5_000;
|
|
489
|
+
const MAX_RATELESS_RESPONSE_HASHES_INSPECTED = 10_000;
|
|
490
|
+
export const MAX_ACTIVE_RATELESS_RESPONSES_PER_PEER = 4;
|
|
491
|
+
export const MAX_ACTIVE_RATELESS_RESPONSES_GLOBAL = 32;
|
|
492
|
+
|
|
493
|
+
const getIncomingRatelessSymbolBudget = (initialSymbols: number): number =>
|
|
494
|
+
Math.min(
|
|
495
|
+
MAX_RATELESS_SYMBOL_BUDGET,
|
|
496
|
+
Math.max(
|
|
497
|
+
MIN_RATELESS_SYMBOL_BUDGET,
|
|
498
|
+
initialSymbols * initialSymbols * RATELESS_SYMBOL_BUDGET_MULTIPLIER,
|
|
499
|
+
),
|
|
500
|
+
);
|
|
501
|
+
|
|
502
|
+
const getOutgoingRatelessSymbolBudget = (
|
|
503
|
+
entries: number,
|
|
504
|
+
initialSymbols: number,
|
|
505
|
+
): number =>
|
|
506
|
+
Math.min(
|
|
507
|
+
MAX_RATELESS_SYMBOL_BUDGET,
|
|
508
|
+
Math.max(
|
|
509
|
+
MIN_RATELESS_SYMBOL_BUDGET,
|
|
510
|
+
initialSymbols,
|
|
511
|
+
entries * RATELESS_SYMBOL_BUDGET_MULTIPLIER,
|
|
512
|
+
),
|
|
513
|
+
);
|
|
514
|
+
|
|
515
|
+
type IncomingRatelessProcessResult = boolean | undefined | "fallback-to-simple";
|
|
470
516
|
|
|
471
517
|
@variant([3, 0])
|
|
472
518
|
export class StartSync extends TransportMessage {
|
|
@@ -607,81 +653,177 @@ const buildEncoderOrDecoderFromRange = async <
|
|
|
607
653
|
await ribltReady;
|
|
608
654
|
const encoder =
|
|
609
655
|
type === "encoder" ? new EncoderWrapper() : new DecoderWrapper();
|
|
656
|
+
let transferred = false;
|
|
657
|
+
try {
|
|
658
|
+
const rangeQueryStartedAt = syncProfileStart(profile);
|
|
659
|
+
let hashNumbers: Array<bigint | number> | BigUint64Array | undefined;
|
|
660
|
+
let source = "index";
|
|
661
|
+
const resolved = await resolveHashNumbersInRange?.({
|
|
662
|
+
end1: ranges.end1,
|
|
663
|
+
start1: ranges.start1,
|
|
664
|
+
end2: ranges.end2,
|
|
665
|
+
start2: ranges.start2,
|
|
666
|
+
});
|
|
667
|
+
if (resolved) {
|
|
668
|
+
source = "native";
|
|
669
|
+
hashNumbers =
|
|
670
|
+
typeof BigUint64Array !== "undefined" &&
|
|
671
|
+
resolved instanceof BigUint64Array
|
|
672
|
+
? resolved
|
|
673
|
+
: [...resolved];
|
|
674
|
+
} else {
|
|
675
|
+
const entries = await entryIndex
|
|
676
|
+
.iterate(
|
|
677
|
+
{
|
|
678
|
+
// Range sync for IBLT is done in hashNumber space.
|
|
679
|
+
query: matchEntriesByHashNumberInRangeQuery({
|
|
680
|
+
end1: ranges.end1,
|
|
681
|
+
start1: ranges.start1,
|
|
682
|
+
end2: ranges.end2,
|
|
683
|
+
start2: ranges.start2,
|
|
684
|
+
}),
|
|
685
|
+
},
|
|
686
|
+
{
|
|
687
|
+
shape: {
|
|
688
|
+
hash: true,
|
|
689
|
+
hashNumber: true,
|
|
690
|
+
},
|
|
691
|
+
},
|
|
692
|
+
)
|
|
693
|
+
.all();
|
|
694
|
+
hashNumbers = entries.map((entry) => entry.value.hashNumber);
|
|
695
|
+
}
|
|
696
|
+
if (profile) {
|
|
697
|
+
emitSyncProfileDuration(profile, rangeQueryStartedAt, {
|
|
698
|
+
name: "rateless.rangeQuery",
|
|
699
|
+
entries: hashNumbers.length,
|
|
700
|
+
details: { type, source },
|
|
701
|
+
});
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
if (hashNumbers.length === 0) {
|
|
705
|
+
return false;
|
|
706
|
+
}
|
|
610
707
|
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
let source = "index";
|
|
614
|
-
const resolved = await resolveHashNumbersInRange?.({
|
|
615
|
-
end1: ranges.end1,
|
|
616
|
-
start1: ranges.start1,
|
|
617
|
-
end2: ranges.end2,
|
|
618
|
-
start2: ranges.start2,
|
|
619
|
-
});
|
|
620
|
-
if (resolved) {
|
|
621
|
-
source = "native";
|
|
622
|
-
hashNumbers =
|
|
708
|
+
const addSymbolsStartedAt = syncProfileStart(profile);
|
|
709
|
+
if (
|
|
623
710
|
typeof BigUint64Array !== "undefined" &&
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
emitSyncProfileDuration(profile, rangeQueryStartedAt, {
|
|
651
|
-
name: "rateless.rangeQuery",
|
|
652
|
-
entries: hashNumbers.length,
|
|
653
|
-
details: { type, source },
|
|
654
|
-
});
|
|
711
|
+
typeof (encoder as RibltSymbolAdder).add_symbols === "function"
|
|
712
|
+
) {
|
|
713
|
+
const symbols =
|
|
714
|
+
hashNumbers instanceof BigUint64Array
|
|
715
|
+
? hashNumbers
|
|
716
|
+
: BigUint64Array.from(hashNumbers, coerceBigInt);
|
|
717
|
+
addSymbolsToRiblt(encoder as RibltSymbolAdder, symbols);
|
|
718
|
+
} else {
|
|
719
|
+
for (const hashNumber of hashNumbers) {
|
|
720
|
+
encoder.add_symbol(coerceBigInt(hashNumber));
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
if (profile) {
|
|
724
|
+
emitSyncProfileDuration(profile, addSymbolsStartedAt, {
|
|
725
|
+
name: "rateless.rangeAddSymbols",
|
|
726
|
+
entries: hashNumbers.length,
|
|
727
|
+
symbols: hashNumbers.length,
|
|
728
|
+
details: { type },
|
|
729
|
+
});
|
|
730
|
+
}
|
|
731
|
+
transferred = true;
|
|
732
|
+
return encoder as E;
|
|
733
|
+
} finally {
|
|
734
|
+
if (!transferred) {
|
|
735
|
+
encoder.free();
|
|
736
|
+
}
|
|
655
737
|
}
|
|
738
|
+
};
|
|
656
739
|
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
740
|
+
type RatelessDispatchLifecycle = {
|
|
741
|
+
ownershipLifecycleController: AbortController;
|
|
742
|
+
callerSignal?: AbortSignal;
|
|
743
|
+
controller: AbortController;
|
|
744
|
+
targets: Map<string, RatelessDispatchTargetLifecycle>;
|
|
745
|
+
onOwnerOrCallerAbort: () => void;
|
|
746
|
+
dispatchFinished: boolean;
|
|
747
|
+
disposed: boolean;
|
|
748
|
+
};
|
|
660
749
|
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
750
|
+
type RatelessDispatchTargetLifecycle = {
|
|
751
|
+
lifecycle: RatelessDispatchLifecycle;
|
|
752
|
+
target: string;
|
|
753
|
+
controller: AbortController;
|
|
754
|
+
retainedByProcess: boolean;
|
|
755
|
+
responseLeases: number;
|
|
756
|
+
};
|
|
757
|
+
|
|
758
|
+
type OutgoingRatelessSyncProcess = {
|
|
759
|
+
target: string;
|
|
760
|
+
targetLifecycle: RatelessDispatchTargetLifecycle;
|
|
761
|
+
outgoingHashes: string[];
|
|
762
|
+
authorizedHashes: ReadonlySet<string>;
|
|
763
|
+
consumedResponseHashes: Set<string>;
|
|
764
|
+
encoder: EncoderWrapper;
|
|
765
|
+
timeout?: ReturnType<typeof setTimeout>;
|
|
766
|
+
deadlineTimeout?: ReturnType<typeof setTimeout>;
|
|
767
|
+
refresh: () => void;
|
|
768
|
+
next: (message: {
|
|
769
|
+
lastSeqNo: bigint;
|
|
770
|
+
}) => { symbols: CodedSymbolBatch; exhaustedAfterSend: boolean } | undefined;
|
|
771
|
+
startSimpleFallback: () => Promise<void>;
|
|
772
|
+
simpleFallbackStarted: boolean;
|
|
773
|
+
free: (reason?: unknown) => void;
|
|
774
|
+
processController: AbortController;
|
|
775
|
+
signal: AbortSignal;
|
|
776
|
+
callerSignal?: AbortSignal;
|
|
777
|
+
};
|
|
778
|
+
|
|
779
|
+
type AuthorizedRatelessResponseLease = {
|
|
780
|
+
process: OutgoingRatelessSyncProcess;
|
|
781
|
+
authorized: string[];
|
|
782
|
+
remaining: string[];
|
|
783
|
+
signal: AbortSignal;
|
|
784
|
+
release: (options?: { rollback?: boolean }) => void;
|
|
785
|
+
};
|
|
786
|
+
|
|
787
|
+
type IncomingRatelessSyncProcess = {
|
|
788
|
+
key: string;
|
|
789
|
+
syncId: string;
|
|
790
|
+
sender: string;
|
|
791
|
+
from: PublicSignKey;
|
|
792
|
+
ownershipLifecycleController: AbortController;
|
|
793
|
+
controller: AbortController;
|
|
794
|
+
decoder?: DecoderWrapper;
|
|
795
|
+
completedSynchronizations: Cache<string>;
|
|
796
|
+
timeout?: ReturnType<typeof setTimeout>;
|
|
797
|
+
deadlineTimeout?: ReturnType<typeof setTimeout>;
|
|
798
|
+
refresh: () => void;
|
|
799
|
+
lastSeqNo: bigint;
|
|
800
|
+
processedSymbols: number;
|
|
801
|
+
queuedSymbols: number;
|
|
802
|
+
symbolBudget: number;
|
|
803
|
+
process: (message: {
|
|
804
|
+
seqNo: bigint;
|
|
805
|
+
symbols: CodedSymbolInput;
|
|
806
|
+
}) => Promise<IncomingRatelessProcessResult>;
|
|
807
|
+
requestAll: () => Promise<void>;
|
|
808
|
+
fallbackToSimple: (reason?: unknown) => Promise<void>;
|
|
809
|
+
free: (reason?: unknown) => void;
|
|
810
|
+
complete: () => void;
|
|
811
|
+
};
|
|
812
|
+
|
|
813
|
+
type IncomingRatelessProcessAdmission = {
|
|
814
|
+
key: string;
|
|
815
|
+
sender: string;
|
|
816
|
+
};
|
|
817
|
+
|
|
818
|
+
type RatelessRepairSessionLifecycle = {
|
|
819
|
+
id: string;
|
|
820
|
+
ownershipLifecycleController: AbortController;
|
|
821
|
+
controller: AbortController;
|
|
822
|
+
trackedSession: RepairSession;
|
|
823
|
+
onOwnershipAbort: () => void;
|
|
824
|
+
cancelled: boolean;
|
|
825
|
+
settled: boolean;
|
|
826
|
+
deadlineTimer?: ReturnType<typeof setTimeout>;
|
|
685
827
|
};
|
|
686
828
|
|
|
687
829
|
export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
@@ -689,6 +831,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
689
831
|
{
|
|
690
832
|
simple: SimpleSyncronizer<D>;
|
|
691
833
|
private repairSessionCounter: number;
|
|
834
|
+
private ratelessRepairSessions: Map<string, RatelessRepairSessionLifecycle>;
|
|
692
835
|
|
|
693
836
|
startedOrCompletedSynchronizations: Cache<string>;
|
|
694
837
|
private localRangeEncoderCacheVersion = 0;
|
|
@@ -698,44 +841,40 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
698
841
|
> = new Map();
|
|
699
842
|
private localRangeEncoderCacheMax = 2;
|
|
700
843
|
|
|
701
|
-
ingoingSyncProcesses: Map<
|
|
702
|
-
|
|
703
|
-
{
|
|
704
|
-
decoder: DecoderWrapper;
|
|
705
|
-
timeout: ReturnType<typeof setTimeout>;
|
|
706
|
-
refresh: () => void;
|
|
707
|
-
process: (message: {
|
|
708
|
-
seqNo: bigint;
|
|
709
|
-
symbols: CodedSymbolInput;
|
|
710
|
-
}) => Promise<boolean | undefined>;
|
|
711
|
-
free: () => void;
|
|
712
|
-
}
|
|
713
|
-
>;
|
|
844
|
+
ingoingSyncProcesses: Map<string, IncomingRatelessSyncProcess>;
|
|
845
|
+
private incomingRatelessProcessAdmissions: Set<IncomingRatelessProcessAdmission>;
|
|
714
846
|
|
|
715
|
-
outgoingSyncProcesses: Map<
|
|
847
|
+
outgoingSyncProcesses: Map<string, OutgoingRatelessSyncProcess>;
|
|
848
|
+
private outgoingSyncProcessByTarget: Map<string, OutgoingRatelessSyncProcess>;
|
|
849
|
+
private ratelessDispatchLifecycleController: AbortController;
|
|
850
|
+
private ratelessDispatchTargets: Map<
|
|
716
851
|
string,
|
|
717
|
-
|
|
718
|
-
outgoingHashes: string[];
|
|
719
|
-
encoder: EncoderWrapper;
|
|
720
|
-
timeout: ReturnType<typeof setTimeout>;
|
|
721
|
-
refresh: () => void;
|
|
722
|
-
next: (message: { lastSeqNo: bigint }) => CodedSymbolBatch;
|
|
723
|
-
free: () => void;
|
|
724
|
-
}
|
|
852
|
+
Set<RatelessDispatchTargetLifecycle>
|
|
725
853
|
>;
|
|
854
|
+
private activeRatelessResponseCount: number;
|
|
855
|
+
private activeRatelessResponseCountByPeer: Map<string, number>;
|
|
856
|
+
private ratelessClosed: boolean;
|
|
726
857
|
|
|
727
858
|
constructor(readonly properties: SynchronizerComponents<D>) {
|
|
728
859
|
this.simple = new SimpleSyncronizer(properties);
|
|
729
860
|
this.repairSessionCounter = 0;
|
|
861
|
+
this.ratelessRepairSessions = new Map();
|
|
730
862
|
this.outgoingSyncProcesses = new Map();
|
|
863
|
+
this.outgoingSyncProcessByTarget = new Map();
|
|
864
|
+
this.ratelessDispatchLifecycleController = new AbortController();
|
|
865
|
+
this.ratelessDispatchTargets = new Map();
|
|
866
|
+
this.activeRatelessResponseCount = 0;
|
|
867
|
+
this.activeRatelessResponseCountByPeer = new Map();
|
|
868
|
+
this.ratelessClosed = false;
|
|
731
869
|
this.ingoingSyncProcesses = new Map();
|
|
870
|
+
this.incomingRatelessProcessAdmissions = new Set();
|
|
732
871
|
this.startedOrCompletedSynchronizations = new Cache({ max: 1e4 });
|
|
733
872
|
}
|
|
734
873
|
|
|
735
874
|
private get maxConvergentTrackedHashes() {
|
|
736
875
|
const value = this.properties.sync?.maxConvergentTrackedHashes;
|
|
737
876
|
return value && Number.isFinite(value) && value > 0
|
|
738
|
-
? Math.floor(value)
|
|
877
|
+
? Math.max(1, Math.floor(value))
|
|
739
878
|
: DEFAULT_MAX_CONVERGENT_TRACKED_HASHES;
|
|
740
879
|
}
|
|
741
880
|
|
|
@@ -775,6 +914,121 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
775
914
|
return scored.map((x) => x.entry);
|
|
776
915
|
}
|
|
777
916
|
|
|
917
|
+
private isRatelessRepairSessionActive(
|
|
918
|
+
session: RatelessRepairSessionLifecycle,
|
|
919
|
+
): boolean {
|
|
920
|
+
return (
|
|
921
|
+
!this.ratelessClosed &&
|
|
922
|
+
!session.cancelled &&
|
|
923
|
+
!session.controller.signal.aborted &&
|
|
924
|
+
!session.ownershipLifecycleController.signal.aborted &&
|
|
925
|
+
session.ownershipLifecycleController ===
|
|
926
|
+
this.ratelessDispatchLifecycleController &&
|
|
927
|
+
this.ratelessRepairSessions.get(session.id) === session
|
|
928
|
+
);
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
private cancelRatelessRepairSession(
|
|
932
|
+
session: RatelessRepairSessionLifecycle,
|
|
933
|
+
reason: unknown = new Error("rateless repair session cancelled"),
|
|
934
|
+
): void {
|
|
935
|
+
if (session.cancelled) {
|
|
936
|
+
return;
|
|
937
|
+
}
|
|
938
|
+
session.cancelled = true;
|
|
939
|
+
if (session.deadlineTimer !== undefined) {
|
|
940
|
+
clearTimeout(session.deadlineTimer);
|
|
941
|
+
session.deadlineTimer = undefined;
|
|
942
|
+
}
|
|
943
|
+
session.controller.abort(reason);
|
|
944
|
+
session.trackedSession.cancel();
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
private disposeRatelessRepairSession(
|
|
948
|
+
session: RatelessRepairSessionLifecycle,
|
|
949
|
+
): void {
|
|
950
|
+
if (session.settled) {
|
|
951
|
+
return;
|
|
952
|
+
}
|
|
953
|
+
session.settled = true;
|
|
954
|
+
if (session.deadlineTimer !== undefined) {
|
|
955
|
+
clearTimeout(session.deadlineTimer);
|
|
956
|
+
session.deadlineTimer = undefined;
|
|
957
|
+
}
|
|
958
|
+
session.ownershipLifecycleController.signal.removeEventListener(
|
|
959
|
+
"abort",
|
|
960
|
+
session.onOwnershipAbort,
|
|
961
|
+
);
|
|
962
|
+
if (this.ratelessRepairSessions.get(session.id) === session) {
|
|
963
|
+
this.ratelessRepairSessions.delete(session.id);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
private cancelRatelessRepairSessions(reason: unknown): void {
|
|
968
|
+
for (const session of [...this.ratelessRepairSessions.values()]) {
|
|
969
|
+
this.cancelRatelessRepairSession(session, reason);
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
|
|
973
|
+
private waitForRatelessRepairRetry(
|
|
974
|
+
delayMs: number,
|
|
975
|
+
signal: AbortSignal,
|
|
976
|
+
): Promise<boolean> {
|
|
977
|
+
if (delayMs <= 0) {
|
|
978
|
+
return Promise.resolve(!signal.aborted);
|
|
979
|
+
}
|
|
980
|
+
if (signal.aborted) {
|
|
981
|
+
return Promise.resolve(false);
|
|
982
|
+
}
|
|
983
|
+
return new Promise<boolean>((resolve) => {
|
|
984
|
+
let settled = false;
|
|
985
|
+
let timer: ReturnType<typeof setTimeout>;
|
|
986
|
+
const settle = (elapsed: boolean) => {
|
|
987
|
+
if (settled) {
|
|
988
|
+
return;
|
|
989
|
+
}
|
|
990
|
+
settled = true;
|
|
991
|
+
clearTimeout(timer);
|
|
992
|
+
signal.removeEventListener("abort", onAbort);
|
|
993
|
+
resolve(elapsed);
|
|
994
|
+
};
|
|
995
|
+
const onAbort = () => settle(false);
|
|
996
|
+
timer = setTimeout(() => settle(true), delayMs);
|
|
997
|
+
timer.unref?.();
|
|
998
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
999
|
+
if (signal.aborted) {
|
|
1000
|
+
onAbort();
|
|
1001
|
+
}
|
|
1002
|
+
});
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
private waitForRatelessRepairDispatch(
|
|
1006
|
+
dispatch: Promise<void> | void,
|
|
1007
|
+
signal: AbortSignal,
|
|
1008
|
+
): Promise<boolean> {
|
|
1009
|
+
const dispatchPromise = Promise.resolve(dispatch);
|
|
1010
|
+
return new Promise<boolean>((resolve) => {
|
|
1011
|
+
let settled = false;
|
|
1012
|
+
const settle = (completed: boolean) => {
|
|
1013
|
+
if (settled) {
|
|
1014
|
+
return;
|
|
1015
|
+
}
|
|
1016
|
+
settled = true;
|
|
1017
|
+
signal.removeEventListener("abort", onAbort);
|
|
1018
|
+
resolve(completed);
|
|
1019
|
+
};
|
|
1020
|
+
const onAbort = () => settle(false);
|
|
1021
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1022
|
+
void dispatchPromise.then(
|
|
1023
|
+
() => settle(true),
|
|
1024
|
+
() => settle(true),
|
|
1025
|
+
);
|
|
1026
|
+
if (signal.aborted) {
|
|
1027
|
+
onAbort();
|
|
1028
|
+
}
|
|
1029
|
+
});
|
|
1030
|
+
}
|
|
1031
|
+
|
|
778
1032
|
startRepairSession(properties: {
|
|
779
1033
|
entries: Map<string, SyncEntryCoordinates<D>>;
|
|
780
1034
|
targets: string[];
|
|
@@ -814,7 +1068,6 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
814
1068
|
trackedEntries.set(entry.hash, entry);
|
|
815
1069
|
}
|
|
816
1070
|
|
|
817
|
-
let cancelled = false;
|
|
818
1071
|
const trackedSession = this.simple.startRepairSession({
|
|
819
1072
|
entries: trackedEntries,
|
|
820
1073
|
targets,
|
|
@@ -822,33 +1075,91 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
822
1075
|
timeoutMs,
|
|
823
1076
|
retryIntervalsMs,
|
|
824
1077
|
});
|
|
1078
|
+
const ownershipLifecycleController =
|
|
1079
|
+
this.ratelessDispatchLifecycleController;
|
|
1080
|
+
let session!: RatelessRepairSessionLifecycle;
|
|
1081
|
+
const cancel = (
|
|
1082
|
+
reason: unknown = new Error("rateless repair session cancelled"),
|
|
1083
|
+
) => this.cancelRatelessRepairSession(session, reason);
|
|
1084
|
+
session = {
|
|
1085
|
+
id,
|
|
1086
|
+
ownershipLifecycleController,
|
|
1087
|
+
controller: new AbortController(),
|
|
1088
|
+
trackedSession,
|
|
1089
|
+
onOwnershipAbort: () =>
|
|
1090
|
+
cancel(
|
|
1091
|
+
ownershipLifecycleController.signal.reason ??
|
|
1092
|
+
new Error("rateless repair generation ended"),
|
|
1093
|
+
),
|
|
1094
|
+
cancelled: false,
|
|
1095
|
+
settled: false,
|
|
1096
|
+
deadlineTimer: undefined,
|
|
1097
|
+
};
|
|
1098
|
+
this.ratelessRepairSessions.set(id, session);
|
|
1099
|
+
session.deadlineTimer = setTimeout(
|
|
1100
|
+
() => cancel(new Error("rateless convergent repair session timed out")),
|
|
1101
|
+
Math.max(0, timeoutMs - (Date.now() - startedAt)),
|
|
1102
|
+
);
|
|
1103
|
+
session.deadlineTimer.unref?.();
|
|
1104
|
+
ownershipLifecycleController.signal.addEventListener(
|
|
1105
|
+
"abort",
|
|
1106
|
+
session.onOwnershipAbort,
|
|
1107
|
+
{ once: true },
|
|
1108
|
+
);
|
|
1109
|
+
if (
|
|
1110
|
+
this.ratelessClosed ||
|
|
1111
|
+
ownershipLifecycleController.signal.aborted ||
|
|
1112
|
+
ownershipLifecycleController !==
|
|
1113
|
+
this.ratelessDispatchLifecycleController
|
|
1114
|
+
) {
|
|
1115
|
+
cancel(
|
|
1116
|
+
ownershipLifecycleController.signal.reason ??
|
|
1117
|
+
new Error("rateless repair generation is not active"),
|
|
1118
|
+
);
|
|
1119
|
+
}
|
|
825
1120
|
|
|
826
1121
|
const runDispatchSchedule = async () => {
|
|
827
1122
|
let previousDelay = 0;
|
|
828
1123
|
for (const delayMs of retryIntervalsMs) {
|
|
829
|
-
if (
|
|
1124
|
+
if (!this.isRatelessRepairSessionActive(session)) {
|
|
830
1125
|
return;
|
|
831
1126
|
}
|
|
832
1127
|
const elapsed = Date.now() - startedAt;
|
|
833
1128
|
if (elapsed >= timeoutMs) {
|
|
834
1129
|
return;
|
|
835
1130
|
}
|
|
836
|
-
const waitMs = Math.
|
|
1131
|
+
const waitMs = Math.min(
|
|
1132
|
+
Math.max(0, delayMs - previousDelay),
|
|
1133
|
+
timeoutMs - elapsed,
|
|
1134
|
+
);
|
|
837
1135
|
previousDelay = delayMs;
|
|
838
|
-
if (
|
|
839
|
-
await
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
1136
|
+
if (
|
|
1137
|
+
!(await this.waitForRatelessRepairRetry(
|
|
1138
|
+
waitMs,
|
|
1139
|
+
session.controller.signal,
|
|
1140
|
+
))
|
|
1141
|
+
) {
|
|
1142
|
+
return;
|
|
843
1143
|
}
|
|
844
|
-
if (
|
|
1144
|
+
if (
|
|
1145
|
+
!this.isRatelessRepairSessionActive(session) ||
|
|
1146
|
+
Date.now() - startedAt >= timeoutMs
|
|
1147
|
+
) {
|
|
845
1148
|
return;
|
|
846
1149
|
}
|
|
847
1150
|
try {
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
1151
|
+
if (
|
|
1152
|
+
!(await this.waitForRatelessRepairDispatch(
|
|
1153
|
+
this.onMaybeMissingEntries({
|
|
1154
|
+
entries: properties.entries,
|
|
1155
|
+
targets,
|
|
1156
|
+
signal: session.controller.signal,
|
|
1157
|
+
}),
|
|
1158
|
+
session.controller.signal,
|
|
1159
|
+
))
|
|
1160
|
+
) {
|
|
1161
|
+
return;
|
|
1162
|
+
}
|
|
852
1163
|
} catch {
|
|
853
1164
|
// Best-effort schedule: tracked session timeout/result decides completion.
|
|
854
1165
|
}
|
|
@@ -856,22 +1167,23 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
856
1167
|
};
|
|
857
1168
|
|
|
858
1169
|
const done = (async (): Promise<RepairSessionResult[]> => {
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
1170
|
+
try {
|
|
1171
|
+
await runDispatchSchedule();
|
|
1172
|
+
const trackedResults = await trackedSession.done;
|
|
1173
|
+
return trackedResults.map((result) => ({
|
|
1174
|
+
...result,
|
|
1175
|
+
requestedTotal: requestedHashes.length,
|
|
1176
|
+
truncated: true,
|
|
1177
|
+
}));
|
|
1178
|
+
} finally {
|
|
1179
|
+
this.disposeRatelessRepairSession(session);
|
|
1180
|
+
}
|
|
866
1181
|
})();
|
|
867
1182
|
|
|
868
1183
|
return {
|
|
869
1184
|
id,
|
|
870
1185
|
done,
|
|
871
|
-
cancel: () =>
|
|
872
|
-
cancelled = true;
|
|
873
|
-
trackedSession.cancel();
|
|
874
|
-
},
|
|
1186
|
+
cancel: () => cancel(),
|
|
875
1187
|
};
|
|
876
1188
|
}
|
|
877
1189
|
|
|
@@ -927,38 +1239,77 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
927
1239
|
)}:${String(ranges.end2)}`;
|
|
928
1240
|
}
|
|
929
1241
|
|
|
930
|
-
private decoderFromCachedEncoder(
|
|
1242
|
+
private decoderFromCachedEncoder(
|
|
1243
|
+
encoder: EncoderWrapper,
|
|
1244
|
+
beforeDecoderTransfer?: () => void,
|
|
1245
|
+
): DecoderWrapper {
|
|
931
1246
|
const clone = encoder.clone();
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
1247
|
+
let cloneFreed = false;
|
|
1248
|
+
let decoder: DecoderWrapper | undefined;
|
|
1249
|
+
let decoderTransferred = false;
|
|
1250
|
+
const freeClone = () => {
|
|
1251
|
+
if (cloneFreed) {
|
|
1252
|
+
return;
|
|
1253
|
+
}
|
|
1254
|
+
cloneFreed = true;
|
|
1255
|
+
clone.free();
|
|
1256
|
+
};
|
|
1257
|
+
try {
|
|
1258
|
+
decoder = clone.to_decoder();
|
|
1259
|
+
beforeDecoderTransfer?.();
|
|
1260
|
+
freeClone();
|
|
1261
|
+
decoderTransferred = true;
|
|
1262
|
+
return decoder;
|
|
1263
|
+
} finally {
|
|
1264
|
+
try {
|
|
1265
|
+
freeClone();
|
|
1266
|
+
} finally {
|
|
1267
|
+
if (!decoderTransferred) {
|
|
1268
|
+
decoder?.free();
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
935
1272
|
}
|
|
936
1273
|
|
|
937
|
-
private async getLocalDecoderForRange(
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
1274
|
+
private async getLocalDecoderForRange(
|
|
1275
|
+
ranges: {
|
|
1276
|
+
start1: NumberOrBigint;
|
|
1277
|
+
end1: NumberOrBigint;
|
|
1278
|
+
start2: NumberOrBigint;
|
|
1279
|
+
end2: NumberOrBigint;
|
|
1280
|
+
},
|
|
1281
|
+
options?: {
|
|
1282
|
+
ownershipLifecycleController?: AbortController;
|
|
1283
|
+
signal?: AbortSignal;
|
|
1284
|
+
},
|
|
1285
|
+
): Promise<DecoderWrapper | false> {
|
|
1286
|
+
const isActive = () =>
|
|
1287
|
+
options?.signal?.aborted !== true &&
|
|
1288
|
+
(options?.ownershipLifecycleController === undefined ||
|
|
1289
|
+
this.isIncomingSyncGenerationActive(
|
|
1290
|
+
options.ownershipLifecycleController,
|
|
1291
|
+
));
|
|
1292
|
+
if (!isActive()) {
|
|
1293
|
+
return false;
|
|
1294
|
+
}
|
|
943
1295
|
const profile = this.properties.sync?.profile;
|
|
944
1296
|
const key = this.localRangeEncoderCacheKey(ranges);
|
|
945
1297
|
const cached = this.localRangeEncoderCache.get(key);
|
|
946
1298
|
if (cached && cached.version === this.localRangeEncoderCacheVersion) {
|
|
947
1299
|
const startedAt = syncProfileStart(profile);
|
|
948
1300
|
cached.lastUsed = Date.now();
|
|
949
|
-
|
|
950
|
-
return this.decoderFromCachedEncoder(cached.encoder);
|
|
951
|
-
} finally {
|
|
1301
|
+
return this.decoderFromCachedEncoder(cached.encoder, () => {
|
|
952
1302
|
if (profile) {
|
|
953
1303
|
emitSyncProfileDuration(profile, startedAt, {
|
|
954
1304
|
name: "rateless.localDecoder",
|
|
955
1305
|
cacheHit: true,
|
|
956
1306
|
});
|
|
957
1307
|
}
|
|
958
|
-
}
|
|
1308
|
+
});
|
|
959
1309
|
}
|
|
960
1310
|
|
|
961
1311
|
const startedAt = syncProfileStart(profile);
|
|
1312
|
+
const cacheVersion = this.localRangeEncoderCacheVersion;
|
|
962
1313
|
const encoder = (await buildEncoderOrDecoderFromRange(
|
|
963
1314
|
ranges,
|
|
964
1315
|
this.properties.entryIndex,
|
|
@@ -976,6 +1327,25 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
976
1327
|
}
|
|
977
1328
|
return false;
|
|
978
1329
|
}
|
|
1330
|
+
if (!isActive()) {
|
|
1331
|
+
encoder.free();
|
|
1332
|
+
return false;
|
|
1333
|
+
}
|
|
1334
|
+
if (cacheVersion !== this.localRangeEncoderCacheVersion) {
|
|
1335
|
+
try {
|
|
1336
|
+
return this.decoderFromCachedEncoder(encoder, () => {
|
|
1337
|
+
if (profile) {
|
|
1338
|
+
emitSyncProfileDuration(profile, startedAt, {
|
|
1339
|
+
name: "rateless.localDecoder",
|
|
1340
|
+
cacheHit: false,
|
|
1341
|
+
details: { cacheInvalidated: true },
|
|
1342
|
+
});
|
|
1343
|
+
}
|
|
1344
|
+
});
|
|
1345
|
+
} finally {
|
|
1346
|
+
encoder.free();
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
979
1349
|
|
|
980
1350
|
const now = Date.now();
|
|
981
1351
|
const existing = this.localRangeEncoderCache.get(key);
|
|
@@ -1007,24 +1377,256 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1007
1377
|
this.localRangeEncoderCache.delete(oldestKey);
|
|
1008
1378
|
}
|
|
1009
1379
|
|
|
1010
|
-
|
|
1011
|
-
return this.decoderFromCachedEncoder(encoder);
|
|
1012
|
-
} finally {
|
|
1380
|
+
return this.decoderFromCachedEncoder(encoder, () => {
|
|
1013
1381
|
if (profile) {
|
|
1014
1382
|
emitSyncProfileDuration(profile, startedAt, {
|
|
1015
1383
|
name: "rateless.localDecoder",
|
|
1016
1384
|
cacheHit: false,
|
|
1017
1385
|
});
|
|
1018
1386
|
}
|
|
1387
|
+
});
|
|
1388
|
+
}
|
|
1389
|
+
|
|
1390
|
+
private captureRatelessDispatchLifecycle(
|
|
1391
|
+
targets: string[],
|
|
1392
|
+
callerSignal?: AbortSignal,
|
|
1393
|
+
): RatelessDispatchLifecycle {
|
|
1394
|
+
const ownershipLifecycleController =
|
|
1395
|
+
this.ratelessDispatchLifecycleController;
|
|
1396
|
+
const lifecycle = {
|
|
1397
|
+
ownershipLifecycleController,
|
|
1398
|
+
callerSignal,
|
|
1399
|
+
controller: new AbortController(),
|
|
1400
|
+
targets: new Map<string, RatelessDispatchTargetLifecycle>(),
|
|
1401
|
+
onOwnerOrCallerAbort: () => {
|
|
1402
|
+
const reason =
|
|
1403
|
+
callerSignal?.aborted === true
|
|
1404
|
+
? callerSignal.reason
|
|
1405
|
+
: ownershipLifecycleController.signal.reason;
|
|
1406
|
+
this.abortRatelessDispatchLifecycle(lifecycle, reason);
|
|
1407
|
+
},
|
|
1408
|
+
dispatchFinished: false,
|
|
1409
|
+
disposed: false,
|
|
1410
|
+
} satisfies RatelessDispatchLifecycle;
|
|
1411
|
+
|
|
1412
|
+
for (const target of [...new Set(targets)]) {
|
|
1413
|
+
const targetLifecycle: RatelessDispatchTargetLifecycle = {
|
|
1414
|
+
lifecycle,
|
|
1415
|
+
target,
|
|
1416
|
+
controller: new AbortController(),
|
|
1417
|
+
retainedByProcess: false,
|
|
1418
|
+
responseLeases: 0,
|
|
1419
|
+
};
|
|
1420
|
+
lifecycle.targets.set(target, targetLifecycle);
|
|
1421
|
+
let activeForTarget = this.ratelessDispatchTargets.get(target);
|
|
1422
|
+
if (!activeForTarget) {
|
|
1423
|
+
activeForTarget = new Set();
|
|
1424
|
+
this.ratelessDispatchTargets.set(target, activeForTarget);
|
|
1425
|
+
}
|
|
1426
|
+
activeForTarget.add(targetLifecycle);
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1429
|
+
ownershipLifecycleController.signal.addEventListener(
|
|
1430
|
+
"abort",
|
|
1431
|
+
lifecycle.onOwnerOrCallerAbort,
|
|
1432
|
+
{ once: true },
|
|
1433
|
+
);
|
|
1434
|
+
if (callerSignal && callerSignal !== ownershipLifecycleController.signal) {
|
|
1435
|
+
callerSignal.addEventListener("abort", lifecycle.onOwnerOrCallerAbort, {
|
|
1436
|
+
once: true,
|
|
1437
|
+
});
|
|
1438
|
+
}
|
|
1439
|
+
if (
|
|
1440
|
+
this.ratelessClosed ||
|
|
1441
|
+
ownershipLifecycleController !==
|
|
1442
|
+
this.ratelessDispatchLifecycleController ||
|
|
1443
|
+
ownershipLifecycleController.signal.aborted ||
|
|
1444
|
+
callerSignal?.aborted
|
|
1445
|
+
) {
|
|
1446
|
+
lifecycle.onOwnerOrCallerAbort();
|
|
1447
|
+
}
|
|
1448
|
+
return lifecycle;
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
private abortRatelessDispatchTarget(
|
|
1452
|
+
targetLifecycle: RatelessDispatchTargetLifecycle,
|
|
1453
|
+
reason?: unknown,
|
|
1454
|
+
): void {
|
|
1455
|
+
if (!targetLifecycle.controller.signal.aborted) {
|
|
1456
|
+
targetLifecycle.controller.abort(reason);
|
|
1457
|
+
}
|
|
1458
|
+
this.maybeDisposeRatelessDispatchLifecycle(targetLifecycle.lifecycle);
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
private abortRatelessDispatchLifecycle(
|
|
1462
|
+
lifecycle: RatelessDispatchLifecycle,
|
|
1463
|
+
reason?: unknown,
|
|
1464
|
+
): void {
|
|
1465
|
+
if (!lifecycle.controller.signal.aborted) {
|
|
1466
|
+
lifecycle.controller.abort(reason);
|
|
1467
|
+
}
|
|
1468
|
+
for (const targetLifecycle of lifecycle.targets.values()) {
|
|
1469
|
+
this.abortRatelessDispatchTarget(targetLifecycle, reason);
|
|
1470
|
+
}
|
|
1471
|
+
this.maybeDisposeRatelessDispatchLifecycle(lifecycle);
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1474
|
+
private finishRatelessDispatchLifecycle(
|
|
1475
|
+
lifecycle: RatelessDispatchLifecycle,
|
|
1476
|
+
): void {
|
|
1477
|
+
lifecycle.dispatchFinished = true;
|
|
1478
|
+
this.maybeDisposeRatelessDispatchLifecycle(lifecycle);
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
private maybeDisposeRatelessDispatchLifecycle(
|
|
1482
|
+
lifecycle: RatelessDispatchLifecycle,
|
|
1483
|
+
): void {
|
|
1484
|
+
if (
|
|
1485
|
+
lifecycle.disposed ||
|
|
1486
|
+
!lifecycle.dispatchFinished ||
|
|
1487
|
+
[...lifecycle.targets.values()].some(
|
|
1488
|
+
(target) => target.retainedByProcess || target.responseLeases > 0,
|
|
1489
|
+
)
|
|
1490
|
+
) {
|
|
1491
|
+
return;
|
|
1492
|
+
}
|
|
1493
|
+
lifecycle.disposed = true;
|
|
1494
|
+
lifecycle.ownershipLifecycleController.signal.removeEventListener(
|
|
1495
|
+
"abort",
|
|
1496
|
+
lifecycle.onOwnerOrCallerAbort,
|
|
1497
|
+
);
|
|
1498
|
+
if (
|
|
1499
|
+
lifecycle.callerSignal &&
|
|
1500
|
+
lifecycle.callerSignal !== lifecycle.ownershipLifecycleController.signal
|
|
1501
|
+
) {
|
|
1502
|
+
lifecycle.callerSignal.removeEventListener(
|
|
1503
|
+
"abort",
|
|
1504
|
+
lifecycle.onOwnerOrCallerAbort,
|
|
1505
|
+
);
|
|
1506
|
+
}
|
|
1507
|
+
for (const targetLifecycle of lifecycle.targets.values()) {
|
|
1508
|
+
const activeForTarget = this.ratelessDispatchTargets.get(
|
|
1509
|
+
targetLifecycle.target,
|
|
1510
|
+
);
|
|
1511
|
+
activeForTarget?.delete(targetLifecycle);
|
|
1512
|
+
if (activeForTarget?.size === 0) {
|
|
1513
|
+
this.ratelessDispatchTargets.delete(targetLifecycle.target);
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
private isRatelessDispatchLifecycleActive(
|
|
1519
|
+
lifecycle: RatelessDispatchLifecycle,
|
|
1520
|
+
target?: string,
|
|
1521
|
+
): boolean {
|
|
1522
|
+
if (
|
|
1523
|
+
this.ratelessClosed ||
|
|
1524
|
+
lifecycle.disposed ||
|
|
1525
|
+
lifecycle.ownershipLifecycleController !==
|
|
1526
|
+
this.ratelessDispatchLifecycleController ||
|
|
1527
|
+
lifecycle.ownershipLifecycleController.signal.aborted ||
|
|
1528
|
+
lifecycle.callerSignal?.aborted ||
|
|
1529
|
+
lifecycle.controller.signal.aborted
|
|
1530
|
+
) {
|
|
1531
|
+
return false;
|
|
1019
1532
|
}
|
|
1533
|
+
if (target === undefined) {
|
|
1534
|
+
return true;
|
|
1535
|
+
}
|
|
1536
|
+
const targetLifecycle = lifecycle.targets.get(target);
|
|
1537
|
+
return (
|
|
1538
|
+
targetLifecycle !== undefined &&
|
|
1539
|
+
!targetLifecycle.controller.signal.aborted &&
|
|
1540
|
+
this.ratelessDispatchTargets.get(target)?.has(targetLifecycle) === true
|
|
1541
|
+
);
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1544
|
+
private getIncomingSyncProcessKey(sender: string, syncId: string): string {
|
|
1545
|
+
return `${sender.length}:${sender}${syncId}`;
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
private isIncomingSyncGenerationActive(
|
|
1549
|
+
ownershipLifecycleController: AbortController,
|
|
1550
|
+
): boolean {
|
|
1551
|
+
return (
|
|
1552
|
+
!this.ratelessClosed &&
|
|
1553
|
+
ownershipLifecycleController ===
|
|
1554
|
+
this.ratelessDispatchLifecycleController &&
|
|
1555
|
+
!ownershipLifecycleController.signal.aborted
|
|
1556
|
+
);
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
private isIncomingSyncProcessActive(
|
|
1560
|
+
process: IncomingRatelessSyncProcess,
|
|
1561
|
+
): boolean {
|
|
1562
|
+
return (
|
|
1563
|
+
this.isIncomingSyncGenerationActive(
|
|
1564
|
+
process.ownershipLifecycleController,
|
|
1565
|
+
) &&
|
|
1566
|
+
!process.controller.signal.aborted &&
|
|
1567
|
+
this.ingoingSyncProcesses.get(process.key) === process
|
|
1568
|
+
);
|
|
1020
1569
|
}
|
|
1021
1570
|
|
|
1022
1571
|
async onMaybeMissingEntries(properties: {
|
|
1023
1572
|
entries: Map<string, SyncEntryCoordinates<D>>;
|
|
1024
1573
|
targets: string[];
|
|
1574
|
+
signal?: AbortSignal;
|
|
1025
1575
|
}): Promise<void> {
|
|
1576
|
+
// Capture this rateless open generation synchronously, before any await.
|
|
1577
|
+
// In particular, signal-less internal repair work must not resume after
|
|
1578
|
+
// close/open and borrow the newly opened Simple generation.
|
|
1579
|
+
const lifecycle = this.captureRatelessDispatchLifecycle(
|
|
1580
|
+
properties.targets,
|
|
1581
|
+
properties.signal,
|
|
1582
|
+
);
|
|
1583
|
+
try {
|
|
1584
|
+
await this.onMaybeMissingEntriesWithLifecycle(properties, lifecycle);
|
|
1585
|
+
} finally {
|
|
1586
|
+
this.finishRatelessDispatchLifecycle(lifecycle);
|
|
1587
|
+
}
|
|
1588
|
+
}
|
|
1589
|
+
|
|
1590
|
+
private async onMaybeMissingEntriesWithLifecycle(
|
|
1591
|
+
properties: {
|
|
1592
|
+
entries: Map<string, SyncEntryCoordinates<D>>;
|
|
1593
|
+
targets: string[];
|
|
1594
|
+
signal?: AbortSignal;
|
|
1595
|
+
},
|
|
1596
|
+
lifecycle: RatelessDispatchLifecycle,
|
|
1597
|
+
): Promise<void> {
|
|
1598
|
+
const signal = lifecycle.controller.signal;
|
|
1026
1599
|
const profile = this.properties.sync?.profile;
|
|
1027
1600
|
const startedAt = syncProfileStart(profile);
|
|
1601
|
+
let topLevelProfileEmitted = false;
|
|
1602
|
+
const emitTopLevelProfile = (
|
|
1603
|
+
details: Record<string, string | number | boolean | undefined>,
|
|
1604
|
+
measurements: { messages?: number; symbols?: number } = {},
|
|
1605
|
+
) => {
|
|
1606
|
+
if (!profile || topLevelProfileEmitted) {
|
|
1607
|
+
return;
|
|
1608
|
+
}
|
|
1609
|
+
topLevelProfileEmitted = true;
|
|
1610
|
+
emitSyncProfileDuration(profile, startedAt, {
|
|
1611
|
+
name: "rateless.onMaybeMissingEntries",
|
|
1612
|
+
entries: properties.entries.size,
|
|
1613
|
+
targets: properties.targets.length,
|
|
1614
|
+
...measurements,
|
|
1615
|
+
details,
|
|
1616
|
+
});
|
|
1617
|
+
};
|
|
1618
|
+
const emitCancelledTopLevelProfile = (
|
|
1619
|
+
phase: string,
|
|
1620
|
+
mode: "simple-small" | "rateless" = "rateless",
|
|
1621
|
+
) =>
|
|
1622
|
+
emitTopLevelProfile(
|
|
1623
|
+
{
|
|
1624
|
+
mode,
|
|
1625
|
+
phase,
|
|
1626
|
+
cancelled: true,
|
|
1627
|
+
},
|
|
1628
|
+
{ messages: 0, symbols: 0 },
|
|
1629
|
+
);
|
|
1028
1630
|
// NOTE: this method is best-effort dispatch, not a per-hash convergence API.
|
|
1029
1631
|
// It may require follow-up repair rounds under churn/loss to fully close all gaps.
|
|
1030
1632
|
// Strategy:
|
|
@@ -1035,6 +1637,15 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1035
1637
|
|
|
1036
1638
|
let minSyncIbltSize = 333; // TODO: make configurable
|
|
1037
1639
|
let maxSyncWithSimpleMethod = 1e3;
|
|
1640
|
+
if (signal?.aborted) {
|
|
1641
|
+
emitCancelledTopLevelProfile(
|
|
1642
|
+
"before-dispatch",
|
|
1643
|
+
properties.entries.size <= minSyncIbltSize
|
|
1644
|
+
? "simple-small"
|
|
1645
|
+
: "rateless",
|
|
1646
|
+
);
|
|
1647
|
+
return;
|
|
1648
|
+
}
|
|
1038
1649
|
const priorityFn = this.properties.sync?.priority;
|
|
1039
1650
|
const maxSimpleEntries = this.properties.sync?.maxSimpleEntries;
|
|
1040
1651
|
|
|
@@ -1052,16 +1663,16 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1052
1663
|
await this.simple.onMaybeMissingEntries({
|
|
1053
1664
|
entries: properties.entries,
|
|
1054
1665
|
targets: properties.targets,
|
|
1666
|
+
signal,
|
|
1055
1667
|
});
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
emitSyncProfileDuration(profile, startedAt, {
|
|
1059
|
-
name: "rateless.onMaybeMissingEntries",
|
|
1060
|
-
entries: properties.entries.size,
|
|
1061
|
-
targets: properties.targets.length,
|
|
1062
|
-
details: { mode: "simple-small" },
|
|
1063
|
-
});
|
|
1668
|
+
if (signal?.aborted) {
|
|
1669
|
+
return;
|
|
1064
1670
|
}
|
|
1671
|
+
} finally {
|
|
1672
|
+
emitTopLevelProfile({
|
|
1673
|
+
mode: "simple-small",
|
|
1674
|
+
cancelled: signal?.aborted || undefined,
|
|
1675
|
+
});
|
|
1065
1676
|
}
|
|
1066
1677
|
return;
|
|
1067
1678
|
}
|
|
@@ -1143,32 +1754,44 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1143
1754
|
}
|
|
1144
1755
|
}
|
|
1145
1756
|
|
|
1146
|
-
|
|
1757
|
+
const useAllCoordinatesForIblt =
|
|
1758
|
+
allCoordinatesToSyncWithIblt.length === 0 ||
|
|
1759
|
+
naiveHashes.length > maxSyncWithSimpleMethod;
|
|
1760
|
+
if (useAllCoordinatesForIblt) {
|
|
1761
|
+
// If every entry is a range-boundary special case, or the special-case
|
|
1762
|
+
// set itself is too large for the Simple prelude, use one bounded
|
|
1763
|
+
// Rateless process for the full set. Sending the oversized Simple
|
|
1764
|
+
// prelude first can complete or abort the repair lifecycle before
|
|
1765
|
+
// StartSync is ever dispatched.
|
|
1766
|
+
allCoordinatesToSyncWithIblt = [];
|
|
1767
|
+
for (const entry of properties.entries.values()) {
|
|
1768
|
+
allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
|
|
1769
|
+
}
|
|
1770
|
+
} else if (naiveHashes.length > 0) {
|
|
1147
1771
|
// If there are special-case entries, sync them simply in parallel
|
|
1148
1772
|
if (priorityFn && naiveEntriesForPriority) {
|
|
1149
1773
|
await this.simple.onMaybeMissingEntries({
|
|
1150
1774
|
entries: naiveEntriesForPriority,
|
|
1151
1775
|
targets: properties.targets,
|
|
1776
|
+
signal,
|
|
1152
1777
|
});
|
|
1153
1778
|
} else {
|
|
1154
1779
|
await this.simple.onMaybeMissingHashes({
|
|
1155
1780
|
hashes: naiveHashes,
|
|
1156
1781
|
targets: properties.targets,
|
|
1782
|
+
signal,
|
|
1157
1783
|
});
|
|
1158
1784
|
}
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
allCoordinatesToSyncWithIblt.length === 0 ||
|
|
1163
|
-
naiveHashes.length > maxSyncWithSimpleMethod
|
|
1164
|
-
) {
|
|
1165
|
-
// Fallback: if nothing left for IBLT (or simple set is too large), include all in IBLT
|
|
1166
|
-
allCoordinatesToSyncWithIblt = [];
|
|
1167
|
-
for (const entry of properties.entries.values()) {
|
|
1168
|
-
allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
|
|
1785
|
+
if (signal?.aborted) {
|
|
1786
|
+
emitCancelledTopLevelProfile("simple-prelude");
|
|
1787
|
+
return;
|
|
1169
1788
|
}
|
|
1170
1789
|
}
|
|
1171
1790
|
|
|
1791
|
+
const simplePreludeEntries = useAllCoordinatesForIblt
|
|
1792
|
+
? 0
|
|
1793
|
+
: naiveHashes.length;
|
|
1794
|
+
|
|
1172
1795
|
if (profile) {
|
|
1173
1796
|
emitSyncProfileDuration(profile, selectStartedAt, {
|
|
1174
1797
|
name: "rateless.selectEntries",
|
|
@@ -1176,11 +1799,15 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1176
1799
|
symbols: allCoordinatesToSyncWithIblt.length,
|
|
1177
1800
|
targets: properties.targets.length,
|
|
1178
1801
|
details: {
|
|
1179
|
-
naiveEntries:
|
|
1802
|
+
naiveEntries: simplePreludeEntries,
|
|
1180
1803
|
priority: priorityFn != null,
|
|
1181
1804
|
},
|
|
1182
1805
|
});
|
|
1183
1806
|
}
|
|
1807
|
+
if (signal?.aborted) {
|
|
1808
|
+
emitCancelledTopLevelProfile("entry-selection");
|
|
1809
|
+
return;
|
|
1810
|
+
}
|
|
1184
1811
|
|
|
1185
1812
|
if (allCoordinatesToSyncWithIblt.length === 0) {
|
|
1186
1813
|
if (profile) {
|
|
@@ -1190,13 +1817,8 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1190
1817
|
targets: properties.targets.length,
|
|
1191
1818
|
details: { mode: "simple-only" },
|
|
1192
1819
|
});
|
|
1193
|
-
emitSyncProfileDuration(profile, startedAt, {
|
|
1194
|
-
name: "rateless.onMaybeMissingEntries",
|
|
1195
|
-
entries: properties.entries.size,
|
|
1196
|
-
targets: properties.targets.length,
|
|
1197
|
-
details: { mode: "simple-only" },
|
|
1198
|
-
});
|
|
1199
1820
|
}
|
|
1821
|
+
emitTopLevelProfile({ mode: "simple-only" });
|
|
1200
1822
|
return;
|
|
1201
1823
|
}
|
|
1202
1824
|
|
|
@@ -1207,6 +1829,10 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1207
1829
|
(entry) => entry.hash,
|
|
1208
1830
|
);
|
|
1209
1831
|
await ribltReady;
|
|
1832
|
+
if (signal?.aborted) {
|
|
1833
|
+
emitCancelledTopLevelProfile("riblt-ready");
|
|
1834
|
+
return;
|
|
1835
|
+
}
|
|
1210
1836
|
|
|
1211
1837
|
// For smaller sets, the original `sqrt(n)` heuristic can occasionally under-provision
|
|
1212
1838
|
// low-degree symbols early, causing an unnecessary `MoreSymbols` round-trip. Use a
|
|
@@ -1214,147 +1840,508 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1214
1840
|
let initialSymbolCount = Math.round(
|
|
1215
1841
|
Math.sqrt(allCoordinatesToSyncWithIblt.length),
|
|
1216
1842
|
); // TODO choose better
|
|
1217
|
-
initialSymbolCount = Math.
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
allCoordinatesToSyncWithIblt,
|
|
1221
|
-
this.properties.numbers.maxValue,
|
|
1222
|
-
initialSymbolCount,
|
|
1843
|
+
initialSymbolCount = Math.min(
|
|
1844
|
+
MAX_MORE_SYMBOLS_BATCH_SIZE,
|
|
1845
|
+
Math.max(64, initialSymbolCount),
|
|
1223
1846
|
);
|
|
1224
|
-
if (
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
entries: allCoordinatesToSyncWithIblt.length,
|
|
1228
|
-
symbols: initialSymbols?.length,
|
|
1229
|
-
details: {
|
|
1230
|
-
initialSymbolCount,
|
|
1231
|
-
includesInitialSymbols: initialSymbols != null,
|
|
1232
|
-
},
|
|
1233
|
-
});
|
|
1847
|
+
if (signal?.aborted) {
|
|
1848
|
+
emitCancelledTopLevelProfile("before-encoder-prepare");
|
|
1849
|
+
return;
|
|
1234
1850
|
}
|
|
1235
|
-
|
|
1236
|
-
let
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1851
|
+
const authorizedHashes = new Set(outgoingHashes);
|
|
1852
|
+
let messages = 0;
|
|
1853
|
+
let symbols = 0;
|
|
1854
|
+
for (const [target, targetLifecycle] of lifecycle.targets) {
|
|
1855
|
+
if (!this.isRatelessDispatchLifecycleActive(lifecycle)) {
|
|
1856
|
+
break;
|
|
1857
|
+
}
|
|
1858
|
+
if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
|
|
1859
|
+
continue;
|
|
1860
|
+
}
|
|
1861
|
+
const activeProcess = this.outgoingSyncProcessByTarget.get(target);
|
|
1862
|
+
if (
|
|
1863
|
+
activeProcess &&
|
|
1864
|
+
!activeProcess.signal.aborted &&
|
|
1865
|
+
this.isRatelessDispatchLifecycleActive(
|
|
1866
|
+
activeProcess.targetLifecycle.lifecycle,
|
|
1867
|
+
target,
|
|
1868
|
+
)
|
|
1869
|
+
) {
|
|
1870
|
+
// A repair sweep can flush adjacent batches for the same target
|
|
1871
|
+
// faster than its first StartSync reaches the transport. Replacing
|
|
1872
|
+
// that process here aborts the in-flight send; repeated large
|
|
1873
|
+
// batches can therefore starve Rateless sync entirely. Keep the
|
|
1874
|
+
// unambiguous active process and use bounded Simple sync only for
|
|
1875
|
+
// hashes that it does not already authorize.
|
|
1876
|
+
const uncoveredHashes = outgoingHashes.filter(
|
|
1877
|
+
(hash) => !activeProcess.authorizedHashes.has(hash),
|
|
1878
|
+
);
|
|
1879
|
+
if (uncoveredHashes.length > 0) {
|
|
1880
|
+
await this.simple.onMaybeMissingHashes({
|
|
1881
|
+
hashes: uncoveredHashes,
|
|
1882
|
+
targets: [target],
|
|
1883
|
+
signal,
|
|
1884
|
+
});
|
|
1885
|
+
}
|
|
1886
|
+
continue;
|
|
1887
|
+
}
|
|
1888
|
+
const startedSymbols = this.startOutgoingRatelessSyncForTarget({
|
|
1889
|
+
targetLifecycle,
|
|
1890
|
+
coordinates: allCoordinatesToSyncWithIblt,
|
|
1891
|
+
outgoingHashes,
|
|
1892
|
+
authorizedHashes,
|
|
1893
|
+
initialSymbolCount,
|
|
1894
|
+
entryCount: properties.entries.size,
|
|
1895
|
+
});
|
|
1896
|
+
if (startedSymbols !== undefined) {
|
|
1897
|
+
messages += 1;
|
|
1898
|
+
symbols += startedSymbols;
|
|
1245
1899
|
}
|
|
1246
1900
|
}
|
|
1901
|
+
if (signal.aborted) {
|
|
1902
|
+
emitTopLevelProfile(
|
|
1903
|
+
{
|
|
1904
|
+
mode: "rateless",
|
|
1905
|
+
phase: "start-sync-send",
|
|
1906
|
+
cancelled: true,
|
|
1907
|
+
ibltEntries: allCoordinatesToSyncWithIblt.length,
|
|
1908
|
+
naiveEntries: simplePreludeEntries,
|
|
1909
|
+
},
|
|
1910
|
+
{ messages, symbols },
|
|
1911
|
+
);
|
|
1912
|
+
return;
|
|
1913
|
+
}
|
|
1914
|
+
emitTopLevelProfile(
|
|
1915
|
+
{
|
|
1916
|
+
mode: "rateless",
|
|
1917
|
+
ibltEntries: allCoordinatesToSyncWithIblt.length,
|
|
1918
|
+
naiveEntries: simplePreludeEntries,
|
|
1919
|
+
},
|
|
1920
|
+
{ messages, symbols },
|
|
1921
|
+
);
|
|
1922
|
+
}
|
|
1247
1923
|
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1924
|
+
private startOutgoingRatelessSyncForTarget(properties: {
|
|
1925
|
+
targetLifecycle: RatelessDispatchTargetLifecycle;
|
|
1926
|
+
coordinates: bigint[];
|
|
1927
|
+
outgoingHashes: string[];
|
|
1928
|
+
authorizedHashes: ReadonlySet<string>;
|
|
1929
|
+
initialSymbolCount: number;
|
|
1930
|
+
entryCount: number;
|
|
1931
|
+
}): number | undefined {
|
|
1932
|
+
const lifecycle = properties.targetLifecycle.lifecycle;
|
|
1933
|
+
const target = properties.targetLifecycle.target;
|
|
1934
|
+
const profile = this.properties.sync?.profile;
|
|
1935
|
+
if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
|
|
1936
|
+
return undefined;
|
|
1937
|
+
}
|
|
1254
1938
|
|
|
1255
|
-
const
|
|
1939
|
+
const prepareStartedAt = syncProfileStart(profile);
|
|
1940
|
+
const { encoder, start, end, initialSymbols } = prepareStartSyncEncoder(
|
|
1941
|
+
properties.coordinates,
|
|
1942
|
+
this.properties.numbers.maxValue,
|
|
1943
|
+
properties.initialSymbolCount,
|
|
1944
|
+
);
|
|
1945
|
+
let encoderFreed = false;
|
|
1946
|
+
let encoderTransferred = false;
|
|
1947
|
+
let obj!: OutgoingRatelessSyncProcess;
|
|
1948
|
+
let processConstructed = false;
|
|
1949
|
+
const freeEncoder = () => {
|
|
1950
|
+
if (encoderFreed) {
|
|
1951
|
+
return;
|
|
1952
|
+
}
|
|
1953
|
+
encoderFreed = true;
|
|
1256
1954
|
encoder.free();
|
|
1257
|
-
clearTimeout(this.outgoingSyncProcesses.get(syncId)?.timeout);
|
|
1258
|
-
this.outgoingSyncProcesses.delete(syncId);
|
|
1259
|
-
};
|
|
1260
|
-
const createTimeout = () => {
|
|
1261
|
-
return setTimeout(clear, 1e4); // TODO arg
|
|
1262
1955
|
};
|
|
1956
|
+
try {
|
|
1957
|
+
if (profile) {
|
|
1958
|
+
emitSyncProfileDuration(profile, prepareStartedAt, {
|
|
1959
|
+
name: "rateless.prepareStartSyncEncoder",
|
|
1960
|
+
entries: properties.coordinates.length,
|
|
1961
|
+
symbols: initialSymbols?.length,
|
|
1962
|
+
targets: 1,
|
|
1963
|
+
details: {
|
|
1964
|
+
initialSymbolCount: properties.initialSymbolCount,
|
|
1965
|
+
includesInitialSymbols: initialSymbols != null,
|
|
1966
|
+
},
|
|
1967
|
+
});
|
|
1968
|
+
}
|
|
1969
|
+
if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
|
|
1970
|
+
freeEncoder();
|
|
1971
|
+
return undefined;
|
|
1972
|
+
}
|
|
1263
1973
|
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
// object with three bigint fields, so very large batches can dominate heap under
|
|
1267
|
-
// concurrent churn even though the native RIBLT encoder itself is compact.
|
|
1268
|
-
const nextBatch = Math.max(
|
|
1269
|
-
MIN_MORE_SYMBOLS_BATCH_SIZE,
|
|
1270
|
-
Math.min(
|
|
1271
|
-
MAX_MORE_SYMBOLS_BATCH_SIZE,
|
|
1272
|
-
Math.ceil(allCoordinatesToSyncWithIblt.length / 4),
|
|
1273
|
-
),
|
|
1274
|
-
);
|
|
1275
|
-
const obj = {
|
|
1276
|
-
encoder,
|
|
1277
|
-
timeout: createTimeout(),
|
|
1278
|
-
refresh: () => {
|
|
1279
|
-
let prevTimeout = obj.timeout;
|
|
1280
|
-
if (prevTimeout) {
|
|
1281
|
-
clearTimeout(prevTimeout);
|
|
1282
|
-
}
|
|
1283
|
-
obj.timeout = createTimeout();
|
|
1284
|
-
},
|
|
1285
|
-
next: (properties: { lastSeqNo: bigint }): CodedSymbolBatch => {
|
|
1286
|
-
if (properties.lastSeqNo <= lastSeqNo) {
|
|
1287
|
-
return CodedSymbolBatch.fromSymbols([]);
|
|
1288
|
-
}
|
|
1289
|
-
lastSeqNo++;
|
|
1290
|
-
obj.refresh(); // TODO use timestamp instead and collective pruning/refresh
|
|
1291
|
-
|
|
1974
|
+
let startSyncSymbols = initialSymbols;
|
|
1975
|
+
if (!startSyncSymbols) {
|
|
1292
1976
|
const produceStartedAt = syncProfileStart(profile);
|
|
1293
|
-
|
|
1977
|
+
startSyncSymbols = produceNextCodedSymbols(
|
|
1978
|
+
encoder,
|
|
1979
|
+
properties.initialSymbolCount,
|
|
1980
|
+
);
|
|
1294
1981
|
if (profile) {
|
|
1295
1982
|
emitSyncProfileDuration(profile, produceStartedAt, {
|
|
1296
|
-
name: "rateless.
|
|
1297
|
-
|
|
1298
|
-
|
|
1983
|
+
name: "rateless.produceStartSyncSymbols",
|
|
1984
|
+
symbols: startSyncSymbols.length,
|
|
1985
|
+
targets: 1,
|
|
1299
1986
|
});
|
|
1300
1987
|
}
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1988
|
+
}
|
|
1989
|
+
if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
|
|
1990
|
+
freeEncoder();
|
|
1991
|
+
return undefined;
|
|
1992
|
+
}
|
|
1306
1993
|
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
entries: properties.entries.size,
|
|
1312
|
-
symbols: startSyncSymbols.length,
|
|
1313
|
-
targets: properties.targets.length,
|
|
1314
|
-
syncId,
|
|
1315
|
-
details: { mode: "rateless" },
|
|
1994
|
+
const startSync = new StartSync({
|
|
1995
|
+
from: start,
|
|
1996
|
+
to: end,
|
|
1997
|
+
symbols: startSyncSymbols,
|
|
1316
1998
|
});
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1999
|
+
const syncId = getSyncIdString(startSync);
|
|
2000
|
+
const targetSignal = properties.targetLifecycle.controller.signal;
|
|
2001
|
+
const processController = new AbortController();
|
|
2002
|
+
const processSignal = processController.signal;
|
|
2003
|
+
let cleared = false;
|
|
2004
|
+
let lastSeqNo = -1n;
|
|
2005
|
+
let symbolsProduced = startSyncSymbols.length;
|
|
2006
|
+
let symbolBudgetExhausted = false;
|
|
2007
|
+
let simpleFallbackPromise: Promise<void> | undefined;
|
|
2008
|
+
const symbolBudget = getOutgoingRatelessSymbolBudget(
|
|
2009
|
+
properties.coordinates.length,
|
|
2010
|
+
startSyncSymbols.length,
|
|
2011
|
+
);
|
|
2012
|
+
let onTargetAbort: (() => void) | undefined;
|
|
2013
|
+
const clear = (
|
|
2014
|
+
reason: unknown = new Error("rateless outgoing process closed"),
|
|
2015
|
+
) => {
|
|
2016
|
+
if (cleared) {
|
|
2017
|
+
return;
|
|
2018
|
+
}
|
|
2019
|
+
cleared = true;
|
|
2020
|
+
|
|
2021
|
+
// Timeout/replacement owns only the encoder process. Response leases
|
|
2022
|
+
// retain the original open/caller/target lifecycle independently, so a
|
|
2023
|
+
// response already accepted for delivery can finish after this process
|
|
2024
|
+
// expires while close, caller abort, and disconnect still cancel it.
|
|
2025
|
+
processController.abort(reason);
|
|
2026
|
+
if (onTargetAbort) {
|
|
2027
|
+
targetSignal.removeEventListener("abort", onTargetAbort);
|
|
2028
|
+
}
|
|
2029
|
+
if (obj.timeout) {
|
|
2030
|
+
clearTimeout(obj.timeout);
|
|
2031
|
+
}
|
|
2032
|
+
if (obj.deadlineTimeout) {
|
|
2033
|
+
clearTimeout(obj.deadlineTimeout);
|
|
2034
|
+
}
|
|
2035
|
+
if (this.outgoingSyncProcesses.get(syncId) === obj) {
|
|
2036
|
+
this.outgoingSyncProcesses.delete(syncId);
|
|
2037
|
+
}
|
|
2038
|
+
if (this.outgoingSyncProcessByTarget.get(target) === obj) {
|
|
2039
|
+
this.outgoingSyncProcessByTarget.delete(target);
|
|
2040
|
+
}
|
|
2041
|
+
properties.targetLifecycle.retainedByProcess = false;
|
|
2042
|
+
freeEncoder();
|
|
2043
|
+
this.maybeDisposeRatelessDispatchLifecycle(lifecycle);
|
|
2044
|
+
};
|
|
2045
|
+
const startSimpleFallback = () => {
|
|
2046
|
+
if (!simpleFallbackPromise) {
|
|
2047
|
+
// From this point overlapping ResponseMaybeSync authorization
|
|
2048
|
+
// belongs to Simple. Keeping Rateless first would ship the response
|
|
2049
|
+
// while leaving the duplicate Simple lease charged until its TTL.
|
|
2050
|
+
obj.simpleFallbackStarted = true;
|
|
2051
|
+
simpleFallbackPromise = Promise.resolve(
|
|
2052
|
+
this.simple.onMaybeMissingHashes({
|
|
2053
|
+
hashes: properties.outgoingHashes,
|
|
2054
|
+
targets: [target],
|
|
2055
|
+
signal: lifecycle.callerSignal,
|
|
2056
|
+
}),
|
|
2057
|
+
);
|
|
2058
|
+
}
|
|
2059
|
+
return simpleFallbackPromise;
|
|
2060
|
+
};
|
|
2061
|
+
const fallbackAndClear = (reason: Error) => {
|
|
2062
|
+
void startSimpleFallback().catch((error) => logger.error(error));
|
|
2063
|
+
clear(reason);
|
|
2064
|
+
};
|
|
2065
|
+
const createTimeout = () => {
|
|
2066
|
+
const timeout = setTimeout(
|
|
2067
|
+
() =>
|
|
2068
|
+
fallbackAndClear(new Error("rateless outgoing process timed out")),
|
|
2069
|
+
OUTGOING_RATELESS_IDLE_TIMEOUT_MS,
|
|
2070
|
+
);
|
|
2071
|
+
timeout.unref?.();
|
|
2072
|
+
return timeout;
|
|
2073
|
+
};
|
|
2074
|
+
const createDeadlineTimeout = () => {
|
|
2075
|
+
const timeout = setTimeout(
|
|
2076
|
+
() =>
|
|
2077
|
+
fallbackAndClear(
|
|
2078
|
+
new Error("rateless outgoing process deadline exceeded"),
|
|
2079
|
+
),
|
|
2080
|
+
RATELESS_PROCESS_ABSOLUTE_TIMEOUT_MS,
|
|
2081
|
+
);
|
|
2082
|
+
timeout.unref?.();
|
|
2083
|
+
return timeout;
|
|
2084
|
+
};
|
|
2085
|
+
onTargetAbort = () => clear(targetSignal.reason);
|
|
2086
|
+
|
|
2087
|
+
// Keep follow-up symbol payloads bounded. Each symbol is serialized as an
|
|
2088
|
+
// object with three bigint fields, so very large batches can dominate heap under
|
|
2089
|
+
// concurrent churn even though the native RIBLT encoder itself is compact.
|
|
2090
|
+
const nextBatch = Math.max(
|
|
2091
|
+
MIN_MORE_SYMBOLS_BATCH_SIZE,
|
|
2092
|
+
Math.min(
|
|
2093
|
+
MAX_MORE_SYMBOLS_BATCH_SIZE,
|
|
2094
|
+
Math.ceil(properties.coordinates.length / 4),
|
|
2095
|
+
),
|
|
2096
|
+
);
|
|
2097
|
+
obj = {
|
|
2098
|
+
target,
|
|
2099
|
+
targetLifecycle: properties.targetLifecycle,
|
|
2100
|
+
encoder,
|
|
2101
|
+
timeout: undefined,
|
|
2102
|
+
deadlineTimeout: undefined,
|
|
2103
|
+
refresh: () => {
|
|
2104
|
+
if (obj.timeout) {
|
|
2105
|
+
clearTimeout(obj.timeout);
|
|
2106
|
+
}
|
|
2107
|
+
obj.timeout = createTimeout();
|
|
2108
|
+
},
|
|
2109
|
+
next: (message: { lastSeqNo: bigint }) => {
|
|
2110
|
+
if (processSignal.aborted || symbolBudgetExhausted) {
|
|
2111
|
+
return undefined;
|
|
2112
|
+
}
|
|
2113
|
+
if (message.lastSeqNo !== lastSeqNo + 1n) {
|
|
2114
|
+
return undefined;
|
|
2115
|
+
}
|
|
2116
|
+
lastSeqNo = message.lastSeqNo;
|
|
2117
|
+
|
|
2118
|
+
const remainingBudget = symbolBudget - symbolsProduced;
|
|
2119
|
+
if (remainingBudget <= 0) {
|
|
2120
|
+
symbolBudgetExhausted = true;
|
|
2121
|
+
return {
|
|
2122
|
+
symbols: CodedSymbolBatch.fromSymbols([]),
|
|
2123
|
+
exhaustedAfterSend: true,
|
|
2124
|
+
};
|
|
2125
|
+
}
|
|
2126
|
+
|
|
2127
|
+
const produceStartedAt = syncProfileStart(profile);
|
|
2128
|
+
const symbols = produceNextCodedSymbols(
|
|
2129
|
+
encoder,
|
|
2130
|
+
Math.min(nextBatch, remainingBudget),
|
|
2131
|
+
);
|
|
2132
|
+
symbolsProduced += symbols.length;
|
|
2133
|
+
const exhaustedAfterSend = symbols.length === 0;
|
|
2134
|
+
if (exhaustedAfterSend) {
|
|
2135
|
+
symbolBudgetExhausted = true;
|
|
2136
|
+
} else {
|
|
2137
|
+
obj.refresh();
|
|
2138
|
+
}
|
|
2139
|
+
if (profile) {
|
|
2140
|
+
emitSyncProfileDuration(profile, produceStartedAt, {
|
|
2141
|
+
name: "rateless.produceMoreSymbols",
|
|
2142
|
+
syncId,
|
|
2143
|
+
symbols: symbols.length,
|
|
2144
|
+
targets: 1,
|
|
2145
|
+
});
|
|
2146
|
+
}
|
|
2147
|
+
return { symbols, exhaustedAfterSend };
|
|
2148
|
+
},
|
|
2149
|
+
startSimpleFallback,
|
|
2150
|
+
simpleFallbackStarted: false,
|
|
2151
|
+
free: clear,
|
|
2152
|
+
outgoingHashes: properties.outgoingHashes,
|
|
2153
|
+
authorizedHashes: properties.authorizedHashes,
|
|
2154
|
+
consumedResponseHashes: new Set(),
|
|
2155
|
+
processController,
|
|
2156
|
+
signal: processSignal,
|
|
2157
|
+
callerSignal: lifecycle.callerSignal,
|
|
2158
|
+
};
|
|
2159
|
+
processConstructed = true;
|
|
2160
|
+
|
|
2161
|
+
if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
|
|
2162
|
+
clear();
|
|
2163
|
+
return undefined;
|
|
2164
|
+
}
|
|
2165
|
+
// Without a wire process id on ResponseMaybeSync, retaining at most one
|
|
2166
|
+
// process per target keeps sender/hash authorization unambiguous.
|
|
2167
|
+
this.outgoingSyncProcessByTarget
|
|
2168
|
+
.get(target)
|
|
2169
|
+
?.free(new Error("rateless outgoing process replaced"));
|
|
2170
|
+
if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
|
|
2171
|
+
clear();
|
|
2172
|
+
return undefined;
|
|
2173
|
+
}
|
|
2174
|
+
properties.targetLifecycle.retainedByProcess = true;
|
|
2175
|
+
this.outgoingSyncProcesses.set(syncId, obj);
|
|
2176
|
+
this.outgoingSyncProcessByTarget.set(target, obj);
|
|
2177
|
+
obj.timeout = createTimeout();
|
|
2178
|
+
obj.deadlineTimeout = createDeadlineTimeout();
|
|
2179
|
+
targetSignal.addEventListener("abort", onTargetAbort, { once: true });
|
|
2180
|
+
encoderTransferred = true;
|
|
2181
|
+
if (targetSignal.aborted) {
|
|
2182
|
+
clear(targetSignal.reason);
|
|
2183
|
+
return undefined;
|
|
2184
|
+
}
|
|
2185
|
+
if (profile) {
|
|
2186
|
+
emitSyncProfileEvent(profile, {
|
|
2187
|
+
name: "rateless.dispatchMode",
|
|
2188
|
+
entries: properties.entryCount,
|
|
2189
|
+
symbols: startSyncSymbols.length,
|
|
2190
|
+
targets: 1,
|
|
2191
|
+
syncId,
|
|
2192
|
+
details: { mode: "rateless" },
|
|
2193
|
+
});
|
|
2194
|
+
}
|
|
2195
|
+
|
|
2196
|
+
const sendStartedAt = syncProfileStart(profile);
|
|
2197
|
+
let sendResult: Promise<unknown>;
|
|
2198
|
+
try {
|
|
2199
|
+
sendResult = Promise.resolve(
|
|
2200
|
+
this.simple.rpc.send(startSync, {
|
|
2201
|
+
mode: new SilentDelivery({
|
|
2202
|
+
to: [target],
|
|
2203
|
+
redundancy: 1,
|
|
2204
|
+
}),
|
|
2205
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
2206
|
+
signal: processSignal,
|
|
1341
2207
|
}),
|
|
2208
|
+
);
|
|
2209
|
+
} catch (error) {
|
|
2210
|
+
sendResult = Promise.reject(error);
|
|
2211
|
+
}
|
|
2212
|
+
void sendResult.then(
|
|
2213
|
+
() => {
|
|
2214
|
+
if (profile) {
|
|
2215
|
+
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
2216
|
+
name: "rateless.sendStartSync",
|
|
2217
|
+
messages: 1,
|
|
2218
|
+
symbols: startSyncSymbols.length,
|
|
2219
|
+
targets: 1,
|
|
2220
|
+
syncId,
|
|
2221
|
+
});
|
|
2222
|
+
}
|
|
2223
|
+
},
|
|
2224
|
+
() => {
|
|
2225
|
+
clear();
|
|
2226
|
+
if (profile) {
|
|
2227
|
+
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
2228
|
+
name: "rateless.sendStartSync",
|
|
2229
|
+
messages: 1,
|
|
2230
|
+
symbols: startSyncSymbols.length,
|
|
2231
|
+
targets: 1,
|
|
2232
|
+
syncId,
|
|
2233
|
+
details: {
|
|
2234
|
+
rejected: true,
|
|
2235
|
+
cancelled: processSignal.aborted || undefined,
|
|
2236
|
+
},
|
|
2237
|
+
});
|
|
2238
|
+
}
|
|
2239
|
+
},
|
|
1342
2240
|
);
|
|
2241
|
+
if (processSignal.aborted) {
|
|
2242
|
+
clear();
|
|
2243
|
+
return undefined;
|
|
2244
|
+
}
|
|
2245
|
+
return startSyncSymbols.length;
|
|
2246
|
+
} catch (error) {
|
|
2247
|
+
if (processConstructed) {
|
|
2248
|
+
obj.free(error);
|
|
2249
|
+
}
|
|
2250
|
+
throw error;
|
|
2251
|
+
} finally {
|
|
2252
|
+
if (!encoderTransferred) {
|
|
2253
|
+
freeEncoder();
|
|
2254
|
+
}
|
|
1343
2255
|
}
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
2256
|
+
}
|
|
2257
|
+
|
|
2258
|
+
private consumeAuthorizedRatelessResponse(
|
|
2259
|
+
hashes: Iterable<string>,
|
|
2260
|
+
from: PublicSignKey,
|
|
2261
|
+
): AuthorizedRatelessResponseLease | undefined {
|
|
2262
|
+
const target = from.hashcode();
|
|
2263
|
+
const process = this.outgoingSyncProcessByTarget.get(target);
|
|
2264
|
+
if (
|
|
2265
|
+
!process ||
|
|
2266
|
+
process.signal.aborted ||
|
|
2267
|
+
!this.isRatelessDispatchLifecycleActive(
|
|
2268
|
+
process.targetLifecycle.lifecycle,
|
|
2269
|
+
target,
|
|
2270
|
+
)
|
|
2271
|
+
) {
|
|
2272
|
+
return undefined;
|
|
2273
|
+
}
|
|
2274
|
+
if (
|
|
2275
|
+
this.activeRatelessResponseCount >=
|
|
2276
|
+
MAX_ACTIVE_RATELESS_RESPONSES_GLOBAL ||
|
|
2277
|
+
(this.activeRatelessResponseCountByPeer.get(target) ?? 0) >=
|
|
2278
|
+
MAX_ACTIVE_RATELESS_RESPONSES_PER_PEER
|
|
2279
|
+
) {
|
|
2280
|
+
return undefined;
|
|
1357
2281
|
}
|
|
2282
|
+
const authorized: string[] = [];
|
|
2283
|
+
const remaining: string[] = [];
|
|
2284
|
+
const seen = new Set<string>();
|
|
2285
|
+
for (const hash of hashes) {
|
|
2286
|
+
if (seen.has(hash)) {
|
|
2287
|
+
continue;
|
|
2288
|
+
}
|
|
2289
|
+
seen.add(hash);
|
|
2290
|
+
if (
|
|
2291
|
+
!process.authorizedHashes.has(hash) ||
|
|
2292
|
+
process.consumedResponseHashes.has(hash)
|
|
2293
|
+
) {
|
|
2294
|
+
remaining.push(hash);
|
|
2295
|
+
continue;
|
|
2296
|
+
}
|
|
2297
|
+
process.consumedResponseHashes.add(hash);
|
|
2298
|
+
authorized.push(hash);
|
|
2299
|
+
}
|
|
2300
|
+
if (authorized.length === 0) {
|
|
2301
|
+
return {
|
|
2302
|
+
process,
|
|
2303
|
+
authorized,
|
|
2304
|
+
remaining,
|
|
2305
|
+
signal: process.targetLifecycle.controller.signal,
|
|
2306
|
+
release: () => {},
|
|
2307
|
+
};
|
|
2308
|
+
}
|
|
2309
|
+
|
|
2310
|
+
const targetLifecycle = process.targetLifecycle;
|
|
2311
|
+
targetLifecycle.responseLeases += 1;
|
|
2312
|
+
this.activeRatelessResponseCount += 1;
|
|
2313
|
+
this.activeRatelessResponseCountByPeer.set(
|
|
2314
|
+
target,
|
|
2315
|
+
(this.activeRatelessResponseCountByPeer.get(target) ?? 0) + 1,
|
|
2316
|
+
);
|
|
2317
|
+
let released = false;
|
|
2318
|
+
return {
|
|
2319
|
+
process,
|
|
2320
|
+
authorized,
|
|
2321
|
+
remaining,
|
|
2322
|
+
signal: targetLifecycle.controller.signal,
|
|
2323
|
+
release: (options) => {
|
|
2324
|
+
if (released) {
|
|
2325
|
+
return;
|
|
2326
|
+
}
|
|
2327
|
+
released = true;
|
|
2328
|
+
if (options?.rollback) {
|
|
2329
|
+
for (const hash of authorized) {
|
|
2330
|
+
process.consumedResponseHashes.delete(hash);
|
|
2331
|
+
}
|
|
2332
|
+
}
|
|
2333
|
+
targetLifecycle.responseLeases -= 1;
|
|
2334
|
+
this.activeRatelessResponseCount -= 1;
|
|
2335
|
+
const activeForPeer =
|
|
2336
|
+
(this.activeRatelessResponseCountByPeer.get(target) ?? 1) - 1;
|
|
2337
|
+
if (activeForPeer === 0) {
|
|
2338
|
+
this.activeRatelessResponseCountByPeer.delete(target);
|
|
2339
|
+
} else {
|
|
2340
|
+
this.activeRatelessResponseCountByPeer.set(target, activeForPeer);
|
|
2341
|
+
}
|
|
2342
|
+
this.maybeDisposeRatelessDispatchLifecycle(targetLifecycle.lifecycle);
|
|
2343
|
+
},
|
|
2344
|
+
};
|
|
1358
2345
|
}
|
|
1359
2346
|
|
|
1360
2347
|
async onMessage(
|
|
@@ -1363,226 +2350,550 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1363
2350
|
): Promise<boolean> {
|
|
1364
2351
|
const profile = this.properties.sync?.profile;
|
|
1365
2352
|
if (message instanceof StartSync) {
|
|
2353
|
+
const from = context.from;
|
|
2354
|
+
const ownershipLifecycleController =
|
|
2355
|
+
this.ratelessDispatchLifecycleController;
|
|
2356
|
+
if (
|
|
2357
|
+
!from ||
|
|
2358
|
+
!this.isIncomingSyncGenerationActive(ownershipLifecycleController)
|
|
2359
|
+
) {
|
|
2360
|
+
return true;
|
|
2361
|
+
}
|
|
2362
|
+
const sender = from.hashcode();
|
|
1366
2363
|
const syncId = getSyncIdString(message);
|
|
1367
|
-
|
|
2364
|
+
const key = this.getIncomingSyncProcessKey(sender, syncId);
|
|
2365
|
+
const completedSynchronizations = this.startedOrCompletedSynchronizations;
|
|
2366
|
+
const admissionSet = this.incomingRatelessProcessAdmissions;
|
|
2367
|
+
if (this.ingoingSyncProcesses.has(key)) {
|
|
1368
2368
|
return true;
|
|
1369
2369
|
}
|
|
1370
2370
|
|
|
1371
|
-
if (
|
|
2371
|
+
if (completedSynchronizations.has(key)) {
|
|
1372
2372
|
return true;
|
|
1373
2373
|
}
|
|
1374
2374
|
|
|
1375
|
-
|
|
2375
|
+
let admissionsForSender = 0;
|
|
2376
|
+
for (const admission of admissionSet) {
|
|
2377
|
+
if (admission.key === key) {
|
|
2378
|
+
return true;
|
|
2379
|
+
}
|
|
2380
|
+
if (admission.sender === sender) {
|
|
2381
|
+
admissionsForSender += 1;
|
|
2382
|
+
}
|
|
2383
|
+
}
|
|
2384
|
+
if (
|
|
2385
|
+
admissionSet.size >= MAX_INCOMING_RATELESS_PROCESSES ||
|
|
2386
|
+
admissionsForSender >= MAX_INCOMING_RATELESS_PROCESSES_PER_SENDER
|
|
2387
|
+
) {
|
|
2388
|
+
return true;
|
|
2389
|
+
}
|
|
2390
|
+
const admission = { key, sender };
|
|
2391
|
+
admissionSet.add(admission);
|
|
2392
|
+
|
|
2393
|
+
const controller = new AbortController();
|
|
2394
|
+
let freed = false;
|
|
2395
|
+
let completed = false;
|
|
2396
|
+
let initializationPending = false;
|
|
2397
|
+
let fallbackSendPending = false;
|
|
2398
|
+
let admissionReleased = false;
|
|
2399
|
+
let fallbackGraceTimeout: ReturnType<typeof setTimeout> | undefined;
|
|
2400
|
+
let onOwnershipAbort: (() => void) | undefined;
|
|
2401
|
+
let obj!: IncomingRatelessSyncProcess;
|
|
2402
|
+
const releaseAdmissionIfSettled = () => {
|
|
2403
|
+
if (
|
|
2404
|
+
admissionReleased ||
|
|
2405
|
+
!freed ||
|
|
2406
|
+
initializationPending ||
|
|
2407
|
+
fallbackSendPending
|
|
2408
|
+
) {
|
|
2409
|
+
return;
|
|
2410
|
+
}
|
|
2411
|
+
admissionReleased = true;
|
|
2412
|
+
admissionSet.delete(admission);
|
|
2413
|
+
};
|
|
2414
|
+
const free = (
|
|
2415
|
+
reason: unknown = new Error("incoming rateless process closed"),
|
|
2416
|
+
) => {
|
|
2417
|
+
if (freed) {
|
|
2418
|
+
return;
|
|
2419
|
+
}
|
|
2420
|
+
freed = true;
|
|
2421
|
+
// Abort first: no queued process/send continuation may observe the
|
|
2422
|
+
// object detached while its native decoder is still usable.
|
|
2423
|
+
controller.abort(reason);
|
|
2424
|
+
if (onOwnershipAbort) {
|
|
2425
|
+
ownershipLifecycleController.signal.removeEventListener(
|
|
2426
|
+
"abort",
|
|
2427
|
+
onOwnershipAbort,
|
|
2428
|
+
);
|
|
2429
|
+
}
|
|
2430
|
+
if (obj.timeout) {
|
|
2431
|
+
clearTimeout(obj.timeout);
|
|
2432
|
+
obj.timeout = undefined;
|
|
2433
|
+
}
|
|
2434
|
+
if (obj.deadlineTimeout) {
|
|
2435
|
+
clearTimeout(obj.deadlineTimeout);
|
|
2436
|
+
obj.deadlineTimeout = undefined;
|
|
2437
|
+
}
|
|
2438
|
+
if (fallbackGraceTimeout) {
|
|
2439
|
+
clearTimeout(fallbackGraceTimeout);
|
|
2440
|
+
fallbackGraceTimeout = undefined;
|
|
2441
|
+
}
|
|
2442
|
+
if (this.ingoingSyncProcesses.get(key) === obj) {
|
|
2443
|
+
this.ingoingSyncProcesses.delete(key);
|
|
2444
|
+
}
|
|
2445
|
+
const ownedDecoder = obj.decoder;
|
|
2446
|
+
obj.decoder = undefined;
|
|
2447
|
+
ownedDecoder?.free();
|
|
2448
|
+
releaseAdmissionIfSettled();
|
|
2449
|
+
};
|
|
2450
|
+
const complete = () => {
|
|
2451
|
+
if (completed || !this.isIncomingSyncProcessActive(obj)) {
|
|
2452
|
+
return;
|
|
2453
|
+
}
|
|
2454
|
+
completed = true;
|
|
2455
|
+
completedSynchronizations.add(key);
|
|
2456
|
+
free(new Error("incoming rateless process completed"));
|
|
2457
|
+
};
|
|
2458
|
+
obj = {
|
|
2459
|
+
key,
|
|
2460
|
+
syncId,
|
|
2461
|
+
sender,
|
|
2462
|
+
from,
|
|
2463
|
+
ownershipLifecycleController,
|
|
2464
|
+
controller,
|
|
2465
|
+
completedSynchronizations,
|
|
2466
|
+
timeout: undefined,
|
|
2467
|
+
deadlineTimeout: undefined,
|
|
2468
|
+
refresh: () => {},
|
|
2469
|
+
lastSeqNo: -1n,
|
|
2470
|
+
processedSymbols: 0,
|
|
2471
|
+
queuedSymbols: 0,
|
|
2472
|
+
symbolBudget: getIncomingRatelessSymbolBudget(message.symbols.length),
|
|
2473
|
+
process: async () => undefined,
|
|
2474
|
+
requestAll: async () => {},
|
|
2475
|
+
fallbackToSimple: async () => {},
|
|
2476
|
+
free,
|
|
2477
|
+
complete,
|
|
2478
|
+
};
|
|
2479
|
+
this.ingoingSyncProcesses.set(key, obj);
|
|
2480
|
+
onOwnershipAbort = () => free(ownershipLifecycleController.signal.reason);
|
|
2481
|
+
ownershipLifecycleController.signal.addEventListener(
|
|
2482
|
+
"abort",
|
|
2483
|
+
onOwnershipAbort,
|
|
2484
|
+
{ once: true },
|
|
2485
|
+
);
|
|
2486
|
+
obj.deadlineTimeout = setTimeout(() => {
|
|
2487
|
+
void obj.fallbackToSimple(
|
|
2488
|
+
new Error("incoming rateless process deadline exceeded"),
|
|
2489
|
+
);
|
|
2490
|
+
}, RATELESS_PROCESS_ABSOLUTE_TIMEOUT_MS);
|
|
2491
|
+
obj.deadlineTimeout.unref?.();
|
|
2492
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2493
|
+
free(ownershipLifecycleController.signal.reason);
|
|
2494
|
+
return true;
|
|
2495
|
+
}
|
|
1376
2496
|
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
2497
|
+
let requestAllPromise: Promise<void> | undefined;
|
|
2498
|
+
obj.requestAll = () => {
|
|
2499
|
+
if (requestAllPromise) {
|
|
2500
|
+
return requestAllPromise;
|
|
2501
|
+
}
|
|
2502
|
+
requestAllPromise = (async () => {
|
|
2503
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2504
|
+
return;
|
|
2505
|
+
}
|
|
2506
|
+
fallbackSendPending = true;
|
|
2507
|
+
const sendStartedAt = syncProfileStart(profile);
|
|
2508
|
+
try {
|
|
2509
|
+
await this.simple.rpc.send(
|
|
2510
|
+
new RequestAll({
|
|
2511
|
+
syncId: message.syncId,
|
|
2512
|
+
}),
|
|
2513
|
+
{
|
|
2514
|
+
mode: new SilentDelivery({
|
|
2515
|
+
to: [obj.sender],
|
|
2516
|
+
redundancy: 1,
|
|
2517
|
+
}),
|
|
2518
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
2519
|
+
signal: controller.signal,
|
|
2520
|
+
},
|
|
2521
|
+
);
|
|
2522
|
+
if (this.isIncomingSyncProcessActive(obj) && profile) {
|
|
2523
|
+
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
2524
|
+
name: "rateless.sendRequestAll",
|
|
2525
|
+
messages: 1,
|
|
2526
|
+
targets: 1,
|
|
2527
|
+
syncId,
|
|
2528
|
+
});
|
|
2529
|
+
}
|
|
2530
|
+
} finally {
|
|
2531
|
+
fallbackSendPending = false;
|
|
2532
|
+
if (this.isIncomingSyncProcessActive(obj)) {
|
|
2533
|
+
complete();
|
|
2534
|
+
}
|
|
2535
|
+
releaseAdmissionIfSettled();
|
|
2536
|
+
}
|
|
2537
|
+
})();
|
|
2538
|
+
return requestAllPromise;
|
|
2539
|
+
};
|
|
2540
|
+
let boundedFallbackPromise: Promise<void> | undefined;
|
|
2541
|
+
obj.fallbackToSimple = (
|
|
2542
|
+
reason: unknown = new Error("incoming rateless fallback timed out"),
|
|
2543
|
+
) => {
|
|
2544
|
+
if (boundedFallbackPromise) {
|
|
2545
|
+
return boundedFallbackPromise;
|
|
2546
|
+
}
|
|
2547
|
+
const requestAll = obj.requestAll();
|
|
2548
|
+
boundedFallbackPromise = new Promise<void>((resolve) => {
|
|
2549
|
+
let settled = false;
|
|
2550
|
+
const settle = () => {
|
|
2551
|
+
if (settled) {
|
|
2552
|
+
return;
|
|
2553
|
+
}
|
|
2554
|
+
settled = true;
|
|
2555
|
+
controller.signal.removeEventListener("abort", onAbort);
|
|
2556
|
+
if (fallbackGraceTimeout) {
|
|
2557
|
+
clearTimeout(fallbackGraceTimeout);
|
|
2558
|
+
fallbackGraceTimeout = undefined;
|
|
2559
|
+
}
|
|
2560
|
+
resolve();
|
|
2561
|
+
};
|
|
2562
|
+
const onAbort = () => settle();
|
|
2563
|
+
controller.signal.addEventListener("abort", onAbort, {
|
|
2564
|
+
once: true,
|
|
2565
|
+
});
|
|
2566
|
+
fallbackGraceTimeout = setTimeout(() => {
|
|
2567
|
+
free(reason);
|
|
2568
|
+
settle();
|
|
2569
|
+
}, INCOMING_RATELESS_FALLBACK_GRACE_MS);
|
|
2570
|
+
fallbackGraceTimeout.unref?.();
|
|
2571
|
+
void requestAll.then(settle, settle);
|
|
2572
|
+
if (controller.signal.aborted) {
|
|
2573
|
+
onAbort();
|
|
2574
|
+
}
|
|
1390
2575
|
});
|
|
2576
|
+
return boundedFallbackPromise;
|
|
2577
|
+
};
|
|
2578
|
+
|
|
2579
|
+
if (message.symbols.length > MAX_MORE_SYMBOLS_BATCH_SIZE) {
|
|
2580
|
+
await obj.fallbackToSimple(
|
|
2581
|
+
new Error("oversized incoming rateless initial batch"),
|
|
2582
|
+
);
|
|
2583
|
+
return true;
|
|
1391
2584
|
}
|
|
1392
2585
|
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
2586
|
+
const wrapped = message.end < message.start;
|
|
2587
|
+
const decoderStartedAt = syncProfileStart(profile);
|
|
2588
|
+
let decoder: DecoderWrapper | false;
|
|
2589
|
+
initializationPending = true;
|
|
2590
|
+
try {
|
|
2591
|
+
decoder = await this.getLocalDecoderForRange(
|
|
1399
2592
|
{
|
|
1400
|
-
|
|
1401
|
-
|
|
2593
|
+
start1: message.start,
|
|
2594
|
+
end1: wrapped ? this.properties.numbers.maxValue : message.end,
|
|
2595
|
+
start2: 0n,
|
|
2596
|
+
end2: wrapped ? message.end : 0n,
|
|
2597
|
+
},
|
|
2598
|
+
{
|
|
2599
|
+
ownershipLifecycleController,
|
|
2600
|
+
signal: controller.signal,
|
|
1402
2601
|
},
|
|
1403
2602
|
);
|
|
2603
|
+
} catch (error) {
|
|
2604
|
+
const processAborted = controller.signal.aborted;
|
|
2605
|
+
free(error);
|
|
2606
|
+
if (
|
|
2607
|
+
processAborted ||
|
|
2608
|
+
!this.isIncomingSyncGenerationActive(ownershipLifecycleController)
|
|
2609
|
+
) {
|
|
2610
|
+
return true;
|
|
2611
|
+
}
|
|
2612
|
+
throw error;
|
|
2613
|
+
} finally {
|
|
2614
|
+
initializationPending = false;
|
|
2615
|
+
releaseAdmissionIfSettled();
|
|
2616
|
+
}
|
|
2617
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2618
|
+
decoder && decoder.free();
|
|
2619
|
+
free(controller.signal.reason);
|
|
2620
|
+
return true;
|
|
2621
|
+
}
|
|
2622
|
+
if (decoder) {
|
|
2623
|
+
// Transfer the native decoder before invoking diagnostics. A profile
|
|
2624
|
+
// sink is caller code and may throw; from this point process cleanup
|
|
2625
|
+
// owns the decoder on every exit.
|
|
2626
|
+
obj.decoder = decoder;
|
|
2627
|
+
}
|
|
2628
|
+
try {
|
|
1404
2629
|
if (profile) {
|
|
1405
|
-
emitSyncProfileDuration(profile,
|
|
1406
|
-
name: "rateless.
|
|
1407
|
-
messages: 1,
|
|
1408
|
-
targets: 1,
|
|
2630
|
+
emitSyncProfileDuration(profile, decoderStartedAt, {
|
|
2631
|
+
name: "rateless.getLocalDecoderForRange",
|
|
1409
2632
|
syncId,
|
|
2633
|
+
details: { wrapped, found: decoder !== false },
|
|
1410
2634
|
});
|
|
1411
2635
|
}
|
|
2636
|
+
} catch (error) {
|
|
2637
|
+
free(error);
|
|
2638
|
+
throw error;
|
|
2639
|
+
}
|
|
2640
|
+
|
|
2641
|
+
if (!decoder) {
|
|
2642
|
+
try {
|
|
2643
|
+
await obj.fallbackToSimple(
|
|
2644
|
+
new Error("incoming rateless decoder unavailable"),
|
|
2645
|
+
);
|
|
2646
|
+
} catch (error) {
|
|
2647
|
+
if (controller.signal.aborted) {
|
|
2648
|
+
return true;
|
|
2649
|
+
}
|
|
2650
|
+
throw error;
|
|
2651
|
+
}
|
|
1412
2652
|
return true;
|
|
1413
2653
|
}
|
|
1414
2654
|
|
|
1415
2655
|
const createTimeout = () => {
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
2656
|
+
const timeout = setTimeout(() => {
|
|
2657
|
+
void obj.fallbackToSimple(
|
|
2658
|
+
new Error("incoming rateless process timed out"),
|
|
2659
|
+
);
|
|
2660
|
+
}, INCOMING_RATELESS_IDLE_TIMEOUT_MS);
|
|
2661
|
+
timeout.unref?.();
|
|
2662
|
+
return timeout;
|
|
1420
2663
|
};
|
|
1421
2664
|
|
|
1422
2665
|
let messageQueue: {
|
|
1423
2666
|
seqNo: bigint;
|
|
1424
2667
|
symbols: CodedSymbolInput;
|
|
2668
|
+
symbolCount: number;
|
|
1425
2669
|
}[] = [];
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
2670
|
+
obj.refresh = () => {
|
|
2671
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2672
|
+
return;
|
|
2673
|
+
}
|
|
2674
|
+
if (obj.timeout) {
|
|
2675
|
+
clearTimeout(obj.timeout);
|
|
2676
|
+
}
|
|
2677
|
+
obj.timeout = createTimeout();
|
|
2678
|
+
};
|
|
2679
|
+
obj.process = async (newMessage: {
|
|
2680
|
+
seqNo: bigint;
|
|
2681
|
+
symbols: CodedSymbolInput;
|
|
2682
|
+
}): Promise<IncomingRatelessProcessResult> => {
|
|
2683
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2684
|
+
return undefined;
|
|
2685
|
+
}
|
|
2686
|
+
|
|
2687
|
+
const symbolCount = CodedSymbolBatch.from(newMessage.symbols).length;
|
|
2688
|
+
if (symbolCount > MAX_MORE_SYMBOLS_BATCH_SIZE) {
|
|
2689
|
+
free(new Error("incoming rateless symbol batch exceeds limit"));
|
|
2690
|
+
return undefined;
|
|
2691
|
+
}
|
|
2692
|
+
if (newMessage.seqNo <= obj.lastSeqNo) {
|
|
2693
|
+
return undefined;
|
|
2694
|
+
}
|
|
2695
|
+
if (
|
|
2696
|
+
newMessage.seqNo >
|
|
2697
|
+
obj.lastSeqNo + MAX_INCOMING_RATELESS_SEQUENCE_GAP
|
|
2698
|
+
) {
|
|
2699
|
+
return undefined;
|
|
2700
|
+
}
|
|
2701
|
+
if (messageQueue.some((queued) => queued.seqNo === newMessage.seqNo)) {
|
|
2702
|
+
return undefined;
|
|
2703
|
+
}
|
|
2704
|
+
if (
|
|
2705
|
+
newMessage.seqNo !== obj.lastSeqNo + 1n &&
|
|
2706
|
+
messageQueue.length >= MAX_INCOMING_RATELESS_QUEUED_BATCHES
|
|
2707
|
+
) {
|
|
2708
|
+
return undefined;
|
|
2709
|
+
}
|
|
2710
|
+
if (
|
|
2711
|
+
obj.processedSymbols + obj.queuedSymbols + symbolCount >
|
|
2712
|
+
obj.symbolBudget
|
|
2713
|
+
) {
|
|
2714
|
+
return "fallback-to-simple";
|
|
2715
|
+
}
|
|
2716
|
+
if (newMessage.seqNo > 0n && symbolCount === 0) {
|
|
2717
|
+
return "fallback-to-simple";
|
|
2718
|
+
}
|
|
2719
|
+
|
|
2720
|
+
messageQueue.push({ ...newMessage, symbolCount });
|
|
2721
|
+
obj.queuedSymbols += symbolCount;
|
|
2722
|
+
messageQueue.sort((a, b) => Number(a.seqNo - b.seqNo));
|
|
2723
|
+
if (messageQueue[0].seqNo !== obj.lastSeqNo + 1n) {
|
|
2724
|
+
return;
|
|
2725
|
+
}
|
|
2726
|
+
|
|
2727
|
+
const finalizeIfDecoded = (): boolean => {
|
|
2728
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2729
|
+
return true;
|
|
2730
|
+
}
|
|
2731
|
+
if (!decoder.decoded()) {
|
|
2732
|
+
return false;
|
|
1434
2733
|
}
|
|
1435
|
-
obj.timeout = createTimeout();
|
|
1436
|
-
},
|
|
1437
|
-
process: async (newMessage: {
|
|
1438
|
-
seqNo: bigint;
|
|
1439
|
-
symbols: CodedSymbolInput;
|
|
1440
|
-
}): Promise<boolean | undefined> => {
|
|
1441
|
-
obj.refresh(); // TODO use timestamp instead and collective pruning/refresh
|
|
1442
2734
|
|
|
1443
|
-
|
|
1444
|
-
|
|
2735
|
+
const remoteStartedAt = syncProfileStart(profile);
|
|
2736
|
+
const allMissingSymbolsInRemote = getRemoteSymbolValues(decoder);
|
|
2737
|
+
if (profile) {
|
|
2738
|
+
emitSyncProfileDuration(profile, remoteStartedAt, {
|
|
2739
|
+
name: "rateless.remoteSymbols",
|
|
2740
|
+
entries: allMissingSymbolsInRemote.length,
|
|
2741
|
+
symbols: allMissingSymbolsInRemote.length,
|
|
2742
|
+
syncId,
|
|
2743
|
+
});
|
|
1445
2744
|
}
|
|
1446
2745
|
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
2746
|
+
// The IBLT decoder is based on a local snapshot. Entries can arrive via
|
|
2747
|
+
// overlapping repair before we issue the follow-up simple request, so
|
|
2748
|
+
// re-check local presence to avoid stale duplicate bounce-back.
|
|
2749
|
+
void this.simple
|
|
2750
|
+
.queueSync(allMissingSymbolsInRemote, obj.from)
|
|
2751
|
+
.catch((error) => logger.error(error));
|
|
2752
|
+
obj.complete();
|
|
2753
|
+
return true;
|
|
2754
|
+
};
|
|
2755
|
+
|
|
2756
|
+
while (
|
|
2757
|
+
messageQueue.length > 0 &&
|
|
2758
|
+
messageQueue[0].seqNo === obj.lastSeqNo + 1n
|
|
2759
|
+
) {
|
|
2760
|
+
const symbolMessage = messageQueue.shift();
|
|
2761
|
+
if (!symbolMessage) {
|
|
2762
|
+
break;
|
|
1451
2763
|
}
|
|
1452
2764
|
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
2765
|
+
obj.queuedSymbols -= symbolMessage.symbolCount;
|
|
2766
|
+
obj.lastSeqNo = symbolMessage.seqNo;
|
|
2767
|
+
obj.processedSymbols += symbolMessage.symbolCount;
|
|
2768
|
+
// Only an authenticated, previously unseen contiguous sequence advances
|
|
2769
|
+
// the idle deadline. Replays and speculative future batches do not.
|
|
2770
|
+
obj.refresh();
|
|
2771
|
+
|
|
2772
|
+
const addBatchAndDecode:
|
|
2773
|
+
| ((symbols: BigUint64Array) => boolean)
|
|
2774
|
+
| undefined = (decoder as BatchDecoderWrapper)
|
|
2775
|
+
.add_coded_symbols_and_try_decode;
|
|
2776
|
+
if (typeof BigUint64Array !== "undefined" && addBatchAndDecode) {
|
|
2777
|
+
const flatStartedAt = syncProfileStart(profile);
|
|
2778
|
+
const flatSymbols = flatFromCodedSymbols(symbolMessage.symbols);
|
|
2779
|
+
if (profile) {
|
|
2780
|
+
emitSyncProfileDuration(profile, flatStartedAt, {
|
|
2781
|
+
name: "rateless.symbolBatchToFlat",
|
|
2782
|
+
symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
|
|
2783
|
+
syncId,
|
|
2784
|
+
});
|
|
1456
2785
|
}
|
|
1457
2786
|
|
|
1458
|
-
const
|
|
1459
|
-
const
|
|
2787
|
+
const decodeStartedAt = syncProfileStart(profile);
|
|
2788
|
+
const decoded = addBatchAndDecode.call(decoder, flatSymbols);
|
|
1460
2789
|
if (profile) {
|
|
1461
|
-
emitSyncProfileDuration(profile,
|
|
1462
|
-
name: "rateless.
|
|
1463
|
-
|
|
1464
|
-
symbols: allMissingSymbolsInRemote.length,
|
|
2790
|
+
emitSyncProfileDuration(profile, decodeStartedAt, {
|
|
2791
|
+
name: "rateless.decodeBatch",
|
|
2792
|
+
symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
|
|
1465
2793
|
syncId,
|
|
2794
|
+
details: { decoded },
|
|
1466
2795
|
});
|
|
1467
2796
|
}
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
// overlapping repair before we issue the follow-up simple request, so
|
|
1471
|
-
// re-check local presence to avoid stale duplicate bounce-back.
|
|
1472
|
-
this.simple.queueSync(allMissingSymbolsInRemote, context.from!);
|
|
1473
|
-
obj.free();
|
|
1474
|
-
return true;
|
|
1475
|
-
};
|
|
1476
|
-
|
|
1477
|
-
while (
|
|
1478
|
-
messageQueue.length > 0 &&
|
|
1479
|
-
messageQueue[0].seqNo === lastSeqNo + 1n
|
|
1480
|
-
) {
|
|
1481
|
-
const symbolMessage = messageQueue.shift();
|
|
1482
|
-
if (!symbolMessage) {
|
|
1483
|
-
break;
|
|
2797
|
+
if (decoded && finalizeIfDecoded()) {
|
|
2798
|
+
return true;
|
|
1484
2799
|
}
|
|
2800
|
+
continue;
|
|
2801
|
+
}
|
|
1485
2802
|
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
const decodeStartedAt = syncProfileStart(profile);
|
|
1504
|
-
const decoded = addBatchAndDecode.call(decoder, flatSymbols);
|
|
1505
|
-
if (profile) {
|
|
1506
|
-
emitSyncProfileDuration(profile, decodeStartedAt, {
|
|
1507
|
-
name: "rateless.decodeBatch",
|
|
1508
|
-
symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
|
|
1509
|
-
syncId,
|
|
1510
|
-
details: { decoded },
|
|
1511
|
-
});
|
|
1512
|
-
}
|
|
1513
|
-
if (decoded && finalizeIfDecoded()) {
|
|
2803
|
+
const decodeLoopStartedAt = syncProfileStart(profile);
|
|
2804
|
+
let symbolsProcessed = 0;
|
|
2805
|
+
for (const symbol of CodedSymbolBatch.from(symbolMessage.symbols)) {
|
|
2806
|
+
symbolsProcessed += 1;
|
|
2807
|
+
const normalizedSymbol =
|
|
2808
|
+
symbol instanceof SymbolSerialized
|
|
2809
|
+
? symbol
|
|
2810
|
+
: new SymbolSerialized({
|
|
2811
|
+
count: symbol.count,
|
|
2812
|
+
hash: symbol.hash,
|
|
2813
|
+
symbol: symbol.symbol,
|
|
2814
|
+
});
|
|
2815
|
+
|
|
2816
|
+
decoder.add_coded_symbol(normalizedSymbol);
|
|
2817
|
+
try {
|
|
2818
|
+
decoder.try_decode();
|
|
2819
|
+
if (finalizeIfDecoded()) {
|
|
1514
2820
|
return true;
|
|
1515
2821
|
}
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
? symbol
|
|
1526
|
-
: new SymbolSerialized({
|
|
1527
|
-
count: symbol.count,
|
|
1528
|
-
hash: symbol.hash,
|
|
1529
|
-
symbol: symbol.symbol,
|
|
1530
|
-
});
|
|
1531
|
-
|
|
1532
|
-
decoder.add_coded_symbol(normalizedSymbol);
|
|
1533
|
-
try {
|
|
1534
|
-
decoder.try_decode();
|
|
1535
|
-
if (finalizeIfDecoded()) {
|
|
1536
|
-
return true;
|
|
1537
|
-
}
|
|
1538
|
-
} catch (error: any) {
|
|
1539
|
-
if (
|
|
1540
|
-
error?.message === "Invalid degree" ||
|
|
1541
|
-
error === "Invalid degree"
|
|
1542
|
-
) {
|
|
1543
|
-
logger.trace(
|
|
1544
|
-
"Decoder reported invalid degree; waiting for more symbols",
|
|
1545
|
-
);
|
|
1546
|
-
continue;
|
|
1547
|
-
}
|
|
1548
|
-
throw error;
|
|
2822
|
+
} catch (error: any) {
|
|
2823
|
+
if (
|
|
2824
|
+
error?.message === "Invalid degree" ||
|
|
2825
|
+
error === "Invalid degree"
|
|
2826
|
+
) {
|
|
2827
|
+
logger.trace(
|
|
2828
|
+
"Decoder reported invalid degree; waiting for more symbols",
|
|
2829
|
+
);
|
|
2830
|
+
continue;
|
|
1549
2831
|
}
|
|
1550
|
-
|
|
1551
|
-
if (profile) {
|
|
1552
|
-
emitSyncProfileDuration(profile, decodeLoopStartedAt, {
|
|
1553
|
-
name: "rateless.decodeSymbolLoop",
|
|
1554
|
-
symbols: symbolsProcessed,
|
|
1555
|
-
syncId,
|
|
1556
|
-
});
|
|
2832
|
+
throw error;
|
|
1557
2833
|
}
|
|
1558
2834
|
}
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
2835
|
+
if (profile) {
|
|
2836
|
+
emitSyncProfileDuration(profile, decodeLoopStartedAt, {
|
|
2837
|
+
name: "rateless.decodeSymbolLoop",
|
|
2838
|
+
symbols: symbolsProcessed,
|
|
2839
|
+
syncId,
|
|
2840
|
+
});
|
|
2841
|
+
}
|
|
2842
|
+
}
|
|
2843
|
+
return false;
|
|
1566
2844
|
};
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
2845
|
+
obj.timeout = createTimeout();
|
|
2846
|
+
let initialResult: IncomingRatelessProcessResult;
|
|
2847
|
+
try {
|
|
2848
|
+
initialResult = await obj.process({
|
|
2849
|
+
seqNo: 0n,
|
|
2850
|
+
symbols: message.symbols,
|
|
2851
|
+
});
|
|
2852
|
+
} catch (error) {
|
|
2853
|
+
const wasActive = this.isIncomingSyncProcessActive(obj);
|
|
2854
|
+
free(error);
|
|
2855
|
+
if (!wasActive) {
|
|
2856
|
+
return true;
|
|
2857
|
+
}
|
|
2858
|
+
throw error;
|
|
2859
|
+
}
|
|
2860
|
+
if (initialResult === true) {
|
|
2861
|
+
return true;
|
|
2862
|
+
}
|
|
2863
|
+
if (initialResult === "fallback-to-simple") {
|
|
2864
|
+
await obj.fallbackToSimple(
|
|
2865
|
+
new Error("incoming rateless symbol budget exhausted"),
|
|
2866
|
+
);
|
|
2867
|
+
return true;
|
|
2868
|
+
}
|
|
2869
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
1571
2870
|
return true;
|
|
1572
2871
|
}
|
|
1573
2872
|
|
|
1574
2873
|
// not done, request more symbols
|
|
1575
2874
|
const sendStartedAt = syncProfileStart(profile);
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
2875
|
+
try {
|
|
2876
|
+
await this.simple.rpc.send(
|
|
2877
|
+
new RequestMoreSymbols({
|
|
2878
|
+
lastSeqNo: obj.lastSeqNo,
|
|
2879
|
+
syncId: message.syncId,
|
|
2880
|
+
}),
|
|
2881
|
+
{
|
|
2882
|
+
mode: new SilentDelivery({ to: [obj.sender], redundancy: 1 }),
|
|
2883
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
2884
|
+
signal: controller.signal,
|
|
2885
|
+
},
|
|
2886
|
+
);
|
|
2887
|
+
} catch (error) {
|
|
2888
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2889
|
+
return true;
|
|
2890
|
+
}
|
|
2891
|
+
free(error);
|
|
2892
|
+
throw error;
|
|
2893
|
+
}
|
|
2894
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2895
|
+
return true;
|
|
2896
|
+
}
|
|
1586
2897
|
if (profile) {
|
|
1587
2898
|
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
1588
2899
|
name: "rateless.sendRequestMoreSymbols",
|
|
@@ -1594,50 +2905,89 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1594
2905
|
|
|
1595
2906
|
return true;
|
|
1596
2907
|
} else if (message instanceof MoreSymbols) {
|
|
2908
|
+
const from = context.from;
|
|
2909
|
+
if (!from) {
|
|
2910
|
+
return true;
|
|
2911
|
+
}
|
|
2912
|
+
const sender = from.hashcode();
|
|
1597
2913
|
const syncId = getSyncIdString(message);
|
|
1598
|
-
const
|
|
1599
|
-
|
|
2914
|
+
const key = this.getIncomingSyncProcessKey(sender, syncId);
|
|
2915
|
+
const obj = this.ingoingSyncProcesses.get(key);
|
|
2916
|
+
if (
|
|
2917
|
+
!obj ||
|
|
2918
|
+
obj.sender !== sender ||
|
|
2919
|
+
!this.isIncomingSyncProcessActive(obj)
|
|
2920
|
+
) {
|
|
1600
2921
|
return true;
|
|
1601
2922
|
}
|
|
1602
|
-
|
|
2923
|
+
let outProcess: IncomingRatelessProcessResult;
|
|
2924
|
+
try {
|
|
2925
|
+
outProcess = await obj.process(message);
|
|
2926
|
+
} catch (error) {
|
|
2927
|
+
const wasActive = this.isIncomingSyncProcessActive(obj);
|
|
2928
|
+
obj.free(error);
|
|
2929
|
+
if (!wasActive) {
|
|
2930
|
+
return true;
|
|
2931
|
+
}
|
|
2932
|
+
throw error;
|
|
2933
|
+
}
|
|
1603
2934
|
|
|
1604
2935
|
if (outProcess === true) {
|
|
1605
2936
|
return true;
|
|
2937
|
+
} else if (outProcess === "fallback-to-simple") {
|
|
2938
|
+
try {
|
|
2939
|
+
await obj.fallbackToSimple(
|
|
2940
|
+
new Error("incoming rateless symbol budget exhausted"),
|
|
2941
|
+
);
|
|
2942
|
+
} catch {
|
|
2943
|
+
// The bounded rateless process is already complete or aborted. A
|
|
2944
|
+
// later repair round may retry if the fallback request was lost.
|
|
2945
|
+
}
|
|
2946
|
+
return true;
|
|
1606
2947
|
} else if (outProcess === undefined) {
|
|
1607
2948
|
return true; // we don't have enough information, or received information that is redundant
|
|
1608
2949
|
}
|
|
2950
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2951
|
+
return true;
|
|
2952
|
+
}
|
|
1609
2953
|
|
|
1610
2954
|
// we are not done
|
|
1611
2955
|
|
|
1612
2956
|
const sendStartedAt = syncProfileStart(profile);
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
void Promise.resolve(sendResult).then(
|
|
1625
|
-
() =>
|
|
1626
|
-
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
1627
|
-
name: "rateless.sendRequestMoreSymbols",
|
|
1628
|
-
messages: 1,
|
|
1629
|
-
targets: 1,
|
|
1630
|
-
syncId,
|
|
1631
|
-
}),
|
|
1632
|
-
() =>
|
|
1633
|
-
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
1634
|
-
name: "rateless.sendRequestMoreSymbols",
|
|
1635
|
-
messages: 1,
|
|
1636
|
-
targets: 1,
|
|
1637
|
-
syncId,
|
|
1638
|
-
details: { rejected: true },
|
|
1639
|
-
}),
|
|
2957
|
+
try {
|
|
2958
|
+
await this.simple.rpc.send(
|
|
2959
|
+
new RequestMoreSymbols({
|
|
2960
|
+
lastSeqNo: obj.lastSeqNo,
|
|
2961
|
+
syncId: message.syncId,
|
|
2962
|
+
}),
|
|
2963
|
+
{
|
|
2964
|
+
mode: new SilentDelivery({ to: [obj.sender], redundancy: 1 }),
|
|
2965
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
2966
|
+
signal: obj.controller.signal,
|
|
2967
|
+
},
|
|
1640
2968
|
);
|
|
2969
|
+
} catch {
|
|
2970
|
+
if (profile) {
|
|
2971
|
+
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
2972
|
+
name: "rateless.sendRequestMoreSymbols",
|
|
2973
|
+
messages: 1,
|
|
2974
|
+
targets: 1,
|
|
2975
|
+
syncId,
|
|
2976
|
+
details: { rejected: true },
|
|
2977
|
+
});
|
|
2978
|
+
}
|
|
2979
|
+
return true;
|
|
2980
|
+
}
|
|
2981
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2982
|
+
return true;
|
|
2983
|
+
}
|
|
2984
|
+
if (profile) {
|
|
2985
|
+
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
2986
|
+
name: "rateless.sendRequestMoreSymbols",
|
|
2987
|
+
messages: 1,
|
|
2988
|
+
targets: 1,
|
|
2989
|
+
syncId,
|
|
2990
|
+
});
|
|
1641
2991
|
}
|
|
1642
2992
|
|
|
1643
2993
|
return true;
|
|
@@ -1647,19 +2997,41 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1647
2997
|
if (!obj) {
|
|
1648
2998
|
return true;
|
|
1649
2999
|
}
|
|
1650
|
-
|
|
3000
|
+
if (context.from?.hashcode() !== obj.target) {
|
|
3001
|
+
return true;
|
|
3002
|
+
}
|
|
3003
|
+
const signal = obj.signal;
|
|
3004
|
+
if (signal.aborted) {
|
|
3005
|
+
return true;
|
|
3006
|
+
}
|
|
3007
|
+
const next = obj.next(message);
|
|
3008
|
+
if (!next) {
|
|
3009
|
+
return true;
|
|
3010
|
+
}
|
|
3011
|
+
const { symbols, exhaustedAfterSend } = next;
|
|
3012
|
+
if (signal.aborted) {
|
|
3013
|
+
return true;
|
|
3014
|
+
}
|
|
1651
3015
|
const sendStartedAt = syncProfileStart(profile);
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
3016
|
+
try {
|
|
3017
|
+
await this.properties.rpc.send(
|
|
3018
|
+
new MoreSymbols({
|
|
3019
|
+
lastSeqNo: message.lastSeqNo,
|
|
3020
|
+
syncId: message.syncId,
|
|
3021
|
+
symbols,
|
|
3022
|
+
}),
|
|
3023
|
+
{
|
|
3024
|
+
mode: new SilentDelivery({ to: [obj.target], redundancy: 1 }),
|
|
3025
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
3026
|
+
signal,
|
|
3027
|
+
},
|
|
3028
|
+
);
|
|
3029
|
+
} catch (error) {
|
|
3030
|
+
if (signal?.aborted) {
|
|
3031
|
+
return true;
|
|
3032
|
+
}
|
|
3033
|
+
throw error;
|
|
3034
|
+
}
|
|
1663
3035
|
if (profile) {
|
|
1664
3036
|
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
1665
3037
|
name: "rateless.sendMoreSymbols",
|
|
@@ -1669,17 +3041,154 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1669
3041
|
syncId,
|
|
1670
3042
|
});
|
|
1671
3043
|
}
|
|
3044
|
+
if (exhaustedAfterSend) {
|
|
3045
|
+
// Latch exhaustion in next() before the send begins, then retain this
|
|
3046
|
+
// bounded process long enough for RequestAll. Starting Simple eagerly
|
|
3047
|
+
// also keeps fallback working with older receivers that ignore an empty
|
|
3048
|
+
// terminal symbol batch.
|
|
3049
|
+
void obj.startSimpleFallback().catch((error) => logger.error(error));
|
|
3050
|
+
}
|
|
1672
3051
|
return true;
|
|
1673
3052
|
} else if (message instanceof RequestAll) {
|
|
1674
3053
|
const p = this.outgoingSyncProcesses.get(getSyncIdString(message));
|
|
1675
3054
|
if (!p) {
|
|
1676
3055
|
return true;
|
|
1677
3056
|
}
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
3057
|
+
if (context.from?.hashcode() !== p.target) {
|
|
3058
|
+
return true;
|
|
3059
|
+
}
|
|
3060
|
+
if (p.signal.aborted) {
|
|
3061
|
+
return true;
|
|
3062
|
+
}
|
|
3063
|
+
// RequestAll ends only this target's rateless attempt. Other target
|
|
3064
|
+
// encoders and response authorizations remain independently owned.
|
|
3065
|
+
const fallback = p.startSimpleFallback();
|
|
3066
|
+
p.free();
|
|
3067
|
+
await fallback;
|
|
1682
3068
|
return true;
|
|
3069
|
+
} else if (
|
|
3070
|
+
message instanceof ResponseMaybeSync ||
|
|
3071
|
+
message instanceof ResponseMaybeSyncCapabilities
|
|
3072
|
+
) {
|
|
3073
|
+
const from = context.from!;
|
|
3074
|
+
// Simple authorizations are exact request leases and take precedence
|
|
3075
|
+
// over Rateless' broader process authorization. This also handles a
|
|
3076
|
+
// delayed fallback/prelude response after the Rateless process for the
|
|
3077
|
+
// same target has been replaced.
|
|
3078
|
+
const simpleLeases = this.simple.consumeAuthorizedMaybeSyncResponse(
|
|
3079
|
+
message.hashes,
|
|
3080
|
+
from,
|
|
3081
|
+
);
|
|
3082
|
+
const simpleHashes = simpleLeases.flatMap((lease) => lease.hashes);
|
|
3083
|
+
const simpleHashSet = new Set(simpleHashes);
|
|
3084
|
+
const ratelessCandidateHashes: string[] = [];
|
|
3085
|
+
let inspected = 0;
|
|
3086
|
+
const iterator = message.hashes[Symbol.iterator]();
|
|
3087
|
+
let exhausted = false;
|
|
3088
|
+
try {
|
|
3089
|
+
while (inspected < MAX_RATELESS_RESPONSE_HASHES_INSPECTED) {
|
|
3090
|
+
const next = iterator.next();
|
|
3091
|
+
if (next.done) {
|
|
3092
|
+
exhausted = true;
|
|
3093
|
+
break;
|
|
3094
|
+
}
|
|
3095
|
+
inspected += 1;
|
|
3096
|
+
if (!simpleHashSet.has(next.value)) {
|
|
3097
|
+
ratelessCandidateHashes.push(next.value);
|
|
3098
|
+
}
|
|
3099
|
+
}
|
|
3100
|
+
} finally {
|
|
3101
|
+
if (!exhausted) {
|
|
3102
|
+
iterator.return?.();
|
|
3103
|
+
}
|
|
3104
|
+
}
|
|
3105
|
+
const response =
|
|
3106
|
+
ratelessCandidateHashes.length > 0
|
|
3107
|
+
? this.consumeAuthorizedRatelessResponse(
|
|
3108
|
+
ratelessCandidateHashes,
|
|
3109
|
+
from,
|
|
3110
|
+
)
|
|
3111
|
+
: undefined;
|
|
3112
|
+
if (
|
|
3113
|
+
simpleLeases.length === 0 &&
|
|
3114
|
+
(!response || response.authorized.length === 0)
|
|
3115
|
+
) {
|
|
3116
|
+
response?.release();
|
|
3117
|
+
return true;
|
|
3118
|
+
}
|
|
3119
|
+
|
|
3120
|
+
let firstError: unknown;
|
|
3121
|
+
let rollbackRatelessAuthorization = false;
|
|
3122
|
+
try {
|
|
3123
|
+
const simpleMessage =
|
|
3124
|
+
message instanceof ResponseMaybeSyncCapabilities
|
|
3125
|
+
? new ResponseMaybeSyncCapabilities({
|
|
3126
|
+
hashes: simpleHashes,
|
|
3127
|
+
capabilities: message.capabilities,
|
|
3128
|
+
})
|
|
3129
|
+
: new ResponseMaybeSync({ hashes: simpleHashes });
|
|
3130
|
+
if (response && response.authorized.length > 0) {
|
|
3131
|
+
const responseStartedAt = syncProfileStart(profile);
|
|
3132
|
+
let responseShipment = {
|
|
3133
|
+
messages: 0,
|
|
3134
|
+
fused: false,
|
|
3135
|
+
entries: 0,
|
|
3136
|
+
};
|
|
3137
|
+
try {
|
|
3138
|
+
responseShipment =
|
|
3139
|
+
await this.simple.shipAuthorizedMaybeSyncResponse({
|
|
3140
|
+
hashes: response.authorized,
|
|
3141
|
+
from,
|
|
3142
|
+
response: message,
|
|
3143
|
+
signal: response.signal,
|
|
3144
|
+
});
|
|
3145
|
+
} catch (error) {
|
|
3146
|
+
firstError = error;
|
|
3147
|
+
rollbackRatelessAuthorization = true;
|
|
3148
|
+
}
|
|
3149
|
+
if (profile) {
|
|
3150
|
+
try {
|
|
3151
|
+
emitSyncProfileDuration(profile, responseStartedAt, {
|
|
3152
|
+
name: "simple.exchangeHeads",
|
|
3153
|
+
entries: responseShipment.entries,
|
|
3154
|
+
messages: responseShipment.messages,
|
|
3155
|
+
targets: 1,
|
|
3156
|
+
details: {
|
|
3157
|
+
source: "ratelessResponseMaybeSync",
|
|
3158
|
+
fused: responseShipment.fused,
|
|
3159
|
+
},
|
|
3160
|
+
});
|
|
3161
|
+
} catch (error) {
|
|
3162
|
+
firstError ??= error;
|
|
3163
|
+
}
|
|
3164
|
+
}
|
|
3165
|
+
}
|
|
3166
|
+
|
|
3167
|
+
if (simpleLeases.length > 0) {
|
|
3168
|
+
try {
|
|
3169
|
+
await this.simple.shipAuthorizedMaybeSyncResponseLeases({
|
|
3170
|
+
leases: simpleLeases,
|
|
3171
|
+
from,
|
|
3172
|
+
response: simpleMessage,
|
|
3173
|
+
});
|
|
3174
|
+
} catch (error) {
|
|
3175
|
+
firstError ??= error;
|
|
3176
|
+
}
|
|
3177
|
+
}
|
|
3178
|
+
if (firstError !== undefined) {
|
|
3179
|
+
throw firstError;
|
|
3180
|
+
}
|
|
3181
|
+
return true;
|
|
3182
|
+
} finally {
|
|
3183
|
+
response?.release({ rollback: rollbackRatelessAuthorization });
|
|
3184
|
+
// The Simple helper owns these leases once entered and releases each
|
|
3185
|
+
// one in its own finally. Keep this outer ownership release as the
|
|
3186
|
+
// final safety net for diagnostics or setup failures that happen
|
|
3187
|
+
// before the helper can take responsibility.
|
|
3188
|
+
for (const lease of simpleLeases) {
|
|
3189
|
+
lease.release();
|
|
3190
|
+
}
|
|
3191
|
+
}
|
|
1683
3192
|
}
|
|
1684
3193
|
return this.simple.onMessage(message, context);
|
|
1685
3194
|
}
|
|
@@ -1724,18 +3233,47 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1724
3233
|
}
|
|
1725
3234
|
|
|
1726
3235
|
onPeerDisconnected(key: PublicSignKey | string) {
|
|
1727
|
-
|
|
3236
|
+
const target = typeof key === "string" ? key : key.hashcode();
|
|
3237
|
+
for (const process of [...this.ingoingSyncProcesses.values()]) {
|
|
3238
|
+
if (process.sender === target) {
|
|
3239
|
+
process.free(new Error("incoming rateless peer disconnected"));
|
|
3240
|
+
}
|
|
3241
|
+
}
|
|
3242
|
+
for (const targetLifecycle of [
|
|
3243
|
+
...(this.ratelessDispatchTargets.get(target) ?? []),
|
|
3244
|
+
]) {
|
|
3245
|
+
this.abortRatelessDispatchTarget(
|
|
3246
|
+
targetLifecycle,
|
|
3247
|
+
new Error("rateless sync target disconnected"),
|
|
3248
|
+
);
|
|
3249
|
+
}
|
|
3250
|
+
return this.simple.onPeerDisconnected(target);
|
|
1728
3251
|
}
|
|
1729
3252
|
|
|
1730
3253
|
open(): Promise<void> | void {
|
|
3254
|
+
const reason = new Error("rateless sync generation replaced");
|
|
3255
|
+
this.cancelRatelessRepairSessions(reason);
|
|
3256
|
+
this.ratelessDispatchLifecycleController.abort(reason);
|
|
3257
|
+
this.ratelessDispatchLifecycleController = new AbortController();
|
|
3258
|
+
this.startedOrCompletedSynchronizations = new Cache({ max: 1e4 });
|
|
3259
|
+
// Admission accounting intentionally spans local generations: native range
|
|
3260
|
+
// initialization and fallback delivery are not guaranteed to stop merely
|
|
3261
|
+
// because their logical process was aborted.
|
|
3262
|
+
this.ratelessClosed = false;
|
|
1731
3263
|
return this.simple.open();
|
|
1732
3264
|
}
|
|
1733
3265
|
|
|
1734
3266
|
close(): Promise<void> | void {
|
|
1735
|
-
|
|
3267
|
+
this.ratelessClosed = true;
|
|
3268
|
+
// Abort ownership first. Process abort listeners then cancel any in-flight
|
|
3269
|
+
// StartSync/MoreSymbols send before they detach or free native state.
|
|
3270
|
+
const reason = new Error("rateless synchronizer closed");
|
|
3271
|
+
this.cancelRatelessRepairSessions(reason);
|
|
3272
|
+
this.ratelessDispatchLifecycleController.abort(reason);
|
|
3273
|
+
for (const obj of [...this.ingoingSyncProcesses.values()]) {
|
|
1736
3274
|
obj.free();
|
|
1737
3275
|
}
|
|
1738
|
-
for (const
|
|
3276
|
+
for (const obj of [...this.outgoingSyncProcesses.values()]) {
|
|
1739
3277
|
obj.free();
|
|
1740
3278
|
}
|
|
1741
3279
|
this.clearLocalRangeEncoderCache();
|