@peerbit/shared-log 13.2.12 → 13.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark/receive-prune.js +31 -29
- package/dist/benchmark/receive-prune.js.map +1 -1
- package/dist/src/checked-prune.d.ts +43 -10
- package/dist/src/checked-prune.d.ts.map +1 -1
- package/dist/src/checked-prune.js +257 -27
- package/dist/src/checked-prune.js.map +1 -1
- package/dist/src/exchange-heads.d.ts +26 -1
- package/dist/src/exchange-heads.d.ts.map +1 -1
- package/dist/src/exchange-heads.js +114 -36
- package/dist/src/exchange-heads.js.map +1 -1
- package/dist/src/index.d.ts +71 -14
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3492 -1493
- package/dist/src/index.js.map +1 -1
- package/dist/src/sync/index.d.ts +2 -0
- package/dist/src/sync/index.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.d.ts +96 -20
- package/dist/src/sync/rateless-iblt.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.js +1616 -428
- package/dist/src/sync/rateless-iblt.js.map +1 -1
- package/dist/src/sync/simple.d.ts +176 -1
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +2919 -525
- package/dist/src/sync/simple.js.map +1 -1
- package/package.json +11 -11
- package/src/checked-prune.ts +338 -29
- package/src/exchange-heads.ts +74 -37
- package/src/index.ts +6370 -2792
- package/src/sync/index.ts +2 -1
- package/src/sync/rateless-iblt.ts +2036 -498
- package/src/sync/simple.ts +3779 -634
|
@@ -43,7 +43,7 @@ import {} from "../exchange-heads.js";
|
|
|
43
43
|
import { TransportMessage } from "../message.js";
|
|
44
44
|
import {} from "../ranges.js";
|
|
45
45
|
import { emitSyncProfileDuration, emitSyncProfileEvent, syncProfileStart, } from "./profile.js";
|
|
46
|
-
import { SYNC_MESSAGE_PRIORITY, SimpleSyncronizer, } from "./simple.js";
|
|
46
|
+
import { ResponseMaybeSync, ResponseMaybeSyncCapabilities, SYNC_MESSAGE_PRIORITY, SimpleSyncronizer, } from "./simple.js";
|
|
47
47
|
export const logger = loggerFn("peerbit:shared-log:rateless");
|
|
48
48
|
const coerceBigInt = (value) => typeof value === "bigint" ? value : BigInt(value);
|
|
49
49
|
let SymbolSerialized = (() => {
|
|
@@ -81,6 +81,7 @@ let SymbolSerialized = (() => {
|
|
|
81
81
|
const CODED_SYMBOL_WORDS = 3;
|
|
82
82
|
const CODED_SYMBOL_WORD_BYTES = 8;
|
|
83
83
|
const CODED_SYMBOL_BYTES = CODED_SYMBOL_WORDS * CODED_SYMBOL_WORD_BYTES;
|
|
84
|
+
const MAX_CODED_SYMBOL_BATCH_SIZE = 1_024;
|
|
84
85
|
const BIG_UINT64_ARRAY_IS_LITTLE_ENDIAN = typeof BigUint64Array !== "undefined" &&
|
|
85
86
|
new Uint8Array(new BigUint64Array([1n]).buffer)[0] === 1;
|
|
86
87
|
const assertValidFlatCodedSymbols = (flat) => {
|
|
@@ -191,6 +192,9 @@ const codedSymbolBatchField = {
|
|
|
191
192
|
},
|
|
192
193
|
deserialize: (reader) => {
|
|
193
194
|
const length = reader.u32();
|
|
195
|
+
if (length > MAX_CODED_SYMBOL_BATCH_SIZE) {
|
|
196
|
+
throw new Error("RIBLT coded symbol batch exceeds the receiver limit");
|
|
197
|
+
}
|
|
194
198
|
const wordLength = length * CODED_SYMBOL_WORDS;
|
|
195
199
|
const byteLength = length * CODED_SYMBOL_BYTES;
|
|
196
200
|
if (typeof BigUint64Array !== "undefined" &&
|
|
@@ -356,7 +360,23 @@ const DEFAULT_CONVERGENT_REPAIR_TIMEOUT_MS = 30_000;
|
|
|
356
360
|
const DEFAULT_CONVERGENT_RETRY_INTERVALS_MS = [0, 1_000, 3_000, 7_000];
|
|
357
361
|
const DEFAULT_MAX_CONVERGENT_TRACKED_HASHES = 4_096;
|
|
358
362
|
const MIN_MORE_SYMBOLS_BATCH_SIZE = 64;
|
|
359
|
-
const MAX_MORE_SYMBOLS_BATCH_SIZE =
|
|
363
|
+
const MAX_MORE_SYMBOLS_BATCH_SIZE = MAX_CODED_SYMBOL_BATCH_SIZE;
|
|
364
|
+
const MAX_INCOMING_RATELESS_PROCESSES = 32;
|
|
365
|
+
const MAX_INCOMING_RATELESS_PROCESSES_PER_SENDER = 4;
|
|
366
|
+
const MAX_INCOMING_RATELESS_QUEUED_BATCHES = 8;
|
|
367
|
+
const MAX_INCOMING_RATELESS_SEQUENCE_GAP = BigInt(MAX_INCOMING_RATELESS_QUEUED_BATCHES);
|
|
368
|
+
const MIN_RATELESS_SYMBOL_BUDGET = 4_096;
|
|
369
|
+
const MAX_RATELESS_SYMBOL_BUDGET = 262_144;
|
|
370
|
+
const RATELESS_SYMBOL_BUDGET_MULTIPLIER = 4;
|
|
371
|
+
const INCOMING_RATELESS_IDLE_TIMEOUT_MS = 20_000;
|
|
372
|
+
const OUTGOING_RATELESS_IDLE_TIMEOUT_MS = 10_000;
|
|
373
|
+
const RATELESS_PROCESS_ABSOLUTE_TIMEOUT_MS = 120_000;
|
|
374
|
+
const INCOMING_RATELESS_FALLBACK_GRACE_MS = 5_000;
|
|
375
|
+
const MAX_RATELESS_RESPONSE_HASHES_INSPECTED = 10_000;
|
|
376
|
+
export const MAX_ACTIVE_RATELESS_RESPONSES_PER_PEER = 4;
|
|
377
|
+
export const MAX_ACTIVE_RATELESS_RESPONSES_GLOBAL = 32;
|
|
378
|
+
const getIncomingRatelessSymbolBudget = (initialSymbols) => Math.min(MAX_RATELESS_SYMBOL_BUDGET, Math.max(MIN_RATELESS_SYMBOL_BUDGET, initialSymbols * initialSymbols * RATELESS_SYMBOL_BUDGET_MULTIPLIER));
|
|
379
|
+
const getOutgoingRatelessSymbolBudget = (entries, initialSymbols) => Math.min(MAX_RATELESS_SYMBOL_BUDGET, Math.max(MIN_RATELESS_SYMBOL_BUDGET, initialSymbols, entries * RATELESS_SYMBOL_BUDGET_MULTIPLIER));
|
|
360
380
|
let StartSync = (() => {
|
|
361
381
|
let _classDecorators = [variant([3, 0])];
|
|
362
382
|
let _classDescriptor;
|
|
@@ -556,97 +576,122 @@ const matchEntriesByHashNumberInRangeQuery = (range) => {
|
|
|
556
576
|
const buildEncoderOrDecoderFromRange = async (ranges, entryIndex, type, profile, resolveHashNumbersInRange) => {
|
|
557
577
|
await ribltReady;
|
|
558
578
|
const encoder = type === "encoder" ? new EncoderWrapper() : new DecoderWrapper();
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
source = "native";
|
|
570
|
-
hashNumbers =
|
|
571
|
-
typeof BigUint64Array !== "undefined" &&
|
|
572
|
-
resolved instanceof BigUint64Array
|
|
573
|
-
? resolved
|
|
574
|
-
: [...resolved];
|
|
575
|
-
}
|
|
576
|
-
else {
|
|
577
|
-
const entries = await entryIndex
|
|
578
|
-
.iterate({
|
|
579
|
-
// Range sync for IBLT is done in hashNumber space.
|
|
580
|
-
query: matchEntriesByHashNumberInRangeQuery({
|
|
581
|
-
end1: ranges.end1,
|
|
582
|
-
start1: ranges.start1,
|
|
583
|
-
end2: ranges.end2,
|
|
584
|
-
start2: ranges.start2,
|
|
585
|
-
}),
|
|
586
|
-
}, {
|
|
587
|
-
shape: {
|
|
588
|
-
hash: true,
|
|
589
|
-
hashNumber: true,
|
|
590
|
-
},
|
|
591
|
-
})
|
|
592
|
-
.all();
|
|
593
|
-
hashNumbers = entries.map((entry) => entry.value.hashNumber);
|
|
594
|
-
}
|
|
595
|
-
if (profile) {
|
|
596
|
-
emitSyncProfileDuration(profile, rangeQueryStartedAt, {
|
|
597
|
-
name: "rateless.rangeQuery",
|
|
598
|
-
entries: hashNumbers.length,
|
|
599
|
-
details: { type, source },
|
|
579
|
+
let transferred = false;
|
|
580
|
+
try {
|
|
581
|
+
const rangeQueryStartedAt = syncProfileStart(profile);
|
|
582
|
+
let hashNumbers;
|
|
583
|
+
let source = "index";
|
|
584
|
+
const resolved = await resolveHashNumbersInRange?.({
|
|
585
|
+
end1: ranges.end1,
|
|
586
|
+
start1: ranges.start1,
|
|
587
|
+
end2: ranges.end2,
|
|
588
|
+
start2: ranges.start2,
|
|
600
589
|
});
|
|
590
|
+
if (resolved) {
|
|
591
|
+
source = "native";
|
|
592
|
+
hashNumbers =
|
|
593
|
+
typeof BigUint64Array !== "undefined" &&
|
|
594
|
+
resolved instanceof BigUint64Array
|
|
595
|
+
? resolved
|
|
596
|
+
: [...resolved];
|
|
597
|
+
}
|
|
598
|
+
else {
|
|
599
|
+
const entries = await entryIndex
|
|
600
|
+
.iterate({
|
|
601
|
+
// Range sync for IBLT is done in hashNumber space.
|
|
602
|
+
query: matchEntriesByHashNumberInRangeQuery({
|
|
603
|
+
end1: ranges.end1,
|
|
604
|
+
start1: ranges.start1,
|
|
605
|
+
end2: ranges.end2,
|
|
606
|
+
start2: ranges.start2,
|
|
607
|
+
}),
|
|
608
|
+
}, {
|
|
609
|
+
shape: {
|
|
610
|
+
hash: true,
|
|
611
|
+
hashNumber: true,
|
|
612
|
+
},
|
|
613
|
+
})
|
|
614
|
+
.all();
|
|
615
|
+
hashNumbers = entries.map((entry) => entry.value.hashNumber);
|
|
616
|
+
}
|
|
617
|
+
if (profile) {
|
|
618
|
+
emitSyncProfileDuration(profile, rangeQueryStartedAt, {
|
|
619
|
+
name: "rateless.rangeQuery",
|
|
620
|
+
entries: hashNumbers.length,
|
|
621
|
+
details: { type, source },
|
|
622
|
+
});
|
|
623
|
+
}
|
|
624
|
+
if (hashNumbers.length === 0) {
|
|
625
|
+
return false;
|
|
626
|
+
}
|
|
627
|
+
const addSymbolsStartedAt = syncProfileStart(profile);
|
|
628
|
+
if (typeof BigUint64Array !== "undefined" &&
|
|
629
|
+
typeof encoder.add_symbols === "function") {
|
|
630
|
+
const symbols = hashNumbers instanceof BigUint64Array
|
|
631
|
+
? hashNumbers
|
|
632
|
+
: BigUint64Array.from(hashNumbers, coerceBigInt);
|
|
633
|
+
addSymbolsToRiblt(encoder, symbols);
|
|
634
|
+
}
|
|
635
|
+
else {
|
|
636
|
+
for (const hashNumber of hashNumbers) {
|
|
637
|
+
encoder.add_symbol(coerceBigInt(hashNumber));
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
if (profile) {
|
|
641
|
+
emitSyncProfileDuration(profile, addSymbolsStartedAt, {
|
|
642
|
+
name: "rateless.rangeAddSymbols",
|
|
643
|
+
entries: hashNumbers.length,
|
|
644
|
+
symbols: hashNumbers.length,
|
|
645
|
+
details: { type },
|
|
646
|
+
});
|
|
647
|
+
}
|
|
648
|
+
transferred = true;
|
|
649
|
+
return encoder;
|
|
601
650
|
}
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
if (typeof BigUint64Array !== "undefined" &&
|
|
607
|
-
typeof encoder.add_symbols === "function") {
|
|
608
|
-
const symbols = hashNumbers instanceof BigUint64Array
|
|
609
|
-
? hashNumbers
|
|
610
|
-
: BigUint64Array.from(hashNumbers, coerceBigInt);
|
|
611
|
-
addSymbolsToRiblt(encoder, symbols);
|
|
612
|
-
}
|
|
613
|
-
else {
|
|
614
|
-
for (const hashNumber of hashNumbers) {
|
|
615
|
-
encoder.add_symbol(coerceBigInt(hashNumber));
|
|
616
|
-
}
|
|
617
|
-
}
|
|
618
|
-
if (profile) {
|
|
619
|
-
emitSyncProfileDuration(profile, addSymbolsStartedAt, {
|
|
620
|
-
name: "rateless.rangeAddSymbols",
|
|
621
|
-
entries: hashNumbers.length,
|
|
622
|
-
symbols: hashNumbers.length,
|
|
623
|
-
details: { type },
|
|
624
|
-
});
|
|
651
|
+
finally {
|
|
652
|
+
if (!transferred) {
|
|
653
|
+
encoder.free();
|
|
654
|
+
}
|
|
625
655
|
}
|
|
626
|
-
return encoder;
|
|
627
656
|
};
|
|
628
657
|
export class RatelessIBLTSynchronizer {
|
|
629
658
|
properties;
|
|
630
659
|
simple;
|
|
631
660
|
repairSessionCounter;
|
|
661
|
+
ratelessRepairSessions;
|
|
632
662
|
startedOrCompletedSynchronizations;
|
|
633
663
|
localRangeEncoderCacheVersion = 0;
|
|
634
664
|
localRangeEncoderCache = new Map();
|
|
635
665
|
localRangeEncoderCacheMax = 2;
|
|
636
666
|
ingoingSyncProcesses;
|
|
667
|
+
incomingRatelessProcessAdmissions;
|
|
637
668
|
outgoingSyncProcesses;
|
|
669
|
+
outgoingSyncProcessByTarget;
|
|
670
|
+
ratelessDispatchLifecycleController;
|
|
671
|
+
ratelessDispatchTargets;
|
|
672
|
+
activeRatelessResponseCount;
|
|
673
|
+
activeRatelessResponseCountByPeer;
|
|
674
|
+
ratelessClosed;
|
|
638
675
|
constructor(properties) {
|
|
639
676
|
this.properties = properties;
|
|
640
677
|
this.simple = new SimpleSyncronizer(properties);
|
|
641
678
|
this.repairSessionCounter = 0;
|
|
679
|
+
this.ratelessRepairSessions = new Map();
|
|
642
680
|
this.outgoingSyncProcesses = new Map();
|
|
681
|
+
this.outgoingSyncProcessByTarget = new Map();
|
|
682
|
+
this.ratelessDispatchLifecycleController = new AbortController();
|
|
683
|
+
this.ratelessDispatchTargets = new Map();
|
|
684
|
+
this.activeRatelessResponseCount = 0;
|
|
685
|
+
this.activeRatelessResponseCountByPeer = new Map();
|
|
686
|
+
this.ratelessClosed = false;
|
|
643
687
|
this.ingoingSyncProcesses = new Map();
|
|
688
|
+
this.incomingRatelessProcessAdmissions = new Set();
|
|
644
689
|
this.startedOrCompletedSynchronizations = new Cache({ max: 1e4 });
|
|
645
690
|
}
|
|
646
691
|
get maxConvergentTrackedHashes() {
|
|
647
692
|
const value = this.properties.sync?.maxConvergentTrackedHashes;
|
|
648
693
|
return value && Number.isFinite(value) && value > 0
|
|
649
|
-
? Math.floor(value)
|
|
694
|
+
? Math.max(1, Math.floor(value))
|
|
650
695
|
: DEFAULT_MAX_CONVERGENT_TRACKED_HASHES;
|
|
651
696
|
}
|
|
652
697
|
normalizeRetryIntervals(retryIntervalsMs) {
|
|
@@ -677,6 +722,94 @@ export class RatelessIBLTSynchronizer {
|
|
|
677
722
|
scored.sort((a, b) => b.priority - a.priority || a.index - b.index);
|
|
678
723
|
return scored.map((x) => x.entry);
|
|
679
724
|
}
|
|
725
|
+
isRatelessRepairSessionActive(session) {
|
|
726
|
+
return (!this.ratelessClosed &&
|
|
727
|
+
!session.cancelled &&
|
|
728
|
+
!session.controller.signal.aborted &&
|
|
729
|
+
!session.ownershipLifecycleController.signal.aborted &&
|
|
730
|
+
session.ownershipLifecycleController ===
|
|
731
|
+
this.ratelessDispatchLifecycleController &&
|
|
732
|
+
this.ratelessRepairSessions.get(session.id) === session);
|
|
733
|
+
}
|
|
734
|
+
cancelRatelessRepairSession(session, reason = new Error("rateless repair session cancelled")) {
|
|
735
|
+
if (session.cancelled) {
|
|
736
|
+
return;
|
|
737
|
+
}
|
|
738
|
+
session.cancelled = true;
|
|
739
|
+
if (session.deadlineTimer !== undefined) {
|
|
740
|
+
clearTimeout(session.deadlineTimer);
|
|
741
|
+
session.deadlineTimer = undefined;
|
|
742
|
+
}
|
|
743
|
+
session.controller.abort(reason);
|
|
744
|
+
session.trackedSession.cancel();
|
|
745
|
+
}
|
|
746
|
+
disposeRatelessRepairSession(session) {
|
|
747
|
+
if (session.settled) {
|
|
748
|
+
return;
|
|
749
|
+
}
|
|
750
|
+
session.settled = true;
|
|
751
|
+
if (session.deadlineTimer !== undefined) {
|
|
752
|
+
clearTimeout(session.deadlineTimer);
|
|
753
|
+
session.deadlineTimer = undefined;
|
|
754
|
+
}
|
|
755
|
+
session.ownershipLifecycleController.signal.removeEventListener("abort", session.onOwnershipAbort);
|
|
756
|
+
if (this.ratelessRepairSessions.get(session.id) === session) {
|
|
757
|
+
this.ratelessRepairSessions.delete(session.id);
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
cancelRatelessRepairSessions(reason) {
|
|
761
|
+
for (const session of [...this.ratelessRepairSessions.values()]) {
|
|
762
|
+
this.cancelRatelessRepairSession(session, reason);
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
waitForRatelessRepairRetry(delayMs, signal) {
|
|
766
|
+
if (delayMs <= 0) {
|
|
767
|
+
return Promise.resolve(!signal.aborted);
|
|
768
|
+
}
|
|
769
|
+
if (signal.aborted) {
|
|
770
|
+
return Promise.resolve(false);
|
|
771
|
+
}
|
|
772
|
+
return new Promise((resolve) => {
|
|
773
|
+
let settled = false;
|
|
774
|
+
let timer;
|
|
775
|
+
const settle = (elapsed) => {
|
|
776
|
+
if (settled) {
|
|
777
|
+
return;
|
|
778
|
+
}
|
|
779
|
+
settled = true;
|
|
780
|
+
clearTimeout(timer);
|
|
781
|
+
signal.removeEventListener("abort", onAbort);
|
|
782
|
+
resolve(elapsed);
|
|
783
|
+
};
|
|
784
|
+
const onAbort = () => settle(false);
|
|
785
|
+
timer = setTimeout(() => settle(true), delayMs);
|
|
786
|
+
timer.unref?.();
|
|
787
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
788
|
+
if (signal.aborted) {
|
|
789
|
+
onAbort();
|
|
790
|
+
}
|
|
791
|
+
});
|
|
792
|
+
}
|
|
793
|
+
waitForRatelessRepairDispatch(dispatch, signal) {
|
|
794
|
+
const dispatchPromise = Promise.resolve(dispatch);
|
|
795
|
+
return new Promise((resolve) => {
|
|
796
|
+
let settled = false;
|
|
797
|
+
const settle = (completed) => {
|
|
798
|
+
if (settled) {
|
|
799
|
+
return;
|
|
800
|
+
}
|
|
801
|
+
settled = true;
|
|
802
|
+
signal.removeEventListener("abort", onAbort);
|
|
803
|
+
resolve(completed);
|
|
804
|
+
};
|
|
805
|
+
const onAbort = () => settle(false);
|
|
806
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
807
|
+
void dispatchPromise.then(() => settle(true), () => settle(true));
|
|
808
|
+
if (signal.aborted) {
|
|
809
|
+
onAbort();
|
|
810
|
+
}
|
|
811
|
+
});
|
|
812
|
+
}
|
|
680
813
|
startRepairSession(properties) {
|
|
681
814
|
const mode = properties.mode ?? "best-effort";
|
|
682
815
|
const targets = [...new Set(properties.targets)];
|
|
@@ -702,7 +835,6 @@ export class RatelessIBLTSynchronizer {
|
|
|
702
835
|
for (const entry of prioritized.slice(0, trackedLimit)) {
|
|
703
836
|
trackedEntries.set(entry.hash, entry);
|
|
704
837
|
}
|
|
705
|
-
let cancelled = false;
|
|
706
838
|
const trackedSession = this.simple.startRepairSession({
|
|
707
839
|
entries: trackedEntries,
|
|
708
840
|
targets,
|
|
@@ -710,32 +842,58 @@ export class RatelessIBLTSynchronizer {
|
|
|
710
842
|
timeoutMs,
|
|
711
843
|
retryIntervalsMs,
|
|
712
844
|
});
|
|
845
|
+
const ownershipLifecycleController = this.ratelessDispatchLifecycleController;
|
|
846
|
+
let session;
|
|
847
|
+
const cancel = (reason = new Error("rateless repair session cancelled")) => this.cancelRatelessRepairSession(session, reason);
|
|
848
|
+
session = {
|
|
849
|
+
id,
|
|
850
|
+
ownershipLifecycleController,
|
|
851
|
+
controller: new AbortController(),
|
|
852
|
+
trackedSession,
|
|
853
|
+
onOwnershipAbort: () => cancel(ownershipLifecycleController.signal.reason ??
|
|
854
|
+
new Error("rateless repair generation ended")),
|
|
855
|
+
cancelled: false,
|
|
856
|
+
settled: false,
|
|
857
|
+
deadlineTimer: undefined,
|
|
858
|
+
};
|
|
859
|
+
this.ratelessRepairSessions.set(id, session);
|
|
860
|
+
session.deadlineTimer = setTimeout(() => cancel(new Error("rateless convergent repair session timed out")), Math.max(0, timeoutMs - (Date.now() - startedAt)));
|
|
861
|
+
session.deadlineTimer.unref?.();
|
|
862
|
+
ownershipLifecycleController.signal.addEventListener("abort", session.onOwnershipAbort, { once: true });
|
|
863
|
+
if (this.ratelessClosed ||
|
|
864
|
+
ownershipLifecycleController.signal.aborted ||
|
|
865
|
+
ownershipLifecycleController !==
|
|
866
|
+
this.ratelessDispatchLifecycleController) {
|
|
867
|
+
cancel(ownershipLifecycleController.signal.reason ??
|
|
868
|
+
new Error("rateless repair generation is not active"));
|
|
869
|
+
}
|
|
713
870
|
const runDispatchSchedule = async () => {
|
|
714
871
|
let previousDelay = 0;
|
|
715
872
|
for (const delayMs of retryIntervalsMs) {
|
|
716
|
-
if (
|
|
873
|
+
if (!this.isRatelessRepairSessionActive(session)) {
|
|
717
874
|
return;
|
|
718
875
|
}
|
|
719
876
|
const elapsed = Date.now() - startedAt;
|
|
720
877
|
if (elapsed >= timeoutMs) {
|
|
721
878
|
return;
|
|
722
879
|
}
|
|
723
|
-
const waitMs = Math.max(0, delayMs - previousDelay);
|
|
880
|
+
const waitMs = Math.min(Math.max(0, delayMs - previousDelay), timeoutMs - elapsed);
|
|
724
881
|
previousDelay = delayMs;
|
|
725
|
-
if (waitMs
|
|
726
|
-
|
|
727
|
-
const timer = setTimeout(resolve, waitMs);
|
|
728
|
-
timer.unref?.();
|
|
729
|
-
});
|
|
882
|
+
if (!(await this.waitForRatelessRepairRetry(waitMs, session.controller.signal))) {
|
|
883
|
+
return;
|
|
730
884
|
}
|
|
731
|
-
if (
|
|
885
|
+
if (!this.isRatelessRepairSessionActive(session) ||
|
|
886
|
+
Date.now() - startedAt >= timeoutMs) {
|
|
732
887
|
return;
|
|
733
888
|
}
|
|
734
889
|
try {
|
|
735
|
-
await this.onMaybeMissingEntries({
|
|
890
|
+
if (!(await this.waitForRatelessRepairDispatch(this.onMaybeMissingEntries({
|
|
736
891
|
entries: properties.entries,
|
|
737
892
|
targets,
|
|
738
|
-
|
|
893
|
+
signal: session.controller.signal,
|
|
894
|
+
}), session.controller.signal))) {
|
|
895
|
+
return;
|
|
896
|
+
}
|
|
739
897
|
}
|
|
740
898
|
catch {
|
|
741
899
|
// Best-effort schedule: tracked session timeout/result decides completion.
|
|
@@ -743,21 +901,23 @@ export class RatelessIBLTSynchronizer {
|
|
|
743
901
|
}
|
|
744
902
|
};
|
|
745
903
|
const done = (async () => {
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
904
|
+
try {
|
|
905
|
+
await runDispatchSchedule();
|
|
906
|
+
const trackedResults = await trackedSession.done;
|
|
907
|
+
return trackedResults.map((result) => ({
|
|
908
|
+
...result,
|
|
909
|
+
requestedTotal: requestedHashes.length,
|
|
910
|
+
truncated: true,
|
|
911
|
+
}));
|
|
912
|
+
}
|
|
913
|
+
finally {
|
|
914
|
+
this.disposeRatelessRepairSession(session);
|
|
915
|
+
}
|
|
753
916
|
})();
|
|
754
917
|
return {
|
|
755
918
|
id,
|
|
756
919
|
done,
|
|
757
|
-
cancel: () =>
|
|
758
|
-
cancelled = true;
|
|
759
|
-
trackedSession.cancel();
|
|
760
|
-
},
|
|
920
|
+
cancel: () => cancel(),
|
|
761
921
|
};
|
|
762
922
|
}
|
|
763
923
|
const id = `rateless-repair-${++this.repairSessionCounter}`;
|
|
@@ -801,32 +961,60 @@ export class RatelessIBLTSynchronizer {
|
|
|
801
961
|
localRangeEncoderCacheKey(ranges) {
|
|
802
962
|
return `${String(ranges.start1)}:${String(ranges.end1)}:${String(ranges.start2)}:${String(ranges.end2)}`;
|
|
803
963
|
}
|
|
804
|
-
decoderFromCachedEncoder(encoder) {
|
|
964
|
+
decoderFromCachedEncoder(encoder, beforeDecoderTransfer) {
|
|
805
965
|
const clone = encoder.clone();
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
966
|
+
let cloneFreed = false;
|
|
967
|
+
let decoder;
|
|
968
|
+
let decoderTransferred = false;
|
|
969
|
+
const freeClone = () => {
|
|
970
|
+
if (cloneFreed) {
|
|
971
|
+
return;
|
|
972
|
+
}
|
|
973
|
+
cloneFreed = true;
|
|
974
|
+
clone.free();
|
|
975
|
+
};
|
|
976
|
+
try {
|
|
977
|
+
decoder = clone.to_decoder();
|
|
978
|
+
beforeDecoderTransfer?.();
|
|
979
|
+
freeClone();
|
|
980
|
+
decoderTransferred = true;
|
|
981
|
+
return decoder;
|
|
982
|
+
}
|
|
983
|
+
finally {
|
|
984
|
+
try {
|
|
985
|
+
freeClone();
|
|
986
|
+
}
|
|
987
|
+
finally {
|
|
988
|
+
if (!decoderTransferred) {
|
|
989
|
+
decoder?.free();
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
}
|
|
809
993
|
}
|
|
810
|
-
async getLocalDecoderForRange(ranges) {
|
|
994
|
+
async getLocalDecoderForRange(ranges, options) {
|
|
995
|
+
const isActive = () => options?.signal?.aborted !== true &&
|
|
996
|
+
(options?.ownershipLifecycleController === undefined ||
|
|
997
|
+
this.isIncomingSyncGenerationActive(options.ownershipLifecycleController));
|
|
998
|
+
if (!isActive()) {
|
|
999
|
+
return false;
|
|
1000
|
+
}
|
|
811
1001
|
const profile = this.properties.sync?.profile;
|
|
812
1002
|
const key = this.localRangeEncoderCacheKey(ranges);
|
|
813
1003
|
const cached = this.localRangeEncoderCache.get(key);
|
|
814
1004
|
if (cached && cached.version === this.localRangeEncoderCacheVersion) {
|
|
815
1005
|
const startedAt = syncProfileStart(profile);
|
|
816
1006
|
cached.lastUsed = Date.now();
|
|
817
|
-
|
|
818
|
-
return this.decoderFromCachedEncoder(cached.encoder);
|
|
819
|
-
}
|
|
820
|
-
finally {
|
|
1007
|
+
return this.decoderFromCachedEncoder(cached.encoder, () => {
|
|
821
1008
|
if (profile) {
|
|
822
1009
|
emitSyncProfileDuration(profile, startedAt, {
|
|
823
1010
|
name: "rateless.localDecoder",
|
|
824
1011
|
cacheHit: true,
|
|
825
1012
|
});
|
|
826
1013
|
}
|
|
827
|
-
}
|
|
1014
|
+
});
|
|
828
1015
|
}
|
|
829
1016
|
const startedAt = syncProfileStart(profile);
|
|
1017
|
+
const cacheVersion = this.localRangeEncoderCacheVersion;
|
|
830
1018
|
const encoder = (await buildEncoderOrDecoderFromRange(ranges, this.properties.entryIndex, "encoder", profile, this.properties.resolveHashNumbersInRange));
|
|
831
1019
|
if (!encoder) {
|
|
832
1020
|
if (profile) {
|
|
@@ -838,6 +1026,26 @@ export class RatelessIBLTSynchronizer {
|
|
|
838
1026
|
}
|
|
839
1027
|
return false;
|
|
840
1028
|
}
|
|
1029
|
+
if (!isActive()) {
|
|
1030
|
+
encoder.free();
|
|
1031
|
+
return false;
|
|
1032
|
+
}
|
|
1033
|
+
if (cacheVersion !== this.localRangeEncoderCacheVersion) {
|
|
1034
|
+
try {
|
|
1035
|
+
return this.decoderFromCachedEncoder(encoder, () => {
|
|
1036
|
+
if (profile) {
|
|
1037
|
+
emitSyncProfileDuration(profile, startedAt, {
|
|
1038
|
+
name: "rateless.localDecoder",
|
|
1039
|
+
cacheHit: false,
|
|
1040
|
+
details: { cacheInvalidated: true },
|
|
1041
|
+
});
|
|
1042
|
+
}
|
|
1043
|
+
});
|
|
1044
|
+
}
|
|
1045
|
+
finally {
|
|
1046
|
+
encoder.free();
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
841
1049
|
const now = Date.now();
|
|
842
1050
|
const existing = this.localRangeEncoderCache.get(key);
|
|
843
1051
|
if (existing) {
|
|
@@ -866,21 +1074,168 @@ export class RatelessIBLTSynchronizer {
|
|
|
866
1074
|
}
|
|
867
1075
|
this.localRangeEncoderCache.delete(oldestKey);
|
|
868
1076
|
}
|
|
869
|
-
|
|
870
|
-
return this.decoderFromCachedEncoder(encoder);
|
|
871
|
-
}
|
|
872
|
-
finally {
|
|
1077
|
+
return this.decoderFromCachedEncoder(encoder, () => {
|
|
873
1078
|
if (profile) {
|
|
874
1079
|
emitSyncProfileDuration(profile, startedAt, {
|
|
875
1080
|
name: "rateless.localDecoder",
|
|
876
1081
|
cacheHit: false,
|
|
877
1082
|
});
|
|
878
1083
|
}
|
|
1084
|
+
});
|
|
1085
|
+
}
|
|
1086
|
+
captureRatelessDispatchLifecycle(targets, callerSignal) {
|
|
1087
|
+
const ownershipLifecycleController = this.ratelessDispatchLifecycleController;
|
|
1088
|
+
const lifecycle = {
|
|
1089
|
+
ownershipLifecycleController,
|
|
1090
|
+
callerSignal,
|
|
1091
|
+
controller: new AbortController(),
|
|
1092
|
+
targets: new Map(),
|
|
1093
|
+
onOwnerOrCallerAbort: () => {
|
|
1094
|
+
const reason = callerSignal?.aborted === true
|
|
1095
|
+
? callerSignal.reason
|
|
1096
|
+
: ownershipLifecycleController.signal.reason;
|
|
1097
|
+
this.abortRatelessDispatchLifecycle(lifecycle, reason);
|
|
1098
|
+
},
|
|
1099
|
+
dispatchFinished: false,
|
|
1100
|
+
disposed: false,
|
|
1101
|
+
};
|
|
1102
|
+
for (const target of [...new Set(targets)]) {
|
|
1103
|
+
const targetLifecycle = {
|
|
1104
|
+
lifecycle,
|
|
1105
|
+
target,
|
|
1106
|
+
controller: new AbortController(),
|
|
1107
|
+
retainedByProcess: false,
|
|
1108
|
+
responseLeases: 0,
|
|
1109
|
+
};
|
|
1110
|
+
lifecycle.targets.set(target, targetLifecycle);
|
|
1111
|
+
let activeForTarget = this.ratelessDispatchTargets.get(target);
|
|
1112
|
+
if (!activeForTarget) {
|
|
1113
|
+
activeForTarget = new Set();
|
|
1114
|
+
this.ratelessDispatchTargets.set(target, activeForTarget);
|
|
1115
|
+
}
|
|
1116
|
+
activeForTarget.add(targetLifecycle);
|
|
1117
|
+
}
|
|
1118
|
+
ownershipLifecycleController.signal.addEventListener("abort", lifecycle.onOwnerOrCallerAbort, { once: true });
|
|
1119
|
+
if (callerSignal && callerSignal !== ownershipLifecycleController.signal) {
|
|
1120
|
+
callerSignal.addEventListener("abort", lifecycle.onOwnerOrCallerAbort, {
|
|
1121
|
+
once: true,
|
|
1122
|
+
});
|
|
1123
|
+
}
|
|
1124
|
+
if (this.ratelessClosed ||
|
|
1125
|
+
ownershipLifecycleController !==
|
|
1126
|
+
this.ratelessDispatchLifecycleController ||
|
|
1127
|
+
ownershipLifecycleController.signal.aborted ||
|
|
1128
|
+
callerSignal?.aborted) {
|
|
1129
|
+
lifecycle.onOwnerOrCallerAbort();
|
|
1130
|
+
}
|
|
1131
|
+
return lifecycle;
|
|
1132
|
+
}
|
|
1133
|
+
abortRatelessDispatchTarget(targetLifecycle, reason) {
|
|
1134
|
+
if (!targetLifecycle.controller.signal.aborted) {
|
|
1135
|
+
targetLifecycle.controller.abort(reason);
|
|
1136
|
+
}
|
|
1137
|
+
this.maybeDisposeRatelessDispatchLifecycle(targetLifecycle.lifecycle);
|
|
1138
|
+
}
|
|
1139
|
+
abortRatelessDispatchLifecycle(lifecycle, reason) {
|
|
1140
|
+
if (!lifecycle.controller.signal.aborted) {
|
|
1141
|
+
lifecycle.controller.abort(reason);
|
|
1142
|
+
}
|
|
1143
|
+
for (const targetLifecycle of lifecycle.targets.values()) {
|
|
1144
|
+
this.abortRatelessDispatchTarget(targetLifecycle, reason);
|
|
1145
|
+
}
|
|
1146
|
+
this.maybeDisposeRatelessDispatchLifecycle(lifecycle);
|
|
1147
|
+
}
|
|
1148
|
+
finishRatelessDispatchLifecycle(lifecycle) {
|
|
1149
|
+
lifecycle.dispatchFinished = true;
|
|
1150
|
+
this.maybeDisposeRatelessDispatchLifecycle(lifecycle);
|
|
1151
|
+
}
|
|
1152
|
+
maybeDisposeRatelessDispatchLifecycle(lifecycle) {
|
|
1153
|
+
if (lifecycle.disposed ||
|
|
1154
|
+
!lifecycle.dispatchFinished ||
|
|
1155
|
+
[...lifecycle.targets.values()].some((target) => target.retainedByProcess || target.responseLeases > 0)) {
|
|
1156
|
+
return;
|
|
1157
|
+
}
|
|
1158
|
+
lifecycle.disposed = true;
|
|
1159
|
+
lifecycle.ownershipLifecycleController.signal.removeEventListener("abort", lifecycle.onOwnerOrCallerAbort);
|
|
1160
|
+
if (lifecycle.callerSignal &&
|
|
1161
|
+
lifecycle.callerSignal !== lifecycle.ownershipLifecycleController.signal) {
|
|
1162
|
+
lifecycle.callerSignal.removeEventListener("abort", lifecycle.onOwnerOrCallerAbort);
|
|
1163
|
+
}
|
|
1164
|
+
for (const targetLifecycle of lifecycle.targets.values()) {
|
|
1165
|
+
const activeForTarget = this.ratelessDispatchTargets.get(targetLifecycle.target);
|
|
1166
|
+
activeForTarget?.delete(targetLifecycle);
|
|
1167
|
+
if (activeForTarget?.size === 0) {
|
|
1168
|
+
this.ratelessDispatchTargets.delete(targetLifecycle.target);
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
isRatelessDispatchLifecycleActive(lifecycle, target) {
|
|
1173
|
+
if (this.ratelessClosed ||
|
|
1174
|
+
lifecycle.disposed ||
|
|
1175
|
+
lifecycle.ownershipLifecycleController !==
|
|
1176
|
+
this.ratelessDispatchLifecycleController ||
|
|
1177
|
+
lifecycle.ownershipLifecycleController.signal.aborted ||
|
|
1178
|
+
lifecycle.callerSignal?.aborted ||
|
|
1179
|
+
lifecycle.controller.signal.aborted) {
|
|
1180
|
+
return false;
|
|
879
1181
|
}
|
|
1182
|
+
if (target === undefined) {
|
|
1183
|
+
return true;
|
|
1184
|
+
}
|
|
1185
|
+
const targetLifecycle = lifecycle.targets.get(target);
|
|
1186
|
+
return (targetLifecycle !== undefined &&
|
|
1187
|
+
!targetLifecycle.controller.signal.aborted &&
|
|
1188
|
+
this.ratelessDispatchTargets.get(target)?.has(targetLifecycle) === true);
|
|
1189
|
+
}
|
|
1190
|
+
getIncomingSyncProcessKey(sender, syncId) {
|
|
1191
|
+
return `${sender.length}:${sender}${syncId}`;
|
|
1192
|
+
}
|
|
1193
|
+
isIncomingSyncGenerationActive(ownershipLifecycleController) {
|
|
1194
|
+
return (!this.ratelessClosed &&
|
|
1195
|
+
ownershipLifecycleController ===
|
|
1196
|
+
this.ratelessDispatchLifecycleController &&
|
|
1197
|
+
!ownershipLifecycleController.signal.aborted);
|
|
1198
|
+
}
|
|
1199
|
+
isIncomingSyncProcessActive(process) {
|
|
1200
|
+
return (this.isIncomingSyncGenerationActive(process.ownershipLifecycleController) &&
|
|
1201
|
+
!process.controller.signal.aborted &&
|
|
1202
|
+
this.ingoingSyncProcesses.get(process.key) === process);
|
|
880
1203
|
}
|
|
881
1204
|
async onMaybeMissingEntries(properties) {
|
|
1205
|
+
// Capture this rateless open generation synchronously, before any await.
|
|
1206
|
+
// In particular, signal-less internal repair work must not resume after
|
|
1207
|
+
// close/open and borrow the newly opened Simple generation.
|
|
1208
|
+
const lifecycle = this.captureRatelessDispatchLifecycle(properties.targets, properties.signal);
|
|
1209
|
+
try {
|
|
1210
|
+
await this.onMaybeMissingEntriesWithLifecycle(properties, lifecycle);
|
|
1211
|
+
}
|
|
1212
|
+
finally {
|
|
1213
|
+
this.finishRatelessDispatchLifecycle(lifecycle);
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
async onMaybeMissingEntriesWithLifecycle(properties, lifecycle) {
|
|
1217
|
+
const signal = lifecycle.controller.signal;
|
|
882
1218
|
const profile = this.properties.sync?.profile;
|
|
883
1219
|
const startedAt = syncProfileStart(profile);
|
|
1220
|
+
let topLevelProfileEmitted = false;
|
|
1221
|
+
const emitTopLevelProfile = (details, measurements = {}) => {
|
|
1222
|
+
if (!profile || topLevelProfileEmitted) {
|
|
1223
|
+
return;
|
|
1224
|
+
}
|
|
1225
|
+
topLevelProfileEmitted = true;
|
|
1226
|
+
emitSyncProfileDuration(profile, startedAt, {
|
|
1227
|
+
name: "rateless.onMaybeMissingEntries",
|
|
1228
|
+
entries: properties.entries.size,
|
|
1229
|
+
targets: properties.targets.length,
|
|
1230
|
+
...measurements,
|
|
1231
|
+
details,
|
|
1232
|
+
});
|
|
1233
|
+
};
|
|
1234
|
+
const emitCancelledTopLevelProfile = (phase, mode = "rateless") => emitTopLevelProfile({
|
|
1235
|
+
mode,
|
|
1236
|
+
phase,
|
|
1237
|
+
cancelled: true,
|
|
1238
|
+
}, { messages: 0, symbols: 0 });
|
|
884
1239
|
// NOTE: this method is best-effort dispatch, not a per-hash convergence API.
|
|
885
1240
|
// It may require follow-up repair rounds under churn/loss to fully close all gaps.
|
|
886
1241
|
// Strategy:
|
|
@@ -890,6 +1245,12 @@ export class RatelessIBLTSynchronizer {
|
|
|
890
1245
|
// such as those assigned to range boundaries.
|
|
891
1246
|
let minSyncIbltSize = 333; // TODO: make configurable
|
|
892
1247
|
let maxSyncWithSimpleMethod = 1e3;
|
|
1248
|
+
if (signal?.aborted) {
|
|
1249
|
+
emitCancelledTopLevelProfile("before-dispatch", properties.entries.size <= minSyncIbltSize
|
|
1250
|
+
? "simple-small"
|
|
1251
|
+
: "rateless");
|
|
1252
|
+
return;
|
|
1253
|
+
}
|
|
893
1254
|
const priorityFn = this.properties.sync?.priority;
|
|
894
1255
|
const maxSimpleEntries = this.properties.sync?.maxSimpleEntries;
|
|
895
1256
|
// Small batch => use simple synchronizer entirely
|
|
@@ -906,17 +1267,17 @@ export class RatelessIBLTSynchronizer {
|
|
|
906
1267
|
await this.simple.onMaybeMissingEntries({
|
|
907
1268
|
entries: properties.entries,
|
|
908
1269
|
targets: properties.targets,
|
|
1270
|
+
signal,
|
|
909
1271
|
});
|
|
1272
|
+
if (signal?.aborted) {
|
|
1273
|
+
return;
|
|
1274
|
+
}
|
|
910
1275
|
}
|
|
911
1276
|
finally {
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
targets: properties.targets.length,
|
|
917
|
-
details: { mode: "simple-small" },
|
|
918
|
-
});
|
|
919
|
-
}
|
|
1277
|
+
emitTopLevelProfile({
|
|
1278
|
+
mode: "simple-small",
|
|
1279
|
+
cancelled: signal?.aborted || undefined,
|
|
1280
|
+
});
|
|
920
1281
|
}
|
|
921
1282
|
return;
|
|
922
1283
|
}
|
|
@@ -980,29 +1341,43 @@ export class RatelessIBLTSynchronizer {
|
|
|
980
1341
|
}
|
|
981
1342
|
}
|
|
982
1343
|
}
|
|
983
|
-
|
|
1344
|
+
const useAllCoordinatesForIblt = allCoordinatesToSyncWithIblt.length === 0 ||
|
|
1345
|
+
naiveHashes.length > maxSyncWithSimpleMethod;
|
|
1346
|
+
if (useAllCoordinatesForIblt) {
|
|
1347
|
+
// If every entry is a range-boundary special case, or the special-case
|
|
1348
|
+
// set itself is too large for the Simple prelude, use one bounded
|
|
1349
|
+
// Rateless process for the full set. Sending the oversized Simple
|
|
1350
|
+
// prelude first can complete or abort the repair lifecycle before
|
|
1351
|
+
// StartSync is ever dispatched.
|
|
1352
|
+
allCoordinatesToSyncWithIblt = [];
|
|
1353
|
+
for (const entry of properties.entries.values()) {
|
|
1354
|
+
allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
else if (naiveHashes.length > 0) {
|
|
984
1358
|
// If there are special-case entries, sync them simply in parallel
|
|
985
1359
|
if (priorityFn && naiveEntriesForPriority) {
|
|
986
1360
|
await this.simple.onMaybeMissingEntries({
|
|
987
1361
|
entries: naiveEntriesForPriority,
|
|
988
1362
|
targets: properties.targets,
|
|
1363
|
+
signal,
|
|
989
1364
|
});
|
|
990
1365
|
}
|
|
991
1366
|
else {
|
|
992
1367
|
await this.simple.onMaybeMissingHashes({
|
|
993
1368
|
hashes: naiveHashes,
|
|
994
1369
|
targets: properties.targets,
|
|
1370
|
+
signal,
|
|
995
1371
|
});
|
|
996
1372
|
}
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
// Fallback: if nothing left for IBLT (or simple set is too large), include all in IBLT
|
|
1001
|
-
allCoordinatesToSyncWithIblt = [];
|
|
1002
|
-
for (const entry of properties.entries.values()) {
|
|
1003
|
-
allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
|
|
1373
|
+
if (signal?.aborted) {
|
|
1374
|
+
emitCancelledTopLevelProfile("simple-prelude");
|
|
1375
|
+
return;
|
|
1004
1376
|
}
|
|
1005
1377
|
}
|
|
1378
|
+
const simplePreludeEntries = useAllCoordinatesForIblt
|
|
1379
|
+
? 0
|
|
1380
|
+
: naiveHashes.length;
|
|
1006
1381
|
if (profile) {
|
|
1007
1382
|
emitSyncProfileDuration(profile, selectStartedAt, {
|
|
1008
1383
|
name: "rateless.selectEntries",
|
|
@@ -1010,11 +1385,15 @@ export class RatelessIBLTSynchronizer {
|
|
|
1010
1385
|
symbols: allCoordinatesToSyncWithIblt.length,
|
|
1011
1386
|
targets: properties.targets.length,
|
|
1012
1387
|
details: {
|
|
1013
|
-
naiveEntries:
|
|
1388
|
+
naiveEntries: simplePreludeEntries,
|
|
1014
1389
|
priority: priorityFn != null,
|
|
1015
1390
|
},
|
|
1016
1391
|
});
|
|
1017
1392
|
}
|
|
1393
|
+
if (signal?.aborted) {
|
|
1394
|
+
emitCancelledTopLevelProfile("entry-selection");
|
|
1395
|
+
return;
|
|
1396
|
+
}
|
|
1018
1397
|
if (allCoordinatesToSyncWithIblt.length === 0) {
|
|
1019
1398
|
if (profile) {
|
|
1020
1399
|
emitSyncProfileEvent(profile, {
|
|
@@ -1023,327 +1402,914 @@ export class RatelessIBLTSynchronizer {
|
|
|
1023
1402
|
targets: properties.targets.length,
|
|
1024
1403
|
details: { mode: "simple-only" },
|
|
1025
1404
|
});
|
|
1026
|
-
emitSyncProfileDuration(profile, startedAt, {
|
|
1027
|
-
name: "rateless.onMaybeMissingEntries",
|
|
1028
|
-
entries: properties.entries.size,
|
|
1029
|
-
targets: properties.targets.length,
|
|
1030
|
-
details: { mode: "simple-only" },
|
|
1031
|
-
});
|
|
1032
1405
|
}
|
|
1406
|
+
emitTopLevelProfile({ mode: "simple-only" });
|
|
1033
1407
|
return;
|
|
1034
1408
|
}
|
|
1035
1409
|
const outgoingHashes = priorityFn == null
|
|
1036
1410
|
? [...properties.entries.keys()]
|
|
1037
1411
|
: this.getPrioritizedEntries(properties.entries).map((entry) => entry.hash);
|
|
1038
1412
|
await ribltReady;
|
|
1413
|
+
if (signal?.aborted) {
|
|
1414
|
+
emitCancelledTopLevelProfile("riblt-ready");
|
|
1415
|
+
return;
|
|
1416
|
+
}
|
|
1039
1417
|
// For smaller sets, the original `sqrt(n)` heuristic can occasionally under-provision
|
|
1040
1418
|
// low-degree symbols early, causing an unnecessary `MoreSymbols` round-trip. Use a
|
|
1041
1419
|
// small floor to make small-delta syncs more reliable without affecting large-n behavior.
|
|
1042
1420
|
let initialSymbolCount = Math.round(Math.sqrt(allCoordinatesToSyncWithIblt.length)); // TODO choose better
|
|
1043
|
-
initialSymbolCount = Math.max(64, initialSymbolCount);
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
emitSyncProfileDuration(profile, prepareStartedAt, {
|
|
1048
|
-
name: "rateless.prepareStartSyncEncoder",
|
|
1049
|
-
entries: allCoordinatesToSyncWithIblt.length,
|
|
1050
|
-
symbols: initialSymbols?.length,
|
|
1051
|
-
details: {
|
|
1052
|
-
initialSymbolCount,
|
|
1053
|
-
includesInitialSymbols: initialSymbols != null,
|
|
1054
|
-
},
|
|
1055
|
-
});
|
|
1421
|
+
initialSymbolCount = Math.min(MAX_MORE_SYMBOLS_BATCH_SIZE, Math.max(64, initialSymbolCount));
|
|
1422
|
+
if (signal?.aborted) {
|
|
1423
|
+
emitCancelledTopLevelProfile("before-encoder-prepare");
|
|
1424
|
+
return;
|
|
1056
1425
|
}
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
if (
|
|
1062
|
-
|
|
1063
|
-
name: "rateless.produceStartSyncSymbols",
|
|
1064
|
-
symbols: startSyncSymbols.length,
|
|
1065
|
-
});
|
|
1426
|
+
const authorizedHashes = new Set(outgoingHashes);
|
|
1427
|
+
let messages = 0;
|
|
1428
|
+
let symbols = 0;
|
|
1429
|
+
for (const [target, targetLifecycle] of lifecycle.targets) {
|
|
1430
|
+
if (!this.isRatelessDispatchLifecycleActive(lifecycle)) {
|
|
1431
|
+
break;
|
|
1066
1432
|
}
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
const nextBatch = Math.max(MIN_MORE_SYMBOLS_BATCH_SIZE, Math.min(MAX_MORE_SYMBOLS_BATCH_SIZE, Math.ceil(allCoordinatesToSyncWithIblt.length / 4)));
|
|
1087
|
-
const obj = {
|
|
1088
|
-
encoder,
|
|
1089
|
-
timeout: createTimeout(),
|
|
1090
|
-
refresh: () => {
|
|
1091
|
-
let prevTimeout = obj.timeout;
|
|
1092
|
-
if (prevTimeout) {
|
|
1093
|
-
clearTimeout(prevTimeout);
|
|
1094
|
-
}
|
|
1095
|
-
obj.timeout = createTimeout();
|
|
1096
|
-
},
|
|
1097
|
-
next: (properties) => {
|
|
1098
|
-
if (properties.lastSeqNo <= lastSeqNo) {
|
|
1099
|
-
return CodedSymbolBatch.fromSymbols([]);
|
|
1100
|
-
}
|
|
1101
|
-
lastSeqNo++;
|
|
1102
|
-
obj.refresh(); // TODO use timestamp instead and collective pruning/refresh
|
|
1103
|
-
const produceStartedAt = syncProfileStart(profile);
|
|
1104
|
-
const symbols = produceNextCodedSymbols(encoder, nextBatch);
|
|
1105
|
-
if (profile) {
|
|
1106
|
-
emitSyncProfileDuration(profile, produceStartedAt, {
|
|
1107
|
-
name: "rateless.produceMoreSymbols",
|
|
1108
|
-
syncId,
|
|
1109
|
-
symbols: symbols.length,
|
|
1433
|
+
if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
|
|
1434
|
+
continue;
|
|
1435
|
+
}
|
|
1436
|
+
const activeProcess = this.outgoingSyncProcessByTarget.get(target);
|
|
1437
|
+
if (activeProcess &&
|
|
1438
|
+
!activeProcess.signal.aborted &&
|
|
1439
|
+
this.isRatelessDispatchLifecycleActive(activeProcess.targetLifecycle.lifecycle, target)) {
|
|
1440
|
+
// A repair sweep can flush adjacent batches for the same target
|
|
1441
|
+
// faster than its first StartSync reaches the transport. Replacing
|
|
1442
|
+
// that process here aborts the in-flight send; repeated large
|
|
1443
|
+
// batches can therefore starve Rateless sync entirely. Keep the
|
|
1444
|
+
// unambiguous active process and use bounded Simple sync only for
|
|
1445
|
+
// hashes that it does not already authorize.
|
|
1446
|
+
const uncoveredHashes = outgoingHashes.filter((hash) => !activeProcess.authorizedHashes.has(hash));
|
|
1447
|
+
if (uncoveredHashes.length > 0) {
|
|
1448
|
+
await this.simple.onMaybeMissingHashes({
|
|
1449
|
+
hashes: uncoveredHashes,
|
|
1450
|
+
targets: [target],
|
|
1451
|
+
signal,
|
|
1110
1452
|
});
|
|
1111
1453
|
}
|
|
1112
|
-
|
|
1113
|
-
}
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
entries: properties.entries.size,
|
|
1122
|
-
symbols: startSyncSymbols.length,
|
|
1123
|
-
targets: properties.targets.length,
|
|
1124
|
-
syncId,
|
|
1125
|
-
details: { mode: "rateless" },
|
|
1454
|
+
continue;
|
|
1455
|
+
}
|
|
1456
|
+
const startedSymbols = this.startOutgoingRatelessSyncForTarget({
|
|
1457
|
+
targetLifecycle,
|
|
1458
|
+
coordinates: allCoordinatesToSyncWithIblt,
|
|
1459
|
+
outgoingHashes,
|
|
1460
|
+
authorizedHashes,
|
|
1461
|
+
initialSymbolCount,
|
|
1462
|
+
entryCount: properties.entries.size,
|
|
1126
1463
|
});
|
|
1464
|
+
if (startedSymbols !== undefined) {
|
|
1465
|
+
messages += 1;
|
|
1466
|
+
symbols += startedSymbols;
|
|
1467
|
+
}
|
|
1127
1468
|
}
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
symbols: startSyncSymbols.length,
|
|
1138
|
-
targets: properties.targets.length,
|
|
1139
|
-
syncId,
|
|
1140
|
-
}), () => emitSyncProfileDuration(profile, sendStartedAt, {
|
|
1141
|
-
name: "rateless.sendStartSync",
|
|
1142
|
-
messages: 1,
|
|
1143
|
-
symbols: startSyncSymbols.length,
|
|
1144
|
-
targets: properties.targets.length,
|
|
1145
|
-
syncId,
|
|
1146
|
-
details: { rejected: true },
|
|
1147
|
-
}));
|
|
1148
|
-
}
|
|
1149
|
-
if (profile) {
|
|
1150
|
-
emitSyncProfileDuration(profile, startedAt, {
|
|
1151
|
-
name: "rateless.onMaybeMissingEntries",
|
|
1152
|
-
entries: properties.entries.size,
|
|
1153
|
-
messages: 1,
|
|
1154
|
-
symbols: startSyncSymbols.length,
|
|
1155
|
-
targets: properties.targets.length,
|
|
1156
|
-
details: {
|
|
1157
|
-
mode: "rateless",
|
|
1158
|
-
ibltEntries: allCoordinatesToSyncWithIblt.length,
|
|
1159
|
-
naiveEntries: naiveHashes.length,
|
|
1160
|
-
},
|
|
1161
|
-
});
|
|
1469
|
+
if (signal.aborted) {
|
|
1470
|
+
emitTopLevelProfile({
|
|
1471
|
+
mode: "rateless",
|
|
1472
|
+
phase: "start-sync-send",
|
|
1473
|
+
cancelled: true,
|
|
1474
|
+
ibltEntries: allCoordinatesToSyncWithIblt.length,
|
|
1475
|
+
naiveEntries: simplePreludeEntries,
|
|
1476
|
+
}, { messages, symbols });
|
|
1477
|
+
return;
|
|
1162
1478
|
}
|
|
1479
|
+
emitTopLevelProfile({
|
|
1480
|
+
mode: "rateless",
|
|
1481
|
+
ibltEntries: allCoordinatesToSyncWithIblt.length,
|
|
1482
|
+
naiveEntries: simplePreludeEntries,
|
|
1483
|
+
}, { messages, symbols });
|
|
1163
1484
|
}
|
|
1164
|
-
|
|
1485
|
+
startOutgoingRatelessSyncForTarget(properties) {
|
|
1486
|
+
const lifecycle = properties.targetLifecycle.lifecycle;
|
|
1487
|
+
const target = properties.targetLifecycle.target;
|
|
1165
1488
|
const profile = this.properties.sync?.profile;
|
|
1166
|
-
if (
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1489
|
+
if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
|
|
1490
|
+
return undefined;
|
|
1491
|
+
}
|
|
1492
|
+
const prepareStartedAt = syncProfileStart(profile);
|
|
1493
|
+
const { encoder, start, end, initialSymbols } = prepareStartSyncEncoder(properties.coordinates, this.properties.numbers.maxValue, properties.initialSymbolCount);
|
|
1494
|
+
let encoderFreed = false;
|
|
1495
|
+
let encoderTransferred = false;
|
|
1496
|
+
let obj;
|
|
1497
|
+
let processConstructed = false;
|
|
1498
|
+
const freeEncoder = () => {
|
|
1499
|
+
if (encoderFreed) {
|
|
1500
|
+
return;
|
|
1173
1501
|
}
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
start1: message.start,
|
|
1179
|
-
end1: wrapped ? this.properties.numbers.maxValue : message.end,
|
|
1180
|
-
start2: 0n,
|
|
1181
|
-
end2: wrapped ? message.end : 0n,
|
|
1182
|
-
});
|
|
1502
|
+
encoderFreed = true;
|
|
1503
|
+
encoder.free();
|
|
1504
|
+
};
|
|
1505
|
+
try {
|
|
1183
1506
|
if (profile) {
|
|
1184
|
-
emitSyncProfileDuration(profile,
|
|
1185
|
-
name: "rateless.
|
|
1186
|
-
|
|
1187
|
-
|
|
1507
|
+
emitSyncProfileDuration(profile, prepareStartedAt, {
|
|
1508
|
+
name: "rateless.prepareStartSyncEncoder",
|
|
1509
|
+
entries: properties.coordinates.length,
|
|
1510
|
+
symbols: initialSymbols?.length,
|
|
1511
|
+
targets: 1,
|
|
1512
|
+
details: {
|
|
1513
|
+
initialSymbolCount: properties.initialSymbolCount,
|
|
1514
|
+
includesInitialSymbols: initialSymbols != null,
|
|
1515
|
+
},
|
|
1188
1516
|
});
|
|
1189
1517
|
}
|
|
1190
|
-
if (!
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1518
|
+
if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
|
|
1519
|
+
freeEncoder();
|
|
1520
|
+
return undefined;
|
|
1521
|
+
}
|
|
1522
|
+
let startSyncSymbols = initialSymbols;
|
|
1523
|
+
if (!startSyncSymbols) {
|
|
1524
|
+
const produceStartedAt = syncProfileStart(profile);
|
|
1525
|
+
startSyncSymbols = produceNextCodedSymbols(encoder, properties.initialSymbolCount);
|
|
1198
1526
|
if (profile) {
|
|
1199
|
-
emitSyncProfileDuration(profile,
|
|
1200
|
-
name: "rateless.
|
|
1201
|
-
|
|
1527
|
+
emitSyncProfileDuration(profile, produceStartedAt, {
|
|
1528
|
+
name: "rateless.produceStartSyncSymbols",
|
|
1529
|
+
symbols: startSyncSymbols.length,
|
|
1202
1530
|
targets: 1,
|
|
1203
|
-
syncId,
|
|
1204
1531
|
});
|
|
1205
1532
|
}
|
|
1206
|
-
return true;
|
|
1207
1533
|
}
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1534
|
+
if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
|
|
1535
|
+
freeEncoder();
|
|
1536
|
+
return undefined;
|
|
1537
|
+
}
|
|
1538
|
+
const startSync = new StartSync({
|
|
1539
|
+
from: start,
|
|
1540
|
+
to: end,
|
|
1541
|
+
symbols: startSyncSymbols,
|
|
1542
|
+
});
|
|
1543
|
+
const syncId = getSyncIdString(startSync);
|
|
1544
|
+
const targetSignal = properties.targetLifecycle.controller.signal;
|
|
1545
|
+
const processController = new AbortController();
|
|
1546
|
+
const processSignal = processController.signal;
|
|
1547
|
+
let cleared = false;
|
|
1215
1548
|
let lastSeqNo = -1n;
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1549
|
+
let symbolsProduced = startSyncSymbols.length;
|
|
1550
|
+
let symbolBudgetExhausted = false;
|
|
1551
|
+
let simpleFallbackPromise;
|
|
1552
|
+
const symbolBudget = getOutgoingRatelessSymbolBudget(properties.coordinates.length, startSyncSymbols.length);
|
|
1553
|
+
let onTargetAbort;
|
|
1554
|
+
const clear = (reason = new Error("rateless outgoing process closed")) => {
|
|
1555
|
+
if (cleared) {
|
|
1556
|
+
return;
|
|
1557
|
+
}
|
|
1558
|
+
cleared = true;
|
|
1559
|
+
// Timeout/replacement owns only the encoder process. Response leases
|
|
1560
|
+
// retain the original open/caller/target lifecycle independently, so a
|
|
1561
|
+
// response already accepted for delivery can finish after this process
|
|
1562
|
+
// expires while close, caller abort, and disconnect still cancel it.
|
|
1563
|
+
processController.abort(reason);
|
|
1564
|
+
if (onTargetAbort) {
|
|
1565
|
+
targetSignal.removeEventListener("abort", onTargetAbort);
|
|
1566
|
+
}
|
|
1567
|
+
if (obj.timeout) {
|
|
1568
|
+
clearTimeout(obj.timeout);
|
|
1569
|
+
}
|
|
1570
|
+
if (obj.deadlineTimeout) {
|
|
1571
|
+
clearTimeout(obj.deadlineTimeout);
|
|
1572
|
+
}
|
|
1573
|
+
if (this.outgoingSyncProcesses.get(syncId) === obj) {
|
|
1574
|
+
this.outgoingSyncProcesses.delete(syncId);
|
|
1575
|
+
}
|
|
1576
|
+
if (this.outgoingSyncProcessByTarget.get(target) === obj) {
|
|
1577
|
+
this.outgoingSyncProcessByTarget.delete(target);
|
|
1578
|
+
}
|
|
1579
|
+
properties.targetLifecycle.retainedByProcess = false;
|
|
1580
|
+
freeEncoder();
|
|
1581
|
+
this.maybeDisposeRatelessDispatchLifecycle(lifecycle);
|
|
1582
|
+
};
|
|
1583
|
+
const startSimpleFallback = () => {
|
|
1584
|
+
if (!simpleFallbackPromise) {
|
|
1585
|
+
// From this point overlapping ResponseMaybeSync authorization
|
|
1586
|
+
// belongs to Simple. Keeping Rateless first would ship the response
|
|
1587
|
+
// while leaving the duplicate Simple lease charged until its TTL.
|
|
1588
|
+
obj.simpleFallbackStarted = true;
|
|
1589
|
+
simpleFallbackPromise = Promise.resolve(this.simple.onMaybeMissingHashes({
|
|
1590
|
+
hashes: properties.outgoingHashes,
|
|
1591
|
+
targets: [target],
|
|
1592
|
+
signal: lifecycle.callerSignal,
|
|
1593
|
+
}));
|
|
1594
|
+
}
|
|
1595
|
+
return simpleFallbackPromise;
|
|
1596
|
+
};
|
|
1597
|
+
const fallbackAndClear = (reason) => {
|
|
1598
|
+
void startSimpleFallback().catch((error) => logger.error(error));
|
|
1599
|
+
clear(reason);
|
|
1600
|
+
};
|
|
1601
|
+
const createTimeout = () => {
|
|
1602
|
+
const timeout = setTimeout(() => fallbackAndClear(new Error("rateless outgoing process timed out")), OUTGOING_RATELESS_IDLE_TIMEOUT_MS);
|
|
1603
|
+
timeout.unref?.();
|
|
1604
|
+
return timeout;
|
|
1605
|
+
};
|
|
1606
|
+
const createDeadlineTimeout = () => {
|
|
1607
|
+
const timeout = setTimeout(() => fallbackAndClear(new Error("rateless outgoing process deadline exceeded")), RATELESS_PROCESS_ABSOLUTE_TIMEOUT_MS);
|
|
1608
|
+
timeout.unref?.();
|
|
1609
|
+
return timeout;
|
|
1610
|
+
};
|
|
1611
|
+
onTargetAbort = () => clear(targetSignal.reason);
|
|
1612
|
+
// Keep follow-up symbol payloads bounded. Each symbol is serialized as an
|
|
1613
|
+
// object with three bigint fields, so very large batches can dominate heap under
|
|
1614
|
+
// concurrent churn even though the native RIBLT encoder itself is compact.
|
|
1615
|
+
const nextBatch = Math.max(MIN_MORE_SYMBOLS_BATCH_SIZE, Math.min(MAX_MORE_SYMBOLS_BATCH_SIZE, Math.ceil(properties.coordinates.length / 4)));
|
|
1616
|
+
obj = {
|
|
1617
|
+
target,
|
|
1618
|
+
targetLifecycle: properties.targetLifecycle,
|
|
1619
|
+
encoder,
|
|
1620
|
+
timeout: undefined,
|
|
1621
|
+
deadlineTimeout: undefined,
|
|
1622
|
+
refresh: () => {
|
|
1623
|
+
if (obj.timeout) {
|
|
1624
|
+
clearTimeout(obj.timeout);
|
|
1625
|
+
}
|
|
1626
|
+
obj.timeout = createTimeout();
|
|
1627
|
+
},
|
|
1628
|
+
next: (message) => {
|
|
1629
|
+
if (processSignal.aborted || symbolBudgetExhausted) {
|
|
1630
|
+
return undefined;
|
|
1631
|
+
}
|
|
1632
|
+
if (message.lastSeqNo !== lastSeqNo + 1n) {
|
|
1633
|
+
return undefined;
|
|
1634
|
+
}
|
|
1635
|
+
lastSeqNo = message.lastSeqNo;
|
|
1636
|
+
const remainingBudget = symbolBudget - symbolsProduced;
|
|
1637
|
+
if (remainingBudget <= 0) {
|
|
1638
|
+
symbolBudgetExhausted = true;
|
|
1639
|
+
return {
|
|
1640
|
+
symbols: CodedSymbolBatch.fromSymbols([]),
|
|
1641
|
+
exhaustedAfterSend: true,
|
|
1642
|
+
};
|
|
1643
|
+
}
|
|
1644
|
+
const produceStartedAt = syncProfileStart(profile);
|
|
1645
|
+
const symbols = produceNextCodedSymbols(encoder, Math.min(nextBatch, remainingBudget));
|
|
1646
|
+
symbolsProduced += symbols.length;
|
|
1647
|
+
const exhaustedAfterSend = symbols.length === 0;
|
|
1648
|
+
if (exhaustedAfterSend) {
|
|
1649
|
+
symbolBudgetExhausted = true;
|
|
1650
|
+
}
|
|
1651
|
+
else {
|
|
1652
|
+
obj.refresh();
|
|
1653
|
+
}
|
|
1654
|
+
if (profile) {
|
|
1655
|
+
emitSyncProfileDuration(profile, produceStartedAt, {
|
|
1656
|
+
name: "rateless.produceMoreSymbols",
|
|
1657
|
+
syncId,
|
|
1658
|
+
symbols: symbols.length,
|
|
1659
|
+
targets: 1,
|
|
1660
|
+
});
|
|
1661
|
+
}
|
|
1662
|
+
return { symbols, exhaustedAfterSend };
|
|
1663
|
+
},
|
|
1664
|
+
startSimpleFallback,
|
|
1665
|
+
simpleFallbackStarted: false,
|
|
1666
|
+
free: clear,
|
|
1667
|
+
outgoingHashes: properties.outgoingHashes,
|
|
1668
|
+
authorizedHashes: properties.authorizedHashes,
|
|
1669
|
+
consumedResponseHashes: new Set(),
|
|
1670
|
+
processController,
|
|
1671
|
+
signal: processSignal,
|
|
1672
|
+
callerSignal: lifecycle.callerSignal,
|
|
1673
|
+
};
|
|
1674
|
+
processConstructed = true;
|
|
1675
|
+
if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
|
|
1676
|
+
clear();
|
|
1677
|
+
return undefined;
|
|
1678
|
+
}
|
|
1679
|
+
// Without a wire process id on ResponseMaybeSync, retaining at most one
|
|
1680
|
+
// process per target keeps sender/hash authorization unambiguous.
|
|
1681
|
+
this.outgoingSyncProcessByTarget
|
|
1682
|
+
.get(target)
|
|
1683
|
+
?.free(new Error("rateless outgoing process replaced"));
|
|
1684
|
+
if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
|
|
1685
|
+
clear();
|
|
1686
|
+
return undefined;
|
|
1687
|
+
}
|
|
1688
|
+
properties.targetLifecycle.retainedByProcess = true;
|
|
1689
|
+
this.outgoingSyncProcesses.set(syncId, obj);
|
|
1690
|
+
this.outgoingSyncProcessByTarget.set(target, obj);
|
|
1691
|
+
obj.timeout = createTimeout();
|
|
1692
|
+
obj.deadlineTimeout = createDeadlineTimeout();
|
|
1693
|
+
targetSignal.addEventListener("abort", onTargetAbort, { once: true });
|
|
1694
|
+
encoderTransferred = true;
|
|
1695
|
+
if (targetSignal.aborted) {
|
|
1696
|
+
clear(targetSignal.reason);
|
|
1697
|
+
return undefined;
|
|
1698
|
+
}
|
|
1699
|
+
if (profile) {
|
|
1700
|
+
emitSyncProfileEvent(profile, {
|
|
1701
|
+
name: "rateless.dispatchMode",
|
|
1702
|
+
entries: properties.entryCount,
|
|
1703
|
+
symbols: startSyncSymbols.length,
|
|
1704
|
+
targets: 1,
|
|
1705
|
+
syncId,
|
|
1706
|
+
details: { mode: "rateless" },
|
|
1707
|
+
});
|
|
1708
|
+
}
|
|
1709
|
+
const sendStartedAt = syncProfileStart(profile);
|
|
1710
|
+
let sendResult;
|
|
1711
|
+
try {
|
|
1712
|
+
sendResult = Promise.resolve(this.simple.rpc.send(startSync, {
|
|
1713
|
+
mode: new SilentDelivery({
|
|
1714
|
+
to: [target],
|
|
1715
|
+
redundancy: 1,
|
|
1716
|
+
}),
|
|
1717
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
1718
|
+
signal: processSignal,
|
|
1719
|
+
}));
|
|
1720
|
+
}
|
|
1721
|
+
catch (error) {
|
|
1722
|
+
sendResult = Promise.reject(error);
|
|
1723
|
+
}
|
|
1724
|
+
void sendResult.then(() => {
|
|
1725
|
+
if (profile) {
|
|
1726
|
+
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
1727
|
+
name: "rateless.sendStartSync",
|
|
1728
|
+
messages: 1,
|
|
1729
|
+
symbols: startSyncSymbols.length,
|
|
1730
|
+
targets: 1,
|
|
1731
|
+
syncId,
|
|
1732
|
+
});
|
|
1733
|
+
}
|
|
1734
|
+
}, () => {
|
|
1735
|
+
clear();
|
|
1736
|
+
if (profile) {
|
|
1737
|
+
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
1738
|
+
name: "rateless.sendStartSync",
|
|
1739
|
+
messages: 1,
|
|
1740
|
+
symbols: startSyncSymbols.length,
|
|
1741
|
+
targets: 1,
|
|
1742
|
+
syncId,
|
|
1743
|
+
details: {
|
|
1744
|
+
rejected: true,
|
|
1745
|
+
cancelled: processSignal.aborted || undefined,
|
|
1746
|
+
},
|
|
1747
|
+
});
|
|
1748
|
+
}
|
|
1749
|
+
});
|
|
1750
|
+
if (processSignal.aborted) {
|
|
1751
|
+
clear();
|
|
1752
|
+
return undefined;
|
|
1753
|
+
}
|
|
1754
|
+
return startSyncSymbols.length;
|
|
1755
|
+
}
|
|
1756
|
+
catch (error) {
|
|
1757
|
+
if (processConstructed) {
|
|
1758
|
+
obj.free(error);
|
|
1759
|
+
}
|
|
1760
|
+
throw error;
|
|
1761
|
+
}
|
|
1762
|
+
finally {
|
|
1763
|
+
if (!encoderTransferred) {
|
|
1764
|
+
freeEncoder();
|
|
1765
|
+
}
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
consumeAuthorizedRatelessResponse(hashes, from) {
|
|
1769
|
+
const target = from.hashcode();
|
|
1770
|
+
const process = this.outgoingSyncProcessByTarget.get(target);
|
|
1771
|
+
if (!process ||
|
|
1772
|
+
process.signal.aborted ||
|
|
1773
|
+
!this.isRatelessDispatchLifecycleActive(process.targetLifecycle.lifecycle, target)) {
|
|
1774
|
+
return undefined;
|
|
1775
|
+
}
|
|
1776
|
+
if (this.activeRatelessResponseCount >=
|
|
1777
|
+
MAX_ACTIVE_RATELESS_RESPONSES_GLOBAL ||
|
|
1778
|
+
(this.activeRatelessResponseCountByPeer.get(target) ?? 0) >=
|
|
1779
|
+
MAX_ACTIVE_RATELESS_RESPONSES_PER_PEER) {
|
|
1780
|
+
return undefined;
|
|
1781
|
+
}
|
|
1782
|
+
const authorized = [];
|
|
1783
|
+
const remaining = [];
|
|
1784
|
+
const seen = new Set();
|
|
1785
|
+
for (const hash of hashes) {
|
|
1786
|
+
if (seen.has(hash)) {
|
|
1787
|
+
continue;
|
|
1788
|
+
}
|
|
1789
|
+
seen.add(hash);
|
|
1790
|
+
if (!process.authorizedHashes.has(hash) ||
|
|
1791
|
+
process.consumedResponseHashes.has(hash)) {
|
|
1792
|
+
remaining.push(hash);
|
|
1793
|
+
continue;
|
|
1794
|
+
}
|
|
1795
|
+
process.consumedResponseHashes.add(hash);
|
|
1796
|
+
authorized.push(hash);
|
|
1797
|
+
}
|
|
1798
|
+
if (authorized.length === 0) {
|
|
1799
|
+
return {
|
|
1800
|
+
process,
|
|
1801
|
+
authorized,
|
|
1802
|
+
remaining,
|
|
1803
|
+
signal: process.targetLifecycle.controller.signal,
|
|
1804
|
+
release: () => { },
|
|
1805
|
+
};
|
|
1806
|
+
}
|
|
1807
|
+
const targetLifecycle = process.targetLifecycle;
|
|
1808
|
+
targetLifecycle.responseLeases += 1;
|
|
1809
|
+
this.activeRatelessResponseCount += 1;
|
|
1810
|
+
this.activeRatelessResponseCountByPeer.set(target, (this.activeRatelessResponseCountByPeer.get(target) ?? 0) + 1);
|
|
1811
|
+
let released = false;
|
|
1812
|
+
return {
|
|
1813
|
+
process,
|
|
1814
|
+
authorized,
|
|
1815
|
+
remaining,
|
|
1816
|
+
signal: targetLifecycle.controller.signal,
|
|
1817
|
+
release: (options) => {
|
|
1818
|
+
if (released) {
|
|
1819
|
+
return;
|
|
1820
|
+
}
|
|
1821
|
+
released = true;
|
|
1822
|
+
if (options?.rollback) {
|
|
1823
|
+
for (const hash of authorized) {
|
|
1824
|
+
process.consumedResponseHashes.delete(hash);
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
1827
|
+
targetLifecycle.responseLeases -= 1;
|
|
1828
|
+
this.activeRatelessResponseCount -= 1;
|
|
1829
|
+
const activeForPeer = (this.activeRatelessResponseCountByPeer.get(target) ?? 1) - 1;
|
|
1830
|
+
if (activeForPeer === 0) {
|
|
1831
|
+
this.activeRatelessResponseCountByPeer.delete(target);
|
|
1832
|
+
}
|
|
1833
|
+
else {
|
|
1834
|
+
this.activeRatelessResponseCountByPeer.set(target, activeForPeer);
|
|
1835
|
+
}
|
|
1836
|
+
this.maybeDisposeRatelessDispatchLifecycle(targetLifecycle.lifecycle);
|
|
1837
|
+
},
|
|
1838
|
+
};
|
|
1839
|
+
}
|
|
1840
|
+
async onMessage(message, context) {
|
|
1841
|
+
const profile = this.properties.sync?.profile;
|
|
1842
|
+
if (message instanceof StartSync) {
|
|
1843
|
+
const from = context.from;
|
|
1844
|
+
const ownershipLifecycleController = this.ratelessDispatchLifecycleController;
|
|
1845
|
+
if (!from ||
|
|
1846
|
+
!this.isIncomingSyncGenerationActive(ownershipLifecycleController)) {
|
|
1847
|
+
return true;
|
|
1848
|
+
}
|
|
1849
|
+
const sender = from.hashcode();
|
|
1850
|
+
const syncId = getSyncIdString(message);
|
|
1851
|
+
const key = this.getIncomingSyncProcessKey(sender, syncId);
|
|
1852
|
+
const completedSynchronizations = this.startedOrCompletedSynchronizations;
|
|
1853
|
+
const admissionSet = this.incomingRatelessProcessAdmissions;
|
|
1854
|
+
if (this.ingoingSyncProcesses.has(key)) {
|
|
1855
|
+
return true;
|
|
1856
|
+
}
|
|
1857
|
+
if (completedSynchronizations.has(key)) {
|
|
1858
|
+
return true;
|
|
1859
|
+
}
|
|
1860
|
+
let admissionsForSender = 0;
|
|
1861
|
+
for (const admission of admissionSet) {
|
|
1862
|
+
if (admission.key === key) {
|
|
1863
|
+
return true;
|
|
1864
|
+
}
|
|
1865
|
+
if (admission.sender === sender) {
|
|
1866
|
+
admissionsForSender += 1;
|
|
1867
|
+
}
|
|
1868
|
+
}
|
|
1869
|
+
if (admissionSet.size >= MAX_INCOMING_RATELESS_PROCESSES ||
|
|
1870
|
+
admissionsForSender >= MAX_INCOMING_RATELESS_PROCESSES_PER_SENDER) {
|
|
1871
|
+
return true;
|
|
1872
|
+
}
|
|
1873
|
+
const admission = { key, sender };
|
|
1874
|
+
admissionSet.add(admission);
|
|
1875
|
+
const controller = new AbortController();
|
|
1876
|
+
let freed = false;
|
|
1877
|
+
let completed = false;
|
|
1878
|
+
let initializationPending = false;
|
|
1879
|
+
let fallbackSendPending = false;
|
|
1880
|
+
let admissionReleased = false;
|
|
1881
|
+
let fallbackGraceTimeout;
|
|
1882
|
+
let onOwnershipAbort;
|
|
1883
|
+
let obj;
|
|
1884
|
+
const releaseAdmissionIfSettled = () => {
|
|
1885
|
+
if (admissionReleased ||
|
|
1886
|
+
!freed ||
|
|
1887
|
+
initializationPending ||
|
|
1888
|
+
fallbackSendPending) {
|
|
1889
|
+
return;
|
|
1890
|
+
}
|
|
1891
|
+
admissionReleased = true;
|
|
1892
|
+
admissionSet.delete(admission);
|
|
1893
|
+
};
|
|
1894
|
+
const free = (reason = new Error("incoming rateless process closed")) => {
|
|
1895
|
+
if (freed) {
|
|
1896
|
+
return;
|
|
1897
|
+
}
|
|
1898
|
+
freed = true;
|
|
1899
|
+
// Abort first: no queued process/send continuation may observe the
|
|
1900
|
+
// object detached while its native decoder is still usable.
|
|
1901
|
+
controller.abort(reason);
|
|
1902
|
+
if (onOwnershipAbort) {
|
|
1903
|
+
ownershipLifecycleController.signal.removeEventListener("abort", onOwnershipAbort);
|
|
1904
|
+
}
|
|
1905
|
+
if (obj.timeout) {
|
|
1906
|
+
clearTimeout(obj.timeout);
|
|
1907
|
+
obj.timeout = undefined;
|
|
1908
|
+
}
|
|
1909
|
+
if (obj.deadlineTimeout) {
|
|
1910
|
+
clearTimeout(obj.deadlineTimeout);
|
|
1911
|
+
obj.deadlineTimeout = undefined;
|
|
1912
|
+
}
|
|
1913
|
+
if (fallbackGraceTimeout) {
|
|
1914
|
+
clearTimeout(fallbackGraceTimeout);
|
|
1915
|
+
fallbackGraceTimeout = undefined;
|
|
1916
|
+
}
|
|
1917
|
+
if (this.ingoingSyncProcesses.get(key) === obj) {
|
|
1918
|
+
this.ingoingSyncProcesses.delete(key);
|
|
1919
|
+
}
|
|
1920
|
+
const ownedDecoder = obj.decoder;
|
|
1921
|
+
obj.decoder = undefined;
|
|
1922
|
+
ownedDecoder?.free();
|
|
1923
|
+
releaseAdmissionIfSettled();
|
|
1924
|
+
};
|
|
1925
|
+
const complete = () => {
|
|
1926
|
+
if (completed || !this.isIncomingSyncProcessActive(obj)) {
|
|
1927
|
+
return;
|
|
1928
|
+
}
|
|
1929
|
+
completed = true;
|
|
1930
|
+
completedSynchronizations.add(key);
|
|
1931
|
+
free(new Error("incoming rateless process completed"));
|
|
1932
|
+
};
|
|
1933
|
+
obj = {
|
|
1934
|
+
key,
|
|
1935
|
+
syncId,
|
|
1936
|
+
sender,
|
|
1937
|
+
from,
|
|
1938
|
+
ownershipLifecycleController,
|
|
1939
|
+
controller,
|
|
1940
|
+
completedSynchronizations,
|
|
1941
|
+
timeout: undefined,
|
|
1942
|
+
deadlineTimeout: undefined,
|
|
1943
|
+
refresh: () => { },
|
|
1944
|
+
lastSeqNo: -1n,
|
|
1945
|
+
processedSymbols: 0,
|
|
1946
|
+
queuedSymbols: 0,
|
|
1947
|
+
symbolBudget: getIncomingRatelessSymbolBudget(message.symbols.length),
|
|
1948
|
+
process: async () => undefined,
|
|
1949
|
+
requestAll: async () => { },
|
|
1950
|
+
fallbackToSimple: async () => { },
|
|
1951
|
+
free,
|
|
1952
|
+
complete,
|
|
1953
|
+
};
|
|
1954
|
+
this.ingoingSyncProcesses.set(key, obj);
|
|
1955
|
+
onOwnershipAbort = () => free(ownershipLifecycleController.signal.reason);
|
|
1956
|
+
ownershipLifecycleController.signal.addEventListener("abort", onOwnershipAbort, { once: true });
|
|
1957
|
+
obj.deadlineTimeout = setTimeout(() => {
|
|
1958
|
+
void obj.fallbackToSimple(new Error("incoming rateless process deadline exceeded"));
|
|
1959
|
+
}, RATELESS_PROCESS_ABSOLUTE_TIMEOUT_MS);
|
|
1960
|
+
obj.deadlineTimeout.unref?.();
|
|
1961
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
1962
|
+
free(ownershipLifecycleController.signal.reason);
|
|
1963
|
+
return true;
|
|
1964
|
+
}
|
|
1965
|
+
let requestAllPromise;
|
|
1966
|
+
obj.requestAll = () => {
|
|
1967
|
+
if (requestAllPromise) {
|
|
1968
|
+
return requestAllPromise;
|
|
1969
|
+
}
|
|
1970
|
+
requestAllPromise = (async () => {
|
|
1971
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
1972
|
+
return;
|
|
1973
|
+
}
|
|
1974
|
+
fallbackSendPending = true;
|
|
1975
|
+
const sendStartedAt = syncProfileStart(profile);
|
|
1976
|
+
try {
|
|
1977
|
+
await this.simple.rpc.send(new RequestAll({
|
|
1978
|
+
syncId: message.syncId,
|
|
1979
|
+
}), {
|
|
1980
|
+
mode: new SilentDelivery({
|
|
1981
|
+
to: [obj.sender],
|
|
1982
|
+
redundancy: 1,
|
|
1983
|
+
}),
|
|
1984
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
1985
|
+
signal: controller.signal,
|
|
1986
|
+
});
|
|
1987
|
+
if (this.isIncomingSyncProcessActive(obj) && profile) {
|
|
1988
|
+
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
1989
|
+
name: "rateless.sendRequestAll",
|
|
1990
|
+
messages: 1,
|
|
1991
|
+
targets: 1,
|
|
1992
|
+
syncId,
|
|
1993
|
+
});
|
|
1994
|
+
}
|
|
1995
|
+
}
|
|
1996
|
+
finally {
|
|
1997
|
+
fallbackSendPending = false;
|
|
1998
|
+
if (this.isIncomingSyncProcessActive(obj)) {
|
|
1999
|
+
complete();
|
|
2000
|
+
}
|
|
2001
|
+
releaseAdmissionIfSettled();
|
|
2002
|
+
}
|
|
2003
|
+
})();
|
|
2004
|
+
return requestAllPromise;
|
|
2005
|
+
};
|
|
2006
|
+
let boundedFallbackPromise;
|
|
2007
|
+
obj.fallbackToSimple = (reason = new Error("incoming rateless fallback timed out")) => {
|
|
2008
|
+
if (boundedFallbackPromise) {
|
|
2009
|
+
return boundedFallbackPromise;
|
|
2010
|
+
}
|
|
2011
|
+
const requestAll = obj.requestAll();
|
|
2012
|
+
boundedFallbackPromise = new Promise((resolve) => {
|
|
2013
|
+
let settled = false;
|
|
2014
|
+
const settle = () => {
|
|
2015
|
+
if (settled) {
|
|
2016
|
+
return;
|
|
2017
|
+
}
|
|
2018
|
+
settled = true;
|
|
2019
|
+
controller.signal.removeEventListener("abort", onAbort);
|
|
2020
|
+
if (fallbackGraceTimeout) {
|
|
2021
|
+
clearTimeout(fallbackGraceTimeout);
|
|
2022
|
+
fallbackGraceTimeout = undefined;
|
|
2023
|
+
}
|
|
2024
|
+
resolve();
|
|
2025
|
+
};
|
|
2026
|
+
const onAbort = () => settle();
|
|
2027
|
+
controller.signal.addEventListener("abort", onAbort, {
|
|
2028
|
+
once: true,
|
|
2029
|
+
});
|
|
2030
|
+
fallbackGraceTimeout = setTimeout(() => {
|
|
2031
|
+
free(reason);
|
|
2032
|
+
settle();
|
|
2033
|
+
}, INCOMING_RATELESS_FALLBACK_GRACE_MS);
|
|
2034
|
+
fallbackGraceTimeout.unref?.();
|
|
2035
|
+
void requestAll.then(settle, settle);
|
|
2036
|
+
if (controller.signal.aborted) {
|
|
2037
|
+
onAbort();
|
|
2038
|
+
}
|
|
2039
|
+
});
|
|
2040
|
+
return boundedFallbackPromise;
|
|
2041
|
+
};
|
|
2042
|
+
if (message.symbols.length > MAX_MORE_SYMBOLS_BATCH_SIZE) {
|
|
2043
|
+
await obj.fallbackToSimple(new Error("oversized incoming rateless initial batch"));
|
|
2044
|
+
return true;
|
|
2045
|
+
}
|
|
2046
|
+
const wrapped = message.end < message.start;
|
|
2047
|
+
const decoderStartedAt = syncProfileStart(profile);
|
|
2048
|
+
let decoder;
|
|
2049
|
+
initializationPending = true;
|
|
2050
|
+
try {
|
|
2051
|
+
decoder = await this.getLocalDecoderForRange({
|
|
2052
|
+
start1: message.start,
|
|
2053
|
+
end1: wrapped ? this.properties.numbers.maxValue : message.end,
|
|
2054
|
+
start2: 0n,
|
|
2055
|
+
end2: wrapped ? message.end : 0n,
|
|
2056
|
+
}, {
|
|
2057
|
+
ownershipLifecycleController,
|
|
2058
|
+
signal: controller.signal,
|
|
2059
|
+
});
|
|
2060
|
+
}
|
|
2061
|
+
catch (error) {
|
|
2062
|
+
const processAborted = controller.signal.aborted;
|
|
2063
|
+
free(error);
|
|
2064
|
+
if (processAborted ||
|
|
2065
|
+
!this.isIncomingSyncGenerationActive(ownershipLifecycleController)) {
|
|
2066
|
+
return true;
|
|
2067
|
+
}
|
|
2068
|
+
throw error;
|
|
2069
|
+
}
|
|
2070
|
+
finally {
|
|
2071
|
+
initializationPending = false;
|
|
2072
|
+
releaseAdmissionIfSettled();
|
|
2073
|
+
}
|
|
2074
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2075
|
+
decoder && decoder.free();
|
|
2076
|
+
free(controller.signal.reason);
|
|
2077
|
+
return true;
|
|
2078
|
+
}
|
|
2079
|
+
if (decoder) {
|
|
2080
|
+
// Transfer the native decoder before invoking diagnostics. A profile
|
|
2081
|
+
// sink is caller code and may throw; from this point process cleanup
|
|
2082
|
+
// owns the decoder on every exit.
|
|
2083
|
+
obj.decoder = decoder;
|
|
2084
|
+
}
|
|
2085
|
+
try {
|
|
2086
|
+
if (profile) {
|
|
2087
|
+
emitSyncProfileDuration(profile, decoderStartedAt, {
|
|
2088
|
+
name: "rateless.getLocalDecoderForRange",
|
|
2089
|
+
syncId,
|
|
2090
|
+
details: { wrapped, found: decoder !== false },
|
|
2091
|
+
});
|
|
2092
|
+
}
|
|
2093
|
+
}
|
|
2094
|
+
catch (error) {
|
|
2095
|
+
free(error);
|
|
2096
|
+
throw error;
|
|
2097
|
+
}
|
|
2098
|
+
if (!decoder) {
|
|
2099
|
+
try {
|
|
2100
|
+
await obj.fallbackToSimple(new Error("incoming rateless decoder unavailable"));
|
|
2101
|
+
}
|
|
2102
|
+
catch (error) {
|
|
2103
|
+
if (controller.signal.aborted) {
|
|
2104
|
+
return true;
|
|
1235
2105
|
}
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
2106
|
+
throw error;
|
|
2107
|
+
}
|
|
2108
|
+
return true;
|
|
2109
|
+
}
|
|
2110
|
+
const createTimeout = () => {
|
|
2111
|
+
const timeout = setTimeout(() => {
|
|
2112
|
+
void obj.fallbackToSimple(new Error("incoming rateless process timed out"));
|
|
2113
|
+
}, INCOMING_RATELESS_IDLE_TIMEOUT_MS);
|
|
2114
|
+
timeout.unref?.();
|
|
2115
|
+
return timeout;
|
|
2116
|
+
};
|
|
2117
|
+
let messageQueue = [];
|
|
2118
|
+
obj.refresh = () => {
|
|
2119
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2120
|
+
return;
|
|
2121
|
+
}
|
|
2122
|
+
if (obj.timeout) {
|
|
2123
|
+
clearTimeout(obj.timeout);
|
|
2124
|
+
}
|
|
2125
|
+
obj.timeout = createTimeout();
|
|
2126
|
+
};
|
|
2127
|
+
obj.process = async (newMessage) => {
|
|
2128
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2129
|
+
return undefined;
|
|
2130
|
+
}
|
|
2131
|
+
const symbolCount = CodedSymbolBatch.from(newMessage.symbols).length;
|
|
2132
|
+
if (symbolCount > MAX_MORE_SYMBOLS_BATCH_SIZE) {
|
|
2133
|
+
free(new Error("incoming rateless symbol batch exceeds limit"));
|
|
2134
|
+
return undefined;
|
|
2135
|
+
}
|
|
2136
|
+
if (newMessage.seqNo <= obj.lastSeqNo) {
|
|
2137
|
+
return undefined;
|
|
2138
|
+
}
|
|
2139
|
+
if (newMessage.seqNo >
|
|
2140
|
+
obj.lastSeqNo + MAX_INCOMING_RATELESS_SEQUENCE_GAP) {
|
|
2141
|
+
return undefined;
|
|
2142
|
+
}
|
|
2143
|
+
if (messageQueue.some((queued) => queued.seqNo === newMessage.seqNo)) {
|
|
2144
|
+
return undefined;
|
|
2145
|
+
}
|
|
2146
|
+
if (newMessage.seqNo !== obj.lastSeqNo + 1n &&
|
|
2147
|
+
messageQueue.length >= MAX_INCOMING_RATELESS_QUEUED_BATCHES) {
|
|
2148
|
+
return undefined;
|
|
2149
|
+
}
|
|
2150
|
+
if (obj.processedSymbols + obj.queuedSymbols + symbolCount >
|
|
2151
|
+
obj.symbolBudget) {
|
|
2152
|
+
return "fallback-to-simple";
|
|
2153
|
+
}
|
|
2154
|
+
if (newMessage.seqNo > 0n && symbolCount === 0) {
|
|
2155
|
+
return "fallback-to-simple";
|
|
2156
|
+
}
|
|
2157
|
+
messageQueue.push({ ...newMessage, symbolCount });
|
|
2158
|
+
obj.queuedSymbols += symbolCount;
|
|
2159
|
+
messageQueue.sort((a, b) => Number(a.seqNo - b.seqNo));
|
|
2160
|
+
if (messageQueue[0].seqNo !== obj.lastSeqNo + 1n) {
|
|
2161
|
+
return;
|
|
2162
|
+
}
|
|
2163
|
+
const finalizeIfDecoded = () => {
|
|
2164
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2165
|
+
return true;
|
|
2166
|
+
}
|
|
2167
|
+
if (!decoder.decoded()) {
|
|
2168
|
+
return false;
|
|
2169
|
+
}
|
|
2170
|
+
const remoteStartedAt = syncProfileStart(profile);
|
|
2171
|
+
const allMissingSymbolsInRemote = getRemoteSymbolValues(decoder);
|
|
2172
|
+
if (profile) {
|
|
2173
|
+
emitSyncProfileDuration(profile, remoteStartedAt, {
|
|
2174
|
+
name: "rateless.remoteSymbols",
|
|
2175
|
+
entries: allMissingSymbolsInRemote.length,
|
|
2176
|
+
symbols: allMissingSymbolsInRemote.length,
|
|
2177
|
+
syncId,
|
|
2178
|
+
});
|
|
2179
|
+
}
|
|
2180
|
+
// The IBLT decoder is based on a local snapshot. Entries can arrive via
|
|
2181
|
+
// overlapping repair before we issue the follow-up simple request, so
|
|
2182
|
+
// re-check local presence to avoid stale duplicate bounce-back.
|
|
2183
|
+
void this.simple
|
|
2184
|
+
.queueSync(allMissingSymbolsInRemote, obj.from)
|
|
2185
|
+
.catch((error) => logger.error(error));
|
|
2186
|
+
obj.complete();
|
|
2187
|
+
return true;
|
|
2188
|
+
};
|
|
2189
|
+
while (messageQueue.length > 0 &&
|
|
2190
|
+
messageQueue[0].seqNo === obj.lastSeqNo + 1n) {
|
|
2191
|
+
const symbolMessage = messageQueue.shift();
|
|
2192
|
+
if (!symbolMessage) {
|
|
2193
|
+
break;
|
|
2194
|
+
}
|
|
2195
|
+
obj.queuedSymbols -= symbolMessage.symbolCount;
|
|
2196
|
+
obj.lastSeqNo = symbolMessage.seqNo;
|
|
2197
|
+
obj.processedSymbols += symbolMessage.symbolCount;
|
|
2198
|
+
// Only an authenticated, previously unseen contiguous sequence advances
|
|
2199
|
+
// the idle deadline. Replays and speculative future batches do not.
|
|
2200
|
+
obj.refresh();
|
|
2201
|
+
const addBatchAndDecode = decoder
|
|
2202
|
+
.add_coded_symbols_and_try_decode;
|
|
2203
|
+
if (typeof BigUint64Array !== "undefined" && addBatchAndDecode) {
|
|
2204
|
+
const flatStartedAt = syncProfileStart(profile);
|
|
2205
|
+
const flatSymbols = flatFromCodedSymbols(symbolMessage.symbols);
|
|
2206
|
+
if (profile) {
|
|
2207
|
+
emitSyncProfileDuration(profile, flatStartedAt, {
|
|
2208
|
+
name: "rateless.symbolBatchToFlat",
|
|
2209
|
+
symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
|
|
2210
|
+
syncId,
|
|
2211
|
+
});
|
|
1239
2212
|
}
|
|
1240
|
-
const
|
|
1241
|
-
const
|
|
2213
|
+
const decodeStartedAt = syncProfileStart(profile);
|
|
2214
|
+
const decoded = addBatchAndDecode.call(decoder, flatSymbols);
|
|
1242
2215
|
if (profile) {
|
|
1243
|
-
emitSyncProfileDuration(profile,
|
|
1244
|
-
name: "rateless.
|
|
1245
|
-
|
|
1246
|
-
symbols: allMissingSymbolsInRemote.length,
|
|
2216
|
+
emitSyncProfileDuration(profile, decodeStartedAt, {
|
|
2217
|
+
name: "rateless.decodeBatch",
|
|
2218
|
+
symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
|
|
1247
2219
|
syncId,
|
|
2220
|
+
details: { decoded },
|
|
1248
2221
|
});
|
|
1249
2222
|
}
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
// re-check local presence to avoid stale duplicate bounce-back.
|
|
1253
|
-
this.simple.queueSync(allMissingSymbolsInRemote, context.from);
|
|
1254
|
-
obj.free();
|
|
1255
|
-
return true;
|
|
1256
|
-
};
|
|
1257
|
-
while (messageQueue.length > 0 &&
|
|
1258
|
-
messageQueue[0].seqNo === lastSeqNo + 1n) {
|
|
1259
|
-
const symbolMessage = messageQueue.shift();
|
|
1260
|
-
if (!symbolMessage) {
|
|
1261
|
-
break;
|
|
2223
|
+
if (decoded && finalizeIfDecoded()) {
|
|
2224
|
+
return true;
|
|
1262
2225
|
}
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
}
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
name: "rateless.decodeBatch",
|
|
1281
|
-
symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
|
|
1282
|
-
syncId,
|
|
1283
|
-
details: { decoded },
|
|
1284
|
-
});
|
|
1285
|
-
}
|
|
1286
|
-
if (decoded && finalizeIfDecoded()) {
|
|
2226
|
+
continue;
|
|
2227
|
+
}
|
|
2228
|
+
const decodeLoopStartedAt = syncProfileStart(profile);
|
|
2229
|
+
let symbolsProcessed = 0;
|
|
2230
|
+
for (const symbol of CodedSymbolBatch.from(symbolMessage.symbols)) {
|
|
2231
|
+
symbolsProcessed += 1;
|
|
2232
|
+
const normalizedSymbol = symbol instanceof SymbolSerialized
|
|
2233
|
+
? symbol
|
|
2234
|
+
: new SymbolSerialized({
|
|
2235
|
+
count: symbol.count,
|
|
2236
|
+
hash: symbol.hash,
|
|
2237
|
+
symbol: symbol.symbol,
|
|
2238
|
+
});
|
|
2239
|
+
decoder.add_coded_symbol(normalizedSymbol);
|
|
2240
|
+
try {
|
|
2241
|
+
decoder.try_decode();
|
|
2242
|
+
if (finalizeIfDecoded()) {
|
|
1287
2243
|
return true;
|
|
1288
2244
|
}
|
|
1289
|
-
continue;
|
|
1290
2245
|
}
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
? symbol
|
|
1297
|
-
: new SymbolSerialized({
|
|
1298
|
-
count: symbol.count,
|
|
1299
|
-
hash: symbol.hash,
|
|
1300
|
-
symbol: symbol.symbol,
|
|
1301
|
-
});
|
|
1302
|
-
decoder.add_coded_symbol(normalizedSymbol);
|
|
1303
|
-
try {
|
|
1304
|
-
decoder.try_decode();
|
|
1305
|
-
if (finalizeIfDecoded()) {
|
|
1306
|
-
return true;
|
|
1307
|
-
}
|
|
2246
|
+
catch (error) {
|
|
2247
|
+
if (error?.message === "Invalid degree" ||
|
|
2248
|
+
error === "Invalid degree") {
|
|
2249
|
+
logger.trace("Decoder reported invalid degree; waiting for more symbols");
|
|
2250
|
+
continue;
|
|
1308
2251
|
}
|
|
1309
|
-
|
|
1310
|
-
if (error?.message === "Invalid degree" ||
|
|
1311
|
-
error === "Invalid degree") {
|
|
1312
|
-
logger.trace("Decoder reported invalid degree; waiting for more symbols");
|
|
1313
|
-
continue;
|
|
1314
|
-
}
|
|
1315
|
-
throw error;
|
|
1316
|
-
}
|
|
1317
|
-
}
|
|
1318
|
-
if (profile) {
|
|
1319
|
-
emitSyncProfileDuration(profile, decodeLoopStartedAt, {
|
|
1320
|
-
name: "rateless.decodeSymbolLoop",
|
|
1321
|
-
symbols: symbolsProcessed,
|
|
1322
|
-
syncId,
|
|
1323
|
-
});
|
|
2252
|
+
throw error;
|
|
1324
2253
|
}
|
|
1325
2254
|
}
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
2255
|
+
if (profile) {
|
|
2256
|
+
emitSyncProfileDuration(profile, decodeLoopStartedAt, {
|
|
2257
|
+
name: "rateless.decodeSymbolLoop",
|
|
2258
|
+
symbols: symbolsProcessed,
|
|
2259
|
+
syncId,
|
|
2260
|
+
});
|
|
2261
|
+
}
|
|
2262
|
+
}
|
|
2263
|
+
return false;
|
|
1333
2264
|
};
|
|
1334
|
-
|
|
1335
|
-
|
|
2265
|
+
obj.timeout = createTimeout();
|
|
2266
|
+
let initialResult;
|
|
2267
|
+
try {
|
|
2268
|
+
initialResult = await obj.process({
|
|
2269
|
+
seqNo: 0n,
|
|
2270
|
+
symbols: message.symbols,
|
|
2271
|
+
});
|
|
2272
|
+
}
|
|
2273
|
+
catch (error) {
|
|
2274
|
+
const wasActive = this.isIncomingSyncProcessActive(obj);
|
|
2275
|
+
free(error);
|
|
2276
|
+
if (!wasActive) {
|
|
2277
|
+
return true;
|
|
2278
|
+
}
|
|
2279
|
+
throw error;
|
|
2280
|
+
}
|
|
2281
|
+
if (initialResult === true) {
|
|
2282
|
+
return true;
|
|
2283
|
+
}
|
|
2284
|
+
if (initialResult === "fallback-to-simple") {
|
|
2285
|
+
await obj.fallbackToSimple(new Error("incoming rateless symbol budget exhausted"));
|
|
2286
|
+
return true;
|
|
2287
|
+
}
|
|
2288
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
1336
2289
|
return true;
|
|
1337
2290
|
}
|
|
1338
2291
|
// not done, request more symbols
|
|
1339
2292
|
const sendStartedAt = syncProfileStart(profile);
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
2293
|
+
try {
|
|
2294
|
+
await this.simple.rpc.send(new RequestMoreSymbols({
|
|
2295
|
+
lastSeqNo: obj.lastSeqNo,
|
|
2296
|
+
syncId: message.syncId,
|
|
2297
|
+
}), {
|
|
2298
|
+
mode: new SilentDelivery({ to: [obj.sender], redundancy: 1 }),
|
|
2299
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
2300
|
+
signal: controller.signal,
|
|
2301
|
+
});
|
|
2302
|
+
}
|
|
2303
|
+
catch (error) {
|
|
2304
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2305
|
+
return true;
|
|
2306
|
+
}
|
|
2307
|
+
free(error);
|
|
2308
|
+
throw error;
|
|
2309
|
+
}
|
|
2310
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2311
|
+
return true;
|
|
2312
|
+
}
|
|
1347
2313
|
if (profile) {
|
|
1348
2314
|
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
1349
2315
|
name: "rateless.sendRequestMoreSymbols",
|
|
@@ -1355,40 +2321,84 @@ export class RatelessIBLTSynchronizer {
|
|
|
1355
2321
|
return true;
|
|
1356
2322
|
}
|
|
1357
2323
|
else if (message instanceof MoreSymbols) {
|
|
2324
|
+
const from = context.from;
|
|
2325
|
+
if (!from) {
|
|
2326
|
+
return true;
|
|
2327
|
+
}
|
|
2328
|
+
const sender = from.hashcode();
|
|
1358
2329
|
const syncId = getSyncIdString(message);
|
|
1359
|
-
const
|
|
1360
|
-
|
|
2330
|
+
const key = this.getIncomingSyncProcessKey(sender, syncId);
|
|
2331
|
+
const obj = this.ingoingSyncProcesses.get(key);
|
|
2332
|
+
if (!obj ||
|
|
2333
|
+
obj.sender !== sender ||
|
|
2334
|
+
!this.isIncomingSyncProcessActive(obj)) {
|
|
1361
2335
|
return true;
|
|
1362
2336
|
}
|
|
1363
|
-
|
|
2337
|
+
let outProcess;
|
|
2338
|
+
try {
|
|
2339
|
+
outProcess = await obj.process(message);
|
|
2340
|
+
}
|
|
2341
|
+
catch (error) {
|
|
2342
|
+
const wasActive = this.isIncomingSyncProcessActive(obj);
|
|
2343
|
+
obj.free(error);
|
|
2344
|
+
if (!wasActive) {
|
|
2345
|
+
return true;
|
|
2346
|
+
}
|
|
2347
|
+
throw error;
|
|
2348
|
+
}
|
|
1364
2349
|
if (outProcess === true) {
|
|
1365
2350
|
return true;
|
|
1366
2351
|
}
|
|
2352
|
+
else if (outProcess === "fallback-to-simple") {
|
|
2353
|
+
try {
|
|
2354
|
+
await obj.fallbackToSimple(new Error("incoming rateless symbol budget exhausted"));
|
|
2355
|
+
}
|
|
2356
|
+
catch {
|
|
2357
|
+
// The bounded rateless process is already complete or aborted. A
|
|
2358
|
+
// later repair round may retry if the fallback request was lost.
|
|
2359
|
+
}
|
|
2360
|
+
return true;
|
|
2361
|
+
}
|
|
1367
2362
|
else if (outProcess === undefined) {
|
|
1368
2363
|
return true; // we don't have enough information, or received information that is redundant
|
|
1369
2364
|
}
|
|
2365
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2366
|
+
return true;
|
|
2367
|
+
}
|
|
1370
2368
|
// we are not done
|
|
1371
2369
|
const sendStartedAt = syncProfileStart(profile);
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
2370
|
+
try {
|
|
2371
|
+
await this.simple.rpc.send(new RequestMoreSymbols({
|
|
2372
|
+
lastSeqNo: obj.lastSeqNo,
|
|
2373
|
+
syncId: message.syncId,
|
|
2374
|
+
}), {
|
|
2375
|
+
mode: new SilentDelivery({ to: [obj.sender], redundancy: 1 }),
|
|
2376
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
2377
|
+
signal: obj.controller.signal,
|
|
2378
|
+
});
|
|
2379
|
+
}
|
|
2380
|
+
catch {
|
|
2381
|
+
if (profile) {
|
|
2382
|
+
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
2383
|
+
name: "rateless.sendRequestMoreSymbols",
|
|
2384
|
+
messages: 1,
|
|
2385
|
+
targets: 1,
|
|
2386
|
+
syncId,
|
|
2387
|
+
details: { rejected: true },
|
|
2388
|
+
});
|
|
2389
|
+
}
|
|
2390
|
+
return true;
|
|
2391
|
+
}
|
|
2392
|
+
if (!this.isIncomingSyncProcessActive(obj)) {
|
|
2393
|
+
return true;
|
|
2394
|
+
}
|
|
1379
2395
|
if (profile) {
|
|
1380
|
-
|
|
1381
|
-
name: "rateless.sendRequestMoreSymbols",
|
|
1382
|
-
messages: 1,
|
|
1383
|
-
targets: 1,
|
|
1384
|
-
syncId,
|
|
1385
|
-
}), () => emitSyncProfileDuration(profile, sendStartedAt, {
|
|
2396
|
+
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
1386
2397
|
name: "rateless.sendRequestMoreSymbols",
|
|
1387
2398
|
messages: 1,
|
|
1388
2399
|
targets: 1,
|
|
1389
2400
|
syncId,
|
|
1390
|
-
|
|
1391
|
-
}));
|
|
2401
|
+
});
|
|
1392
2402
|
}
|
|
1393
2403
|
return true;
|
|
1394
2404
|
}
|
|
@@ -1398,16 +2408,39 @@ export class RatelessIBLTSynchronizer {
|
|
|
1398
2408
|
if (!obj) {
|
|
1399
2409
|
return true;
|
|
1400
2410
|
}
|
|
1401
|
-
|
|
2411
|
+
if (context.from?.hashcode() !== obj.target) {
|
|
2412
|
+
return true;
|
|
2413
|
+
}
|
|
2414
|
+
const signal = obj.signal;
|
|
2415
|
+
if (signal.aborted) {
|
|
2416
|
+
return true;
|
|
2417
|
+
}
|
|
2418
|
+
const next = obj.next(message);
|
|
2419
|
+
if (!next) {
|
|
2420
|
+
return true;
|
|
2421
|
+
}
|
|
2422
|
+
const { symbols, exhaustedAfterSend } = next;
|
|
2423
|
+
if (signal.aborted) {
|
|
2424
|
+
return true;
|
|
2425
|
+
}
|
|
1402
2426
|
const sendStartedAt = syncProfileStart(profile);
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
2427
|
+
try {
|
|
2428
|
+
await this.properties.rpc.send(new MoreSymbols({
|
|
2429
|
+
lastSeqNo: message.lastSeqNo,
|
|
2430
|
+
syncId: message.syncId,
|
|
2431
|
+
symbols,
|
|
2432
|
+
}), {
|
|
2433
|
+
mode: new SilentDelivery({ to: [obj.target], redundancy: 1 }),
|
|
2434
|
+
priority: SYNC_MESSAGE_PRIORITY,
|
|
2435
|
+
signal,
|
|
2436
|
+
});
|
|
2437
|
+
}
|
|
2438
|
+
catch (error) {
|
|
2439
|
+
if (signal?.aborted) {
|
|
2440
|
+
return true;
|
|
2441
|
+
}
|
|
2442
|
+
throw error;
|
|
2443
|
+
}
|
|
1411
2444
|
if (profile) {
|
|
1412
2445
|
emitSyncProfileDuration(profile, sendStartedAt, {
|
|
1413
2446
|
name: "rateless.sendMoreSymbols",
|
|
@@ -1417,6 +2450,13 @@ export class RatelessIBLTSynchronizer {
|
|
|
1417
2450
|
syncId,
|
|
1418
2451
|
});
|
|
1419
2452
|
}
|
|
2453
|
+
if (exhaustedAfterSend) {
|
|
2454
|
+
// Latch exhaustion in next() before the send begins, then retain this
|
|
2455
|
+
// bounded process long enough for RequestAll. Starting Simple eagerly
|
|
2456
|
+
// also keeps fallback working with older receivers that ignore an empty
|
|
2457
|
+
// terminal symbol batch.
|
|
2458
|
+
void obj.startSimpleFallback().catch((error) => logger.error(error));
|
|
2459
|
+
}
|
|
1420
2460
|
return true;
|
|
1421
2461
|
}
|
|
1422
2462
|
else if (message instanceof RequestAll) {
|
|
@@ -1424,12 +2464,134 @@ export class RatelessIBLTSynchronizer {
|
|
|
1424
2464
|
if (!p) {
|
|
1425
2465
|
return true;
|
|
1426
2466
|
}
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
2467
|
+
if (context.from?.hashcode() !== p.target) {
|
|
2468
|
+
return true;
|
|
2469
|
+
}
|
|
2470
|
+
if (p.signal.aborted) {
|
|
2471
|
+
return true;
|
|
2472
|
+
}
|
|
2473
|
+
// RequestAll ends only this target's rateless attempt. Other target
|
|
2474
|
+
// encoders and response authorizations remain independently owned.
|
|
2475
|
+
const fallback = p.startSimpleFallback();
|
|
2476
|
+
p.free();
|
|
2477
|
+
await fallback;
|
|
1431
2478
|
return true;
|
|
1432
2479
|
}
|
|
2480
|
+
else if (message instanceof ResponseMaybeSync ||
|
|
2481
|
+
message instanceof ResponseMaybeSyncCapabilities) {
|
|
2482
|
+
const from = context.from;
|
|
2483
|
+
// Simple authorizations are exact request leases and take precedence
|
|
2484
|
+
// over Rateless' broader process authorization. This also handles a
|
|
2485
|
+
// delayed fallback/prelude response after the Rateless process for the
|
|
2486
|
+
// same target has been replaced.
|
|
2487
|
+
const simpleLeases = this.simple.consumeAuthorizedMaybeSyncResponse(message.hashes, from);
|
|
2488
|
+
const simpleHashes = simpleLeases.flatMap((lease) => lease.hashes);
|
|
2489
|
+
const simpleHashSet = new Set(simpleHashes);
|
|
2490
|
+
const ratelessCandidateHashes = [];
|
|
2491
|
+
let inspected = 0;
|
|
2492
|
+
const iterator = message.hashes[Symbol.iterator]();
|
|
2493
|
+
let exhausted = false;
|
|
2494
|
+
try {
|
|
2495
|
+
while (inspected < MAX_RATELESS_RESPONSE_HASHES_INSPECTED) {
|
|
2496
|
+
const next = iterator.next();
|
|
2497
|
+
if (next.done) {
|
|
2498
|
+
exhausted = true;
|
|
2499
|
+
break;
|
|
2500
|
+
}
|
|
2501
|
+
inspected += 1;
|
|
2502
|
+
if (!simpleHashSet.has(next.value)) {
|
|
2503
|
+
ratelessCandidateHashes.push(next.value);
|
|
2504
|
+
}
|
|
2505
|
+
}
|
|
2506
|
+
}
|
|
2507
|
+
finally {
|
|
2508
|
+
if (!exhausted) {
|
|
2509
|
+
iterator.return?.();
|
|
2510
|
+
}
|
|
2511
|
+
}
|
|
2512
|
+
const response = ratelessCandidateHashes.length > 0
|
|
2513
|
+
? this.consumeAuthorizedRatelessResponse(ratelessCandidateHashes, from)
|
|
2514
|
+
: undefined;
|
|
2515
|
+
if (simpleLeases.length === 0 &&
|
|
2516
|
+
(!response || response.authorized.length === 0)) {
|
|
2517
|
+
response?.release();
|
|
2518
|
+
return true;
|
|
2519
|
+
}
|
|
2520
|
+
let firstError;
|
|
2521
|
+
let rollbackRatelessAuthorization = false;
|
|
2522
|
+
try {
|
|
2523
|
+
const simpleMessage = message instanceof ResponseMaybeSyncCapabilities
|
|
2524
|
+
? new ResponseMaybeSyncCapabilities({
|
|
2525
|
+
hashes: simpleHashes,
|
|
2526
|
+
capabilities: message.capabilities,
|
|
2527
|
+
})
|
|
2528
|
+
: new ResponseMaybeSync({ hashes: simpleHashes });
|
|
2529
|
+
if (response && response.authorized.length > 0) {
|
|
2530
|
+
const responseStartedAt = syncProfileStart(profile);
|
|
2531
|
+
let responseShipment = {
|
|
2532
|
+
messages: 0,
|
|
2533
|
+
fused: false,
|
|
2534
|
+
entries: 0,
|
|
2535
|
+
};
|
|
2536
|
+
try {
|
|
2537
|
+
responseShipment =
|
|
2538
|
+
await this.simple.shipAuthorizedMaybeSyncResponse({
|
|
2539
|
+
hashes: response.authorized,
|
|
2540
|
+
from,
|
|
2541
|
+
response: message,
|
|
2542
|
+
signal: response.signal,
|
|
2543
|
+
});
|
|
2544
|
+
}
|
|
2545
|
+
catch (error) {
|
|
2546
|
+
firstError = error;
|
|
2547
|
+
rollbackRatelessAuthorization = true;
|
|
2548
|
+
}
|
|
2549
|
+
if (profile) {
|
|
2550
|
+
try {
|
|
2551
|
+
emitSyncProfileDuration(profile, responseStartedAt, {
|
|
2552
|
+
name: "simple.exchangeHeads",
|
|
2553
|
+
entries: responseShipment.entries,
|
|
2554
|
+
messages: responseShipment.messages,
|
|
2555
|
+
targets: 1,
|
|
2556
|
+
details: {
|
|
2557
|
+
source: "ratelessResponseMaybeSync",
|
|
2558
|
+
fused: responseShipment.fused,
|
|
2559
|
+
},
|
|
2560
|
+
});
|
|
2561
|
+
}
|
|
2562
|
+
catch (error) {
|
|
2563
|
+
firstError ??= error;
|
|
2564
|
+
}
|
|
2565
|
+
}
|
|
2566
|
+
}
|
|
2567
|
+
if (simpleLeases.length > 0) {
|
|
2568
|
+
try {
|
|
2569
|
+
await this.simple.shipAuthorizedMaybeSyncResponseLeases({
|
|
2570
|
+
leases: simpleLeases,
|
|
2571
|
+
from,
|
|
2572
|
+
response: simpleMessage,
|
|
2573
|
+
});
|
|
2574
|
+
}
|
|
2575
|
+
catch (error) {
|
|
2576
|
+
firstError ??= error;
|
|
2577
|
+
}
|
|
2578
|
+
}
|
|
2579
|
+
if (firstError !== undefined) {
|
|
2580
|
+
throw firstError;
|
|
2581
|
+
}
|
|
2582
|
+
return true;
|
|
2583
|
+
}
|
|
2584
|
+
finally {
|
|
2585
|
+
response?.release({ rollback: rollbackRatelessAuthorization });
|
|
2586
|
+
// The Simple helper owns these leases once entered and releases each
|
|
2587
|
+
// one in its own finally. Keep this outer ownership release as the
|
|
2588
|
+
// final safety net for diagnostics or setup failures that happen
|
|
2589
|
+
// before the helper can take responsibility.
|
|
2590
|
+
for (const lease of simpleLeases) {
|
|
2591
|
+
lease.release();
|
|
2592
|
+
}
|
|
2593
|
+
}
|
|
2594
|
+
}
|
|
1433
2595
|
return this.simple.onMessage(message, context);
|
|
1434
2596
|
}
|
|
1435
2597
|
onReceivedEntries(properties) {
|
|
@@ -1459,16 +2621,42 @@ export class RatelessIBLTSynchronizer {
|
|
|
1459
2621
|
return this.simple.onEntryRemovedHashes(hashes);
|
|
1460
2622
|
}
|
|
1461
2623
|
onPeerDisconnected(key) {
|
|
1462
|
-
|
|
2624
|
+
const target = typeof key === "string" ? key : key.hashcode();
|
|
2625
|
+
for (const process of [...this.ingoingSyncProcesses.values()]) {
|
|
2626
|
+
if (process.sender === target) {
|
|
2627
|
+
process.free(new Error("incoming rateless peer disconnected"));
|
|
2628
|
+
}
|
|
2629
|
+
}
|
|
2630
|
+
for (const targetLifecycle of [
|
|
2631
|
+
...(this.ratelessDispatchTargets.get(target) ?? []),
|
|
2632
|
+
]) {
|
|
2633
|
+
this.abortRatelessDispatchTarget(targetLifecycle, new Error("rateless sync target disconnected"));
|
|
2634
|
+
}
|
|
2635
|
+
return this.simple.onPeerDisconnected(target);
|
|
1463
2636
|
}
|
|
1464
2637
|
open() {
|
|
2638
|
+
const reason = new Error("rateless sync generation replaced");
|
|
2639
|
+
this.cancelRatelessRepairSessions(reason);
|
|
2640
|
+
this.ratelessDispatchLifecycleController.abort(reason);
|
|
2641
|
+
this.ratelessDispatchLifecycleController = new AbortController();
|
|
2642
|
+
this.startedOrCompletedSynchronizations = new Cache({ max: 1e4 });
|
|
2643
|
+
// Admission accounting intentionally spans local generations: native range
|
|
2644
|
+
// initialization and fallback delivery are not guaranteed to stop merely
|
|
2645
|
+
// because their logical process was aborted.
|
|
2646
|
+
this.ratelessClosed = false;
|
|
1465
2647
|
return this.simple.open();
|
|
1466
2648
|
}
|
|
1467
2649
|
close() {
|
|
1468
|
-
|
|
2650
|
+
this.ratelessClosed = true;
|
|
2651
|
+
// Abort ownership first. Process abort listeners then cancel any in-flight
|
|
2652
|
+
// StartSync/MoreSymbols send before they detach or free native state.
|
|
2653
|
+
const reason = new Error("rateless synchronizer closed");
|
|
2654
|
+
this.cancelRatelessRepairSessions(reason);
|
|
2655
|
+
this.ratelessDispatchLifecycleController.abort(reason);
|
|
2656
|
+
for (const obj of [...this.ingoingSyncProcesses.values()]) {
|
|
1469
2657
|
obj.free();
|
|
1470
2658
|
}
|
|
1471
|
-
for (const
|
|
2659
|
+
for (const obj of [...this.outgoingSyncProcesses.values()]) {
|
|
1472
2660
|
obj.free();
|
|
1473
2661
|
}
|
|
1474
2662
|
this.clearLocalRangeEncoderCache();
|