@peerbit/shared-log 13.2.12 → 13.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -43,7 +43,7 @@ import {} from "../exchange-heads.js";
43
43
  import { TransportMessage } from "../message.js";
44
44
  import {} from "../ranges.js";
45
45
  import { emitSyncProfileDuration, emitSyncProfileEvent, syncProfileStart, } from "./profile.js";
46
- import { SYNC_MESSAGE_PRIORITY, SimpleSyncronizer, } from "./simple.js";
46
+ import { ResponseMaybeSync, ResponseMaybeSyncCapabilities, SYNC_MESSAGE_PRIORITY, SimpleSyncronizer, } from "./simple.js";
47
47
  export const logger = loggerFn("peerbit:shared-log:rateless");
48
48
  const coerceBigInt = (value) => typeof value === "bigint" ? value : BigInt(value);
49
49
  let SymbolSerialized = (() => {
@@ -81,6 +81,7 @@ let SymbolSerialized = (() => {
81
81
  const CODED_SYMBOL_WORDS = 3;
82
82
  const CODED_SYMBOL_WORD_BYTES = 8;
83
83
  const CODED_SYMBOL_BYTES = CODED_SYMBOL_WORDS * CODED_SYMBOL_WORD_BYTES;
84
+ const MAX_CODED_SYMBOL_BATCH_SIZE = 1_024;
84
85
  const BIG_UINT64_ARRAY_IS_LITTLE_ENDIAN = typeof BigUint64Array !== "undefined" &&
85
86
  new Uint8Array(new BigUint64Array([1n]).buffer)[0] === 1;
86
87
  const assertValidFlatCodedSymbols = (flat) => {
@@ -191,6 +192,9 @@ const codedSymbolBatchField = {
191
192
  },
192
193
  deserialize: (reader) => {
193
194
  const length = reader.u32();
195
+ if (length > MAX_CODED_SYMBOL_BATCH_SIZE) {
196
+ throw new Error("RIBLT coded symbol batch exceeds the receiver limit");
197
+ }
194
198
  const wordLength = length * CODED_SYMBOL_WORDS;
195
199
  const byteLength = length * CODED_SYMBOL_BYTES;
196
200
  if (typeof BigUint64Array !== "undefined" &&
@@ -356,7 +360,23 @@ const DEFAULT_CONVERGENT_REPAIR_TIMEOUT_MS = 30_000;
356
360
  const DEFAULT_CONVERGENT_RETRY_INTERVALS_MS = [0, 1_000, 3_000, 7_000];
357
361
  const DEFAULT_MAX_CONVERGENT_TRACKED_HASHES = 4_096;
358
362
  const MIN_MORE_SYMBOLS_BATCH_SIZE = 64;
359
- const MAX_MORE_SYMBOLS_BATCH_SIZE = 1_024;
363
+ const MAX_MORE_SYMBOLS_BATCH_SIZE = MAX_CODED_SYMBOL_BATCH_SIZE;
364
+ const MAX_INCOMING_RATELESS_PROCESSES = 32;
365
+ const MAX_INCOMING_RATELESS_PROCESSES_PER_SENDER = 4;
366
+ const MAX_INCOMING_RATELESS_QUEUED_BATCHES = 8;
367
+ const MAX_INCOMING_RATELESS_SEQUENCE_GAP = BigInt(MAX_INCOMING_RATELESS_QUEUED_BATCHES);
368
+ const MIN_RATELESS_SYMBOL_BUDGET = 4_096;
369
+ const MAX_RATELESS_SYMBOL_BUDGET = 262_144;
370
+ const RATELESS_SYMBOL_BUDGET_MULTIPLIER = 4;
371
+ const INCOMING_RATELESS_IDLE_TIMEOUT_MS = 20_000;
372
+ const OUTGOING_RATELESS_IDLE_TIMEOUT_MS = 10_000;
373
+ const RATELESS_PROCESS_ABSOLUTE_TIMEOUT_MS = 120_000;
374
+ const INCOMING_RATELESS_FALLBACK_GRACE_MS = 5_000;
375
+ const MAX_RATELESS_RESPONSE_HASHES_INSPECTED = 10_000;
376
+ export const MAX_ACTIVE_RATELESS_RESPONSES_PER_PEER = 4;
377
+ export const MAX_ACTIVE_RATELESS_RESPONSES_GLOBAL = 32;
378
+ const getIncomingRatelessSymbolBudget = (initialSymbols) => Math.min(MAX_RATELESS_SYMBOL_BUDGET, Math.max(MIN_RATELESS_SYMBOL_BUDGET, initialSymbols * initialSymbols * RATELESS_SYMBOL_BUDGET_MULTIPLIER));
379
+ const getOutgoingRatelessSymbolBudget = (entries, initialSymbols) => Math.min(MAX_RATELESS_SYMBOL_BUDGET, Math.max(MIN_RATELESS_SYMBOL_BUDGET, initialSymbols, entries * RATELESS_SYMBOL_BUDGET_MULTIPLIER));
360
380
  let StartSync = (() => {
361
381
  let _classDecorators = [variant([3, 0])];
362
382
  let _classDescriptor;
@@ -556,97 +576,122 @@ const matchEntriesByHashNumberInRangeQuery = (range) => {
556
576
  const buildEncoderOrDecoderFromRange = async (ranges, entryIndex, type, profile, resolveHashNumbersInRange) => {
557
577
  await ribltReady;
558
578
  const encoder = type === "encoder" ? new EncoderWrapper() : new DecoderWrapper();
559
- const rangeQueryStartedAt = syncProfileStart(profile);
560
- let hashNumbers;
561
- let source = "index";
562
- const resolved = await resolveHashNumbersInRange?.({
563
- end1: ranges.end1,
564
- start1: ranges.start1,
565
- end2: ranges.end2,
566
- start2: ranges.start2,
567
- });
568
- if (resolved) {
569
- source = "native";
570
- hashNumbers =
571
- typeof BigUint64Array !== "undefined" &&
572
- resolved instanceof BigUint64Array
573
- ? resolved
574
- : [...resolved];
575
- }
576
- else {
577
- const entries = await entryIndex
578
- .iterate({
579
- // Range sync for IBLT is done in hashNumber space.
580
- query: matchEntriesByHashNumberInRangeQuery({
581
- end1: ranges.end1,
582
- start1: ranges.start1,
583
- end2: ranges.end2,
584
- start2: ranges.start2,
585
- }),
586
- }, {
587
- shape: {
588
- hash: true,
589
- hashNumber: true,
590
- },
591
- })
592
- .all();
593
- hashNumbers = entries.map((entry) => entry.value.hashNumber);
594
- }
595
- if (profile) {
596
- emitSyncProfileDuration(profile, rangeQueryStartedAt, {
597
- name: "rateless.rangeQuery",
598
- entries: hashNumbers.length,
599
- details: { type, source },
579
+ let transferred = false;
580
+ try {
581
+ const rangeQueryStartedAt = syncProfileStart(profile);
582
+ let hashNumbers;
583
+ let source = "index";
584
+ const resolved = await resolveHashNumbersInRange?.({
585
+ end1: ranges.end1,
586
+ start1: ranges.start1,
587
+ end2: ranges.end2,
588
+ start2: ranges.start2,
600
589
  });
590
+ if (resolved) {
591
+ source = "native";
592
+ hashNumbers =
593
+ typeof BigUint64Array !== "undefined" &&
594
+ resolved instanceof BigUint64Array
595
+ ? resolved
596
+ : [...resolved];
597
+ }
598
+ else {
599
+ const entries = await entryIndex
600
+ .iterate({
601
+ // Range sync for IBLT is done in hashNumber space.
602
+ query: matchEntriesByHashNumberInRangeQuery({
603
+ end1: ranges.end1,
604
+ start1: ranges.start1,
605
+ end2: ranges.end2,
606
+ start2: ranges.start2,
607
+ }),
608
+ }, {
609
+ shape: {
610
+ hash: true,
611
+ hashNumber: true,
612
+ },
613
+ })
614
+ .all();
615
+ hashNumbers = entries.map((entry) => entry.value.hashNumber);
616
+ }
617
+ if (profile) {
618
+ emitSyncProfileDuration(profile, rangeQueryStartedAt, {
619
+ name: "rateless.rangeQuery",
620
+ entries: hashNumbers.length,
621
+ details: { type, source },
622
+ });
623
+ }
624
+ if (hashNumbers.length === 0) {
625
+ return false;
626
+ }
627
+ const addSymbolsStartedAt = syncProfileStart(profile);
628
+ if (typeof BigUint64Array !== "undefined" &&
629
+ typeof encoder.add_symbols === "function") {
630
+ const symbols = hashNumbers instanceof BigUint64Array
631
+ ? hashNumbers
632
+ : BigUint64Array.from(hashNumbers, coerceBigInt);
633
+ addSymbolsToRiblt(encoder, symbols);
634
+ }
635
+ else {
636
+ for (const hashNumber of hashNumbers) {
637
+ encoder.add_symbol(coerceBigInt(hashNumber));
638
+ }
639
+ }
640
+ if (profile) {
641
+ emitSyncProfileDuration(profile, addSymbolsStartedAt, {
642
+ name: "rateless.rangeAddSymbols",
643
+ entries: hashNumbers.length,
644
+ symbols: hashNumbers.length,
645
+ details: { type },
646
+ });
647
+ }
648
+ transferred = true;
649
+ return encoder;
601
650
  }
602
- if (hashNumbers.length === 0) {
603
- return false;
604
- }
605
- const addSymbolsStartedAt = syncProfileStart(profile);
606
- if (typeof BigUint64Array !== "undefined" &&
607
- typeof encoder.add_symbols === "function") {
608
- const symbols = hashNumbers instanceof BigUint64Array
609
- ? hashNumbers
610
- : BigUint64Array.from(hashNumbers, coerceBigInt);
611
- addSymbolsToRiblt(encoder, symbols);
612
- }
613
- else {
614
- for (const hashNumber of hashNumbers) {
615
- encoder.add_symbol(coerceBigInt(hashNumber));
616
- }
617
- }
618
- if (profile) {
619
- emitSyncProfileDuration(profile, addSymbolsStartedAt, {
620
- name: "rateless.rangeAddSymbols",
621
- entries: hashNumbers.length,
622
- symbols: hashNumbers.length,
623
- details: { type },
624
- });
651
+ finally {
652
+ if (!transferred) {
653
+ encoder.free();
654
+ }
625
655
  }
626
- return encoder;
627
656
  };
628
657
  export class RatelessIBLTSynchronizer {
629
658
  properties;
630
659
  simple;
631
660
  repairSessionCounter;
661
+ ratelessRepairSessions;
632
662
  startedOrCompletedSynchronizations;
633
663
  localRangeEncoderCacheVersion = 0;
634
664
  localRangeEncoderCache = new Map();
635
665
  localRangeEncoderCacheMax = 2;
636
666
  ingoingSyncProcesses;
667
+ incomingRatelessProcessAdmissions;
637
668
  outgoingSyncProcesses;
669
+ outgoingSyncProcessByTarget;
670
+ ratelessDispatchLifecycleController;
671
+ ratelessDispatchTargets;
672
+ activeRatelessResponseCount;
673
+ activeRatelessResponseCountByPeer;
674
+ ratelessClosed;
638
675
  constructor(properties) {
639
676
  this.properties = properties;
640
677
  this.simple = new SimpleSyncronizer(properties);
641
678
  this.repairSessionCounter = 0;
679
+ this.ratelessRepairSessions = new Map();
642
680
  this.outgoingSyncProcesses = new Map();
681
+ this.outgoingSyncProcessByTarget = new Map();
682
+ this.ratelessDispatchLifecycleController = new AbortController();
683
+ this.ratelessDispatchTargets = new Map();
684
+ this.activeRatelessResponseCount = 0;
685
+ this.activeRatelessResponseCountByPeer = new Map();
686
+ this.ratelessClosed = false;
643
687
  this.ingoingSyncProcesses = new Map();
688
+ this.incomingRatelessProcessAdmissions = new Set();
644
689
  this.startedOrCompletedSynchronizations = new Cache({ max: 1e4 });
645
690
  }
646
691
  get maxConvergentTrackedHashes() {
647
692
  const value = this.properties.sync?.maxConvergentTrackedHashes;
648
693
  return value && Number.isFinite(value) && value > 0
649
- ? Math.floor(value)
694
+ ? Math.max(1, Math.floor(value))
650
695
  : DEFAULT_MAX_CONVERGENT_TRACKED_HASHES;
651
696
  }
652
697
  normalizeRetryIntervals(retryIntervalsMs) {
@@ -677,6 +722,94 @@ export class RatelessIBLTSynchronizer {
677
722
  scored.sort((a, b) => b.priority - a.priority || a.index - b.index);
678
723
  return scored.map((x) => x.entry);
679
724
  }
725
+ isRatelessRepairSessionActive(session) {
726
+ return (!this.ratelessClosed &&
727
+ !session.cancelled &&
728
+ !session.controller.signal.aborted &&
729
+ !session.ownershipLifecycleController.signal.aborted &&
730
+ session.ownershipLifecycleController ===
731
+ this.ratelessDispatchLifecycleController &&
732
+ this.ratelessRepairSessions.get(session.id) === session);
733
+ }
734
+ cancelRatelessRepairSession(session, reason = new Error("rateless repair session cancelled")) {
735
+ if (session.cancelled) {
736
+ return;
737
+ }
738
+ session.cancelled = true;
739
+ if (session.deadlineTimer !== undefined) {
740
+ clearTimeout(session.deadlineTimer);
741
+ session.deadlineTimer = undefined;
742
+ }
743
+ session.controller.abort(reason);
744
+ session.trackedSession.cancel();
745
+ }
746
+ disposeRatelessRepairSession(session) {
747
+ if (session.settled) {
748
+ return;
749
+ }
750
+ session.settled = true;
751
+ if (session.deadlineTimer !== undefined) {
752
+ clearTimeout(session.deadlineTimer);
753
+ session.deadlineTimer = undefined;
754
+ }
755
+ session.ownershipLifecycleController.signal.removeEventListener("abort", session.onOwnershipAbort);
756
+ if (this.ratelessRepairSessions.get(session.id) === session) {
757
+ this.ratelessRepairSessions.delete(session.id);
758
+ }
759
+ }
760
+ cancelRatelessRepairSessions(reason) {
761
+ for (const session of [...this.ratelessRepairSessions.values()]) {
762
+ this.cancelRatelessRepairSession(session, reason);
763
+ }
764
+ }
765
+ waitForRatelessRepairRetry(delayMs, signal) {
766
+ if (delayMs <= 0) {
767
+ return Promise.resolve(!signal.aborted);
768
+ }
769
+ if (signal.aborted) {
770
+ return Promise.resolve(false);
771
+ }
772
+ return new Promise((resolve) => {
773
+ let settled = false;
774
+ let timer;
775
+ const settle = (elapsed) => {
776
+ if (settled) {
777
+ return;
778
+ }
779
+ settled = true;
780
+ clearTimeout(timer);
781
+ signal.removeEventListener("abort", onAbort);
782
+ resolve(elapsed);
783
+ };
784
+ const onAbort = () => settle(false);
785
+ timer = setTimeout(() => settle(true), delayMs);
786
+ timer.unref?.();
787
+ signal.addEventListener("abort", onAbort, { once: true });
788
+ if (signal.aborted) {
789
+ onAbort();
790
+ }
791
+ });
792
+ }
793
+ waitForRatelessRepairDispatch(dispatch, signal) {
794
+ const dispatchPromise = Promise.resolve(dispatch);
795
+ return new Promise((resolve) => {
796
+ let settled = false;
797
+ const settle = (completed) => {
798
+ if (settled) {
799
+ return;
800
+ }
801
+ settled = true;
802
+ signal.removeEventListener("abort", onAbort);
803
+ resolve(completed);
804
+ };
805
+ const onAbort = () => settle(false);
806
+ signal.addEventListener("abort", onAbort, { once: true });
807
+ void dispatchPromise.then(() => settle(true), () => settle(true));
808
+ if (signal.aborted) {
809
+ onAbort();
810
+ }
811
+ });
812
+ }
680
813
  startRepairSession(properties) {
681
814
  const mode = properties.mode ?? "best-effort";
682
815
  const targets = [...new Set(properties.targets)];
@@ -702,7 +835,6 @@ export class RatelessIBLTSynchronizer {
702
835
  for (const entry of prioritized.slice(0, trackedLimit)) {
703
836
  trackedEntries.set(entry.hash, entry);
704
837
  }
705
- let cancelled = false;
706
838
  const trackedSession = this.simple.startRepairSession({
707
839
  entries: trackedEntries,
708
840
  targets,
@@ -710,32 +842,58 @@ export class RatelessIBLTSynchronizer {
710
842
  timeoutMs,
711
843
  retryIntervalsMs,
712
844
  });
845
+ const ownershipLifecycleController = this.ratelessDispatchLifecycleController;
846
+ let session;
847
+ const cancel = (reason = new Error("rateless repair session cancelled")) => this.cancelRatelessRepairSession(session, reason);
848
+ session = {
849
+ id,
850
+ ownershipLifecycleController,
851
+ controller: new AbortController(),
852
+ trackedSession,
853
+ onOwnershipAbort: () => cancel(ownershipLifecycleController.signal.reason ??
854
+ new Error("rateless repair generation ended")),
855
+ cancelled: false,
856
+ settled: false,
857
+ deadlineTimer: undefined,
858
+ };
859
+ this.ratelessRepairSessions.set(id, session);
860
+ session.deadlineTimer = setTimeout(() => cancel(new Error("rateless convergent repair session timed out")), Math.max(0, timeoutMs - (Date.now() - startedAt)));
861
+ session.deadlineTimer.unref?.();
862
+ ownershipLifecycleController.signal.addEventListener("abort", session.onOwnershipAbort, { once: true });
863
+ if (this.ratelessClosed ||
864
+ ownershipLifecycleController.signal.aborted ||
865
+ ownershipLifecycleController !==
866
+ this.ratelessDispatchLifecycleController) {
867
+ cancel(ownershipLifecycleController.signal.reason ??
868
+ new Error("rateless repair generation is not active"));
869
+ }
713
870
  const runDispatchSchedule = async () => {
714
871
  let previousDelay = 0;
715
872
  for (const delayMs of retryIntervalsMs) {
716
- if (cancelled) {
873
+ if (!this.isRatelessRepairSessionActive(session)) {
717
874
  return;
718
875
  }
719
876
  const elapsed = Date.now() - startedAt;
720
877
  if (elapsed >= timeoutMs) {
721
878
  return;
722
879
  }
723
- const waitMs = Math.max(0, delayMs - previousDelay);
880
+ const waitMs = Math.min(Math.max(0, delayMs - previousDelay), timeoutMs - elapsed);
724
881
  previousDelay = delayMs;
725
- if (waitMs > 0) {
726
- await new Promise((resolve) => {
727
- const timer = setTimeout(resolve, waitMs);
728
- timer.unref?.();
729
- });
882
+ if (!(await this.waitForRatelessRepairRetry(waitMs, session.controller.signal))) {
883
+ return;
730
884
  }
731
- if (cancelled) {
885
+ if (!this.isRatelessRepairSessionActive(session) ||
886
+ Date.now() - startedAt >= timeoutMs) {
732
887
  return;
733
888
  }
734
889
  try {
735
- await this.onMaybeMissingEntries({
890
+ if (!(await this.waitForRatelessRepairDispatch(this.onMaybeMissingEntries({
736
891
  entries: properties.entries,
737
892
  targets,
738
- });
893
+ signal: session.controller.signal,
894
+ }), session.controller.signal))) {
895
+ return;
896
+ }
739
897
  }
740
898
  catch {
741
899
  // Best-effort schedule: tracked session timeout/result decides completion.
@@ -743,21 +901,23 @@ export class RatelessIBLTSynchronizer {
743
901
  }
744
902
  };
745
903
  const done = (async () => {
746
- await runDispatchSchedule();
747
- const trackedResults = await trackedSession.done;
748
- return trackedResults.map((result) => ({
749
- ...result,
750
- requestedTotal: requestedHashes.length,
751
- truncated: true,
752
- }));
904
+ try {
905
+ await runDispatchSchedule();
906
+ const trackedResults = await trackedSession.done;
907
+ return trackedResults.map((result) => ({
908
+ ...result,
909
+ requestedTotal: requestedHashes.length,
910
+ truncated: true,
911
+ }));
912
+ }
913
+ finally {
914
+ this.disposeRatelessRepairSession(session);
915
+ }
753
916
  })();
754
917
  return {
755
918
  id,
756
919
  done,
757
- cancel: () => {
758
- cancelled = true;
759
- trackedSession.cancel();
760
- },
920
+ cancel: () => cancel(),
761
921
  };
762
922
  }
763
923
  const id = `rateless-repair-${++this.repairSessionCounter}`;
@@ -801,32 +961,60 @@ export class RatelessIBLTSynchronizer {
801
961
  localRangeEncoderCacheKey(ranges) {
802
962
  return `${String(ranges.start1)}:${String(ranges.end1)}:${String(ranges.start2)}:${String(ranges.end2)}`;
803
963
  }
804
- decoderFromCachedEncoder(encoder) {
964
+ decoderFromCachedEncoder(encoder, beforeDecoderTransfer) {
805
965
  const clone = encoder.clone();
806
- const decoder = clone.to_decoder();
807
- clone.free();
808
- return decoder;
966
+ let cloneFreed = false;
967
+ let decoder;
968
+ let decoderTransferred = false;
969
+ const freeClone = () => {
970
+ if (cloneFreed) {
971
+ return;
972
+ }
973
+ cloneFreed = true;
974
+ clone.free();
975
+ };
976
+ try {
977
+ decoder = clone.to_decoder();
978
+ beforeDecoderTransfer?.();
979
+ freeClone();
980
+ decoderTransferred = true;
981
+ return decoder;
982
+ }
983
+ finally {
984
+ try {
985
+ freeClone();
986
+ }
987
+ finally {
988
+ if (!decoderTransferred) {
989
+ decoder?.free();
990
+ }
991
+ }
992
+ }
809
993
  }
810
- async getLocalDecoderForRange(ranges) {
994
+ async getLocalDecoderForRange(ranges, options) {
995
+ const isActive = () => options?.signal?.aborted !== true &&
996
+ (options?.ownershipLifecycleController === undefined ||
997
+ this.isIncomingSyncGenerationActive(options.ownershipLifecycleController));
998
+ if (!isActive()) {
999
+ return false;
1000
+ }
811
1001
  const profile = this.properties.sync?.profile;
812
1002
  const key = this.localRangeEncoderCacheKey(ranges);
813
1003
  const cached = this.localRangeEncoderCache.get(key);
814
1004
  if (cached && cached.version === this.localRangeEncoderCacheVersion) {
815
1005
  const startedAt = syncProfileStart(profile);
816
1006
  cached.lastUsed = Date.now();
817
- try {
818
- return this.decoderFromCachedEncoder(cached.encoder);
819
- }
820
- finally {
1007
+ return this.decoderFromCachedEncoder(cached.encoder, () => {
821
1008
  if (profile) {
822
1009
  emitSyncProfileDuration(profile, startedAt, {
823
1010
  name: "rateless.localDecoder",
824
1011
  cacheHit: true,
825
1012
  });
826
1013
  }
827
- }
1014
+ });
828
1015
  }
829
1016
  const startedAt = syncProfileStart(profile);
1017
+ const cacheVersion = this.localRangeEncoderCacheVersion;
830
1018
  const encoder = (await buildEncoderOrDecoderFromRange(ranges, this.properties.entryIndex, "encoder", profile, this.properties.resolveHashNumbersInRange));
831
1019
  if (!encoder) {
832
1020
  if (profile) {
@@ -838,6 +1026,26 @@ export class RatelessIBLTSynchronizer {
838
1026
  }
839
1027
  return false;
840
1028
  }
1029
+ if (!isActive()) {
1030
+ encoder.free();
1031
+ return false;
1032
+ }
1033
+ if (cacheVersion !== this.localRangeEncoderCacheVersion) {
1034
+ try {
1035
+ return this.decoderFromCachedEncoder(encoder, () => {
1036
+ if (profile) {
1037
+ emitSyncProfileDuration(profile, startedAt, {
1038
+ name: "rateless.localDecoder",
1039
+ cacheHit: false,
1040
+ details: { cacheInvalidated: true },
1041
+ });
1042
+ }
1043
+ });
1044
+ }
1045
+ finally {
1046
+ encoder.free();
1047
+ }
1048
+ }
841
1049
  const now = Date.now();
842
1050
  const existing = this.localRangeEncoderCache.get(key);
843
1051
  if (existing) {
@@ -866,21 +1074,168 @@ export class RatelessIBLTSynchronizer {
866
1074
  }
867
1075
  this.localRangeEncoderCache.delete(oldestKey);
868
1076
  }
869
- try {
870
- return this.decoderFromCachedEncoder(encoder);
871
- }
872
- finally {
1077
+ return this.decoderFromCachedEncoder(encoder, () => {
873
1078
  if (profile) {
874
1079
  emitSyncProfileDuration(profile, startedAt, {
875
1080
  name: "rateless.localDecoder",
876
1081
  cacheHit: false,
877
1082
  });
878
1083
  }
1084
+ });
1085
+ }
1086
+ captureRatelessDispatchLifecycle(targets, callerSignal) {
1087
+ const ownershipLifecycleController = this.ratelessDispatchLifecycleController;
1088
+ const lifecycle = {
1089
+ ownershipLifecycleController,
1090
+ callerSignal,
1091
+ controller: new AbortController(),
1092
+ targets: new Map(),
1093
+ onOwnerOrCallerAbort: () => {
1094
+ const reason = callerSignal?.aborted === true
1095
+ ? callerSignal.reason
1096
+ : ownershipLifecycleController.signal.reason;
1097
+ this.abortRatelessDispatchLifecycle(lifecycle, reason);
1098
+ },
1099
+ dispatchFinished: false,
1100
+ disposed: false,
1101
+ };
1102
+ for (const target of [...new Set(targets)]) {
1103
+ const targetLifecycle = {
1104
+ lifecycle,
1105
+ target,
1106
+ controller: new AbortController(),
1107
+ retainedByProcess: false,
1108
+ responseLeases: 0,
1109
+ };
1110
+ lifecycle.targets.set(target, targetLifecycle);
1111
+ let activeForTarget = this.ratelessDispatchTargets.get(target);
1112
+ if (!activeForTarget) {
1113
+ activeForTarget = new Set();
1114
+ this.ratelessDispatchTargets.set(target, activeForTarget);
1115
+ }
1116
+ activeForTarget.add(targetLifecycle);
1117
+ }
1118
+ ownershipLifecycleController.signal.addEventListener("abort", lifecycle.onOwnerOrCallerAbort, { once: true });
1119
+ if (callerSignal && callerSignal !== ownershipLifecycleController.signal) {
1120
+ callerSignal.addEventListener("abort", lifecycle.onOwnerOrCallerAbort, {
1121
+ once: true,
1122
+ });
1123
+ }
1124
+ if (this.ratelessClosed ||
1125
+ ownershipLifecycleController !==
1126
+ this.ratelessDispatchLifecycleController ||
1127
+ ownershipLifecycleController.signal.aborted ||
1128
+ callerSignal?.aborted) {
1129
+ lifecycle.onOwnerOrCallerAbort();
1130
+ }
1131
+ return lifecycle;
1132
+ }
1133
+ abortRatelessDispatchTarget(targetLifecycle, reason) {
1134
+ if (!targetLifecycle.controller.signal.aborted) {
1135
+ targetLifecycle.controller.abort(reason);
1136
+ }
1137
+ this.maybeDisposeRatelessDispatchLifecycle(targetLifecycle.lifecycle);
1138
+ }
1139
+ abortRatelessDispatchLifecycle(lifecycle, reason) {
1140
+ if (!lifecycle.controller.signal.aborted) {
1141
+ lifecycle.controller.abort(reason);
1142
+ }
1143
+ for (const targetLifecycle of lifecycle.targets.values()) {
1144
+ this.abortRatelessDispatchTarget(targetLifecycle, reason);
1145
+ }
1146
+ this.maybeDisposeRatelessDispatchLifecycle(lifecycle);
1147
+ }
1148
+ finishRatelessDispatchLifecycle(lifecycle) {
1149
+ lifecycle.dispatchFinished = true;
1150
+ this.maybeDisposeRatelessDispatchLifecycle(lifecycle);
1151
+ }
1152
+ maybeDisposeRatelessDispatchLifecycle(lifecycle) {
1153
+ if (lifecycle.disposed ||
1154
+ !lifecycle.dispatchFinished ||
1155
+ [...lifecycle.targets.values()].some((target) => target.retainedByProcess || target.responseLeases > 0)) {
1156
+ return;
1157
+ }
1158
+ lifecycle.disposed = true;
1159
+ lifecycle.ownershipLifecycleController.signal.removeEventListener("abort", lifecycle.onOwnerOrCallerAbort);
1160
+ if (lifecycle.callerSignal &&
1161
+ lifecycle.callerSignal !== lifecycle.ownershipLifecycleController.signal) {
1162
+ lifecycle.callerSignal.removeEventListener("abort", lifecycle.onOwnerOrCallerAbort);
1163
+ }
1164
+ for (const targetLifecycle of lifecycle.targets.values()) {
1165
+ const activeForTarget = this.ratelessDispatchTargets.get(targetLifecycle.target);
1166
+ activeForTarget?.delete(targetLifecycle);
1167
+ if (activeForTarget?.size === 0) {
1168
+ this.ratelessDispatchTargets.delete(targetLifecycle.target);
1169
+ }
1170
+ }
1171
+ }
1172
+ isRatelessDispatchLifecycleActive(lifecycle, target) {
1173
+ if (this.ratelessClosed ||
1174
+ lifecycle.disposed ||
1175
+ lifecycle.ownershipLifecycleController !==
1176
+ this.ratelessDispatchLifecycleController ||
1177
+ lifecycle.ownershipLifecycleController.signal.aborted ||
1178
+ lifecycle.callerSignal?.aborted ||
1179
+ lifecycle.controller.signal.aborted) {
1180
+ return false;
879
1181
  }
1182
+ if (target === undefined) {
1183
+ return true;
1184
+ }
1185
+ const targetLifecycle = lifecycle.targets.get(target);
1186
+ return (targetLifecycle !== undefined &&
1187
+ !targetLifecycle.controller.signal.aborted &&
1188
+ this.ratelessDispatchTargets.get(target)?.has(targetLifecycle) === true);
1189
+ }
1190
+ getIncomingSyncProcessKey(sender, syncId) {
1191
+ return `${sender.length}:${sender}${syncId}`;
1192
+ }
1193
+ isIncomingSyncGenerationActive(ownershipLifecycleController) {
1194
+ return (!this.ratelessClosed &&
1195
+ ownershipLifecycleController ===
1196
+ this.ratelessDispatchLifecycleController &&
1197
+ !ownershipLifecycleController.signal.aborted);
1198
+ }
1199
+ isIncomingSyncProcessActive(process) {
1200
+ return (this.isIncomingSyncGenerationActive(process.ownershipLifecycleController) &&
1201
+ !process.controller.signal.aborted &&
1202
+ this.ingoingSyncProcesses.get(process.key) === process);
880
1203
  }
881
1204
  async onMaybeMissingEntries(properties) {
1205
+ // Capture this rateless open generation synchronously, before any await.
1206
+ // In particular, signal-less internal repair work must not resume after
1207
+ // close/open and borrow the newly opened Simple generation.
1208
+ const lifecycle = this.captureRatelessDispatchLifecycle(properties.targets, properties.signal);
1209
+ try {
1210
+ await this.onMaybeMissingEntriesWithLifecycle(properties, lifecycle);
1211
+ }
1212
+ finally {
1213
+ this.finishRatelessDispatchLifecycle(lifecycle);
1214
+ }
1215
+ }
1216
+ async onMaybeMissingEntriesWithLifecycle(properties, lifecycle) {
1217
+ const signal = lifecycle.controller.signal;
882
1218
  const profile = this.properties.sync?.profile;
883
1219
  const startedAt = syncProfileStart(profile);
1220
+ let topLevelProfileEmitted = false;
1221
+ const emitTopLevelProfile = (details, measurements = {}) => {
1222
+ if (!profile || topLevelProfileEmitted) {
1223
+ return;
1224
+ }
1225
+ topLevelProfileEmitted = true;
1226
+ emitSyncProfileDuration(profile, startedAt, {
1227
+ name: "rateless.onMaybeMissingEntries",
1228
+ entries: properties.entries.size,
1229
+ targets: properties.targets.length,
1230
+ ...measurements,
1231
+ details,
1232
+ });
1233
+ };
1234
+ const emitCancelledTopLevelProfile = (phase, mode = "rateless") => emitTopLevelProfile({
1235
+ mode,
1236
+ phase,
1237
+ cancelled: true,
1238
+ }, { messages: 0, symbols: 0 });
884
1239
  // NOTE: this method is best-effort dispatch, not a per-hash convergence API.
885
1240
  // It may require follow-up repair rounds under churn/loss to fully close all gaps.
886
1241
  // Strategy:
@@ -890,6 +1245,12 @@ export class RatelessIBLTSynchronizer {
890
1245
  // such as those assigned to range boundaries.
891
1246
  let minSyncIbltSize = 333; // TODO: make configurable
892
1247
  let maxSyncWithSimpleMethod = 1e3;
1248
+ if (signal?.aborted) {
1249
+ emitCancelledTopLevelProfile("before-dispatch", properties.entries.size <= minSyncIbltSize
1250
+ ? "simple-small"
1251
+ : "rateless");
1252
+ return;
1253
+ }
893
1254
  const priorityFn = this.properties.sync?.priority;
894
1255
  const maxSimpleEntries = this.properties.sync?.maxSimpleEntries;
895
1256
  // Small batch => use simple synchronizer entirely
@@ -906,17 +1267,17 @@ export class RatelessIBLTSynchronizer {
906
1267
  await this.simple.onMaybeMissingEntries({
907
1268
  entries: properties.entries,
908
1269
  targets: properties.targets,
1270
+ signal,
909
1271
  });
1272
+ if (signal?.aborted) {
1273
+ return;
1274
+ }
910
1275
  }
911
1276
  finally {
912
- if (profile) {
913
- emitSyncProfileDuration(profile, startedAt, {
914
- name: "rateless.onMaybeMissingEntries",
915
- entries: properties.entries.size,
916
- targets: properties.targets.length,
917
- details: { mode: "simple-small" },
918
- });
919
- }
1277
+ emitTopLevelProfile({
1278
+ mode: "simple-small",
1279
+ cancelled: signal?.aborted || undefined,
1280
+ });
920
1281
  }
921
1282
  return;
922
1283
  }
@@ -980,29 +1341,43 @@ export class RatelessIBLTSynchronizer {
980
1341
  }
981
1342
  }
982
1343
  }
983
- if (naiveHashes.length > 0) {
1344
+ const useAllCoordinatesForIblt = allCoordinatesToSyncWithIblt.length === 0 ||
1345
+ naiveHashes.length > maxSyncWithSimpleMethod;
1346
+ if (useAllCoordinatesForIblt) {
1347
+ // If every entry is a range-boundary special case, or the special-case
1348
+ // set itself is too large for the Simple prelude, use one bounded
1349
+ // Rateless process for the full set. Sending the oversized Simple
1350
+ // prelude first can complete or abort the repair lifecycle before
1351
+ // StartSync is ever dispatched.
1352
+ allCoordinatesToSyncWithIblt = [];
1353
+ for (const entry of properties.entries.values()) {
1354
+ allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
1355
+ }
1356
+ }
1357
+ else if (naiveHashes.length > 0) {
984
1358
  // If there are special-case entries, sync them simply in parallel
985
1359
  if (priorityFn && naiveEntriesForPriority) {
986
1360
  await this.simple.onMaybeMissingEntries({
987
1361
  entries: naiveEntriesForPriority,
988
1362
  targets: properties.targets,
1363
+ signal,
989
1364
  });
990
1365
  }
991
1366
  else {
992
1367
  await this.simple.onMaybeMissingHashes({
993
1368
  hashes: naiveHashes,
994
1369
  targets: properties.targets,
1370
+ signal,
995
1371
  });
996
1372
  }
997
- }
998
- if (allCoordinatesToSyncWithIblt.length === 0 ||
999
- naiveHashes.length > maxSyncWithSimpleMethod) {
1000
- // Fallback: if nothing left for IBLT (or simple set is too large), include all in IBLT
1001
- allCoordinatesToSyncWithIblt = [];
1002
- for (const entry of properties.entries.values()) {
1003
- allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
1373
+ if (signal?.aborted) {
1374
+ emitCancelledTopLevelProfile("simple-prelude");
1375
+ return;
1004
1376
  }
1005
1377
  }
1378
+ const simplePreludeEntries = useAllCoordinatesForIblt
1379
+ ? 0
1380
+ : naiveHashes.length;
1006
1381
  if (profile) {
1007
1382
  emitSyncProfileDuration(profile, selectStartedAt, {
1008
1383
  name: "rateless.selectEntries",
@@ -1010,11 +1385,15 @@ export class RatelessIBLTSynchronizer {
1010
1385
  symbols: allCoordinatesToSyncWithIblt.length,
1011
1386
  targets: properties.targets.length,
1012
1387
  details: {
1013
- naiveEntries: naiveHashes.length,
1388
+ naiveEntries: simplePreludeEntries,
1014
1389
  priority: priorityFn != null,
1015
1390
  },
1016
1391
  });
1017
1392
  }
1393
+ if (signal?.aborted) {
1394
+ emitCancelledTopLevelProfile("entry-selection");
1395
+ return;
1396
+ }
1018
1397
  if (allCoordinatesToSyncWithIblt.length === 0) {
1019
1398
  if (profile) {
1020
1399
  emitSyncProfileEvent(profile, {
@@ -1023,327 +1402,914 @@ export class RatelessIBLTSynchronizer {
1023
1402
  targets: properties.targets.length,
1024
1403
  details: { mode: "simple-only" },
1025
1404
  });
1026
- emitSyncProfileDuration(profile, startedAt, {
1027
- name: "rateless.onMaybeMissingEntries",
1028
- entries: properties.entries.size,
1029
- targets: properties.targets.length,
1030
- details: { mode: "simple-only" },
1031
- });
1032
1405
  }
1406
+ emitTopLevelProfile({ mode: "simple-only" });
1033
1407
  return;
1034
1408
  }
1035
1409
  const outgoingHashes = priorityFn == null
1036
1410
  ? [...properties.entries.keys()]
1037
1411
  : this.getPrioritizedEntries(properties.entries).map((entry) => entry.hash);
1038
1412
  await ribltReady;
1413
+ if (signal?.aborted) {
1414
+ emitCancelledTopLevelProfile("riblt-ready");
1415
+ return;
1416
+ }
1039
1417
  // For smaller sets, the original `sqrt(n)` heuristic can occasionally under-provision
1040
1418
  // low-degree symbols early, causing an unnecessary `MoreSymbols` round-trip. Use a
1041
1419
  // small floor to make small-delta syncs more reliable without affecting large-n behavior.
1042
1420
  let initialSymbolCount = Math.round(Math.sqrt(allCoordinatesToSyncWithIblt.length)); // TODO choose better
1043
- initialSymbolCount = Math.max(64, initialSymbolCount);
1044
- const prepareStartedAt = syncProfileStart(profile);
1045
- const { encoder, start, end, initialSymbols } = prepareStartSyncEncoder(allCoordinatesToSyncWithIblt, this.properties.numbers.maxValue, initialSymbolCount);
1046
- if (profile) {
1047
- emitSyncProfileDuration(profile, prepareStartedAt, {
1048
- name: "rateless.prepareStartSyncEncoder",
1049
- entries: allCoordinatesToSyncWithIblt.length,
1050
- symbols: initialSymbols?.length,
1051
- details: {
1052
- initialSymbolCount,
1053
- includesInitialSymbols: initialSymbols != null,
1054
- },
1055
- });
1421
+ initialSymbolCount = Math.min(MAX_MORE_SYMBOLS_BATCH_SIZE, Math.max(64, initialSymbolCount));
1422
+ if (signal?.aborted) {
1423
+ emitCancelledTopLevelProfile("before-encoder-prepare");
1424
+ return;
1056
1425
  }
1057
- let startSyncSymbols = initialSymbols;
1058
- if (!startSyncSymbols) {
1059
- const produceStartedAt = syncProfileStart(profile);
1060
- startSyncSymbols = produceNextCodedSymbols(encoder, initialSymbolCount);
1061
- if (profile) {
1062
- emitSyncProfileDuration(profile, produceStartedAt, {
1063
- name: "rateless.produceStartSyncSymbols",
1064
- symbols: startSyncSymbols.length,
1065
- });
1426
+ const authorizedHashes = new Set(outgoingHashes);
1427
+ let messages = 0;
1428
+ let symbols = 0;
1429
+ for (const [target, targetLifecycle] of lifecycle.targets) {
1430
+ if (!this.isRatelessDispatchLifecycleActive(lifecycle)) {
1431
+ break;
1066
1432
  }
1067
- }
1068
- const startSync = new StartSync({
1069
- from: start,
1070
- to: end,
1071
- symbols: startSyncSymbols,
1072
- });
1073
- const syncId = getSyncIdString(startSync);
1074
- const clear = () => {
1075
- encoder.free();
1076
- clearTimeout(this.outgoingSyncProcesses.get(syncId)?.timeout);
1077
- this.outgoingSyncProcesses.delete(syncId);
1078
- };
1079
- const createTimeout = () => {
1080
- return setTimeout(clear, 1e4); // TODO arg
1081
- };
1082
- let lastSeqNo = -1n;
1083
- // Keep follow-up symbol payloads bounded. Each symbol is serialized as an
1084
- // object with three bigint fields, so very large batches can dominate heap under
1085
- // concurrent churn even though the native RIBLT encoder itself is compact.
1086
- const nextBatch = Math.max(MIN_MORE_SYMBOLS_BATCH_SIZE, Math.min(MAX_MORE_SYMBOLS_BATCH_SIZE, Math.ceil(allCoordinatesToSyncWithIblt.length / 4)));
1087
- const obj = {
1088
- encoder,
1089
- timeout: createTimeout(),
1090
- refresh: () => {
1091
- let prevTimeout = obj.timeout;
1092
- if (prevTimeout) {
1093
- clearTimeout(prevTimeout);
1094
- }
1095
- obj.timeout = createTimeout();
1096
- },
1097
- next: (properties) => {
1098
- if (properties.lastSeqNo <= lastSeqNo) {
1099
- return CodedSymbolBatch.fromSymbols([]);
1100
- }
1101
- lastSeqNo++;
1102
- obj.refresh(); // TODO use timestamp instead and collective pruning/refresh
1103
- const produceStartedAt = syncProfileStart(profile);
1104
- const symbols = produceNextCodedSymbols(encoder, nextBatch);
1105
- if (profile) {
1106
- emitSyncProfileDuration(profile, produceStartedAt, {
1107
- name: "rateless.produceMoreSymbols",
1108
- syncId,
1109
- symbols: symbols.length,
1433
+ if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
1434
+ continue;
1435
+ }
1436
+ const activeProcess = this.outgoingSyncProcessByTarget.get(target);
1437
+ if (activeProcess &&
1438
+ !activeProcess.signal.aborted &&
1439
+ this.isRatelessDispatchLifecycleActive(activeProcess.targetLifecycle.lifecycle, target)) {
1440
+ // A repair sweep can flush adjacent batches for the same target
1441
+ // faster than its first StartSync reaches the transport. Replacing
1442
+ // that process here aborts the in-flight send; repeated large
1443
+ // batches can therefore starve Rateless sync entirely. Keep the
1444
+ // unambiguous active process and use bounded Simple sync only for
1445
+ // hashes that it does not already authorize.
1446
+ const uncoveredHashes = outgoingHashes.filter((hash) => !activeProcess.authorizedHashes.has(hash));
1447
+ if (uncoveredHashes.length > 0) {
1448
+ await this.simple.onMaybeMissingHashes({
1449
+ hashes: uncoveredHashes,
1450
+ targets: [target],
1451
+ signal,
1110
1452
  });
1111
1453
  }
1112
- return symbols;
1113
- },
1114
- free: clear,
1115
- outgoingHashes,
1116
- };
1117
- this.outgoingSyncProcesses.set(syncId, obj);
1118
- if (profile) {
1119
- emitSyncProfileEvent(profile, {
1120
- name: "rateless.dispatchMode",
1121
- entries: properties.entries.size,
1122
- symbols: startSyncSymbols.length,
1123
- targets: properties.targets.length,
1124
- syncId,
1125
- details: { mode: "rateless" },
1454
+ continue;
1455
+ }
1456
+ const startedSymbols = this.startOutgoingRatelessSyncForTarget({
1457
+ targetLifecycle,
1458
+ coordinates: allCoordinatesToSyncWithIblt,
1459
+ outgoingHashes,
1460
+ authorizedHashes,
1461
+ initialSymbolCount,
1462
+ entryCount: properties.entries.size,
1126
1463
  });
1464
+ if (startedSymbols !== undefined) {
1465
+ messages += 1;
1466
+ symbols += startedSymbols;
1467
+ }
1127
1468
  }
1128
- const sendStartedAt = syncProfileStart(profile);
1129
- const sendResult = this.simple.rpc.send(startSync, {
1130
- mode: new SilentDelivery({ to: properties.targets, redundancy: 1 }),
1131
- priority: SYNC_MESSAGE_PRIORITY,
1132
- });
1133
- if (profile) {
1134
- void Promise.resolve(sendResult).then(() => emitSyncProfileDuration(profile, sendStartedAt, {
1135
- name: "rateless.sendStartSync",
1136
- messages: 1,
1137
- symbols: startSyncSymbols.length,
1138
- targets: properties.targets.length,
1139
- syncId,
1140
- }), () => emitSyncProfileDuration(profile, sendStartedAt, {
1141
- name: "rateless.sendStartSync",
1142
- messages: 1,
1143
- symbols: startSyncSymbols.length,
1144
- targets: properties.targets.length,
1145
- syncId,
1146
- details: { rejected: true },
1147
- }));
1148
- }
1149
- if (profile) {
1150
- emitSyncProfileDuration(profile, startedAt, {
1151
- name: "rateless.onMaybeMissingEntries",
1152
- entries: properties.entries.size,
1153
- messages: 1,
1154
- symbols: startSyncSymbols.length,
1155
- targets: properties.targets.length,
1156
- details: {
1157
- mode: "rateless",
1158
- ibltEntries: allCoordinatesToSyncWithIblt.length,
1159
- naiveEntries: naiveHashes.length,
1160
- },
1161
- });
1469
+ if (signal.aborted) {
1470
+ emitTopLevelProfile({
1471
+ mode: "rateless",
1472
+ phase: "start-sync-send",
1473
+ cancelled: true,
1474
+ ibltEntries: allCoordinatesToSyncWithIblt.length,
1475
+ naiveEntries: simplePreludeEntries,
1476
+ }, { messages, symbols });
1477
+ return;
1162
1478
  }
1479
+ emitTopLevelProfile({
1480
+ mode: "rateless",
1481
+ ibltEntries: allCoordinatesToSyncWithIblt.length,
1482
+ naiveEntries: simplePreludeEntries,
1483
+ }, { messages, symbols });
1163
1484
  }
1164
- async onMessage(message, context) {
1485
+ startOutgoingRatelessSyncForTarget(properties) {
1486
+ const lifecycle = properties.targetLifecycle.lifecycle;
1487
+ const target = properties.targetLifecycle.target;
1165
1488
  const profile = this.properties.sync?.profile;
1166
- if (message instanceof StartSync) {
1167
- const syncId = getSyncIdString(message);
1168
- if (this.ingoingSyncProcesses.has(syncId)) {
1169
- return true;
1170
- }
1171
- if (this.startedOrCompletedSynchronizations.has(syncId)) {
1172
- return true;
1489
+ if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
1490
+ return undefined;
1491
+ }
1492
+ const prepareStartedAt = syncProfileStart(profile);
1493
+ const { encoder, start, end, initialSymbols } = prepareStartSyncEncoder(properties.coordinates, this.properties.numbers.maxValue, properties.initialSymbolCount);
1494
+ let encoderFreed = false;
1495
+ let encoderTransferred = false;
1496
+ let obj;
1497
+ let processConstructed = false;
1498
+ const freeEncoder = () => {
1499
+ if (encoderFreed) {
1500
+ return;
1173
1501
  }
1174
- this.startedOrCompletedSynchronizations.add(syncId);
1175
- const wrapped = message.end < message.start;
1176
- const decoderStartedAt = syncProfileStart(profile);
1177
- const decoder = await this.getLocalDecoderForRange({
1178
- start1: message.start,
1179
- end1: wrapped ? this.properties.numbers.maxValue : message.end,
1180
- start2: 0n,
1181
- end2: wrapped ? message.end : 0n,
1182
- });
1502
+ encoderFreed = true;
1503
+ encoder.free();
1504
+ };
1505
+ try {
1183
1506
  if (profile) {
1184
- emitSyncProfileDuration(profile, decoderStartedAt, {
1185
- name: "rateless.getLocalDecoderForRange",
1186
- syncId,
1187
- details: { wrapped, found: decoder !== false },
1507
+ emitSyncProfileDuration(profile, prepareStartedAt, {
1508
+ name: "rateless.prepareStartSyncEncoder",
1509
+ entries: properties.coordinates.length,
1510
+ symbols: initialSymbols?.length,
1511
+ targets: 1,
1512
+ details: {
1513
+ initialSymbolCount: properties.initialSymbolCount,
1514
+ includesInitialSymbols: initialSymbols != null,
1515
+ },
1188
1516
  });
1189
1517
  }
1190
- if (!decoder) {
1191
- const sendStartedAt = syncProfileStart(profile);
1192
- await this.simple.rpc.send(new RequestAll({
1193
- syncId: message.syncId,
1194
- }), {
1195
- mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
1196
- priority: SYNC_MESSAGE_PRIORITY,
1197
- });
1518
+ if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
1519
+ freeEncoder();
1520
+ return undefined;
1521
+ }
1522
+ let startSyncSymbols = initialSymbols;
1523
+ if (!startSyncSymbols) {
1524
+ const produceStartedAt = syncProfileStart(profile);
1525
+ startSyncSymbols = produceNextCodedSymbols(encoder, properties.initialSymbolCount);
1198
1526
  if (profile) {
1199
- emitSyncProfileDuration(profile, sendStartedAt, {
1200
- name: "rateless.sendRequestAll",
1201
- messages: 1,
1527
+ emitSyncProfileDuration(profile, produceStartedAt, {
1528
+ name: "rateless.produceStartSyncSymbols",
1529
+ symbols: startSyncSymbols.length,
1202
1530
  targets: 1,
1203
- syncId,
1204
1531
  });
1205
1532
  }
1206
- return true;
1207
1533
  }
1208
- const createTimeout = () => {
1209
- return setTimeout(() => {
1210
- decoder.free();
1211
- this.ingoingSyncProcesses.delete(syncId);
1212
- }, 2e4); // TODO arg
1213
- };
1214
- let messageQueue = [];
1534
+ if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
1535
+ freeEncoder();
1536
+ return undefined;
1537
+ }
1538
+ const startSync = new StartSync({
1539
+ from: start,
1540
+ to: end,
1541
+ symbols: startSyncSymbols,
1542
+ });
1543
+ const syncId = getSyncIdString(startSync);
1544
+ const targetSignal = properties.targetLifecycle.controller.signal;
1545
+ const processController = new AbortController();
1546
+ const processSignal = processController.signal;
1547
+ let cleared = false;
1215
1548
  let lastSeqNo = -1n;
1216
- const obj = {
1217
- decoder,
1218
- timeout: createTimeout(),
1219
- refresh: () => {
1220
- let prevTimeout = obj.timeout;
1221
- if (prevTimeout) {
1222
- clearTimeout(prevTimeout);
1223
- }
1224
- obj.timeout = createTimeout();
1225
- },
1226
- process: async (newMessage) => {
1227
- obj.refresh(); // TODO use timestamp instead and collective pruning/refresh
1228
- if (newMessage.seqNo <= lastSeqNo) {
1229
- return undefined;
1230
- }
1231
- messageQueue.push(newMessage);
1232
- messageQueue.sort((a, b) => Number(a.seqNo - b.seqNo));
1233
- if (messageQueue[0].seqNo !== lastSeqNo + 1n) {
1234
- return;
1549
+ let symbolsProduced = startSyncSymbols.length;
1550
+ let symbolBudgetExhausted = false;
1551
+ let simpleFallbackPromise;
1552
+ const symbolBudget = getOutgoingRatelessSymbolBudget(properties.coordinates.length, startSyncSymbols.length);
1553
+ let onTargetAbort;
1554
+ const clear = (reason = new Error("rateless outgoing process closed")) => {
1555
+ if (cleared) {
1556
+ return;
1557
+ }
1558
+ cleared = true;
1559
+ // Timeout/replacement owns only the encoder process. Response leases
1560
+ // retain the original open/caller/target lifecycle independently, so a
1561
+ // response already accepted for delivery can finish after this process
1562
+ // expires while close, caller abort, and disconnect still cancel it.
1563
+ processController.abort(reason);
1564
+ if (onTargetAbort) {
1565
+ targetSignal.removeEventListener("abort", onTargetAbort);
1566
+ }
1567
+ if (obj.timeout) {
1568
+ clearTimeout(obj.timeout);
1569
+ }
1570
+ if (obj.deadlineTimeout) {
1571
+ clearTimeout(obj.deadlineTimeout);
1572
+ }
1573
+ if (this.outgoingSyncProcesses.get(syncId) === obj) {
1574
+ this.outgoingSyncProcesses.delete(syncId);
1575
+ }
1576
+ if (this.outgoingSyncProcessByTarget.get(target) === obj) {
1577
+ this.outgoingSyncProcessByTarget.delete(target);
1578
+ }
1579
+ properties.targetLifecycle.retainedByProcess = false;
1580
+ freeEncoder();
1581
+ this.maybeDisposeRatelessDispatchLifecycle(lifecycle);
1582
+ };
1583
+ const startSimpleFallback = () => {
1584
+ if (!simpleFallbackPromise) {
1585
+ // From this point overlapping ResponseMaybeSync authorization
1586
+ // belongs to Simple. Keeping Rateless first would ship the response
1587
+ // while leaving the duplicate Simple lease charged until its TTL.
1588
+ obj.simpleFallbackStarted = true;
1589
+ simpleFallbackPromise = Promise.resolve(this.simple.onMaybeMissingHashes({
1590
+ hashes: properties.outgoingHashes,
1591
+ targets: [target],
1592
+ signal: lifecycle.callerSignal,
1593
+ }));
1594
+ }
1595
+ return simpleFallbackPromise;
1596
+ };
1597
+ const fallbackAndClear = (reason) => {
1598
+ void startSimpleFallback().catch((error) => logger.error(error));
1599
+ clear(reason);
1600
+ };
1601
+ const createTimeout = () => {
1602
+ const timeout = setTimeout(() => fallbackAndClear(new Error("rateless outgoing process timed out")), OUTGOING_RATELESS_IDLE_TIMEOUT_MS);
1603
+ timeout.unref?.();
1604
+ return timeout;
1605
+ };
1606
+ const createDeadlineTimeout = () => {
1607
+ const timeout = setTimeout(() => fallbackAndClear(new Error("rateless outgoing process deadline exceeded")), RATELESS_PROCESS_ABSOLUTE_TIMEOUT_MS);
1608
+ timeout.unref?.();
1609
+ return timeout;
1610
+ };
1611
+ onTargetAbort = () => clear(targetSignal.reason);
1612
+ // Keep follow-up symbol payloads bounded. Each symbol is serialized as an
1613
+ // object with three bigint fields, so very large batches can dominate heap under
1614
+ // concurrent churn even though the native RIBLT encoder itself is compact.
1615
+ const nextBatch = Math.max(MIN_MORE_SYMBOLS_BATCH_SIZE, Math.min(MAX_MORE_SYMBOLS_BATCH_SIZE, Math.ceil(properties.coordinates.length / 4)));
1616
+ obj = {
1617
+ target,
1618
+ targetLifecycle: properties.targetLifecycle,
1619
+ encoder,
1620
+ timeout: undefined,
1621
+ deadlineTimeout: undefined,
1622
+ refresh: () => {
1623
+ if (obj.timeout) {
1624
+ clearTimeout(obj.timeout);
1625
+ }
1626
+ obj.timeout = createTimeout();
1627
+ },
1628
+ next: (message) => {
1629
+ if (processSignal.aborted || symbolBudgetExhausted) {
1630
+ return undefined;
1631
+ }
1632
+ if (message.lastSeqNo !== lastSeqNo + 1n) {
1633
+ return undefined;
1634
+ }
1635
+ lastSeqNo = message.lastSeqNo;
1636
+ const remainingBudget = symbolBudget - symbolsProduced;
1637
+ if (remainingBudget <= 0) {
1638
+ symbolBudgetExhausted = true;
1639
+ return {
1640
+ symbols: CodedSymbolBatch.fromSymbols([]),
1641
+ exhaustedAfterSend: true,
1642
+ };
1643
+ }
1644
+ const produceStartedAt = syncProfileStart(profile);
1645
+ const symbols = produceNextCodedSymbols(encoder, Math.min(nextBatch, remainingBudget));
1646
+ symbolsProduced += symbols.length;
1647
+ const exhaustedAfterSend = symbols.length === 0;
1648
+ if (exhaustedAfterSend) {
1649
+ symbolBudgetExhausted = true;
1650
+ }
1651
+ else {
1652
+ obj.refresh();
1653
+ }
1654
+ if (profile) {
1655
+ emitSyncProfileDuration(profile, produceStartedAt, {
1656
+ name: "rateless.produceMoreSymbols",
1657
+ syncId,
1658
+ symbols: symbols.length,
1659
+ targets: 1,
1660
+ });
1661
+ }
1662
+ return { symbols, exhaustedAfterSend };
1663
+ },
1664
+ startSimpleFallback,
1665
+ simpleFallbackStarted: false,
1666
+ free: clear,
1667
+ outgoingHashes: properties.outgoingHashes,
1668
+ authorizedHashes: properties.authorizedHashes,
1669
+ consumedResponseHashes: new Set(),
1670
+ processController,
1671
+ signal: processSignal,
1672
+ callerSignal: lifecycle.callerSignal,
1673
+ };
1674
+ processConstructed = true;
1675
+ if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
1676
+ clear();
1677
+ return undefined;
1678
+ }
1679
+ // Without a wire process id on ResponseMaybeSync, retaining at most one
1680
+ // process per target keeps sender/hash authorization unambiguous.
1681
+ this.outgoingSyncProcessByTarget
1682
+ .get(target)
1683
+ ?.free(new Error("rateless outgoing process replaced"));
1684
+ if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
1685
+ clear();
1686
+ return undefined;
1687
+ }
1688
+ properties.targetLifecycle.retainedByProcess = true;
1689
+ this.outgoingSyncProcesses.set(syncId, obj);
1690
+ this.outgoingSyncProcessByTarget.set(target, obj);
1691
+ obj.timeout = createTimeout();
1692
+ obj.deadlineTimeout = createDeadlineTimeout();
1693
+ targetSignal.addEventListener("abort", onTargetAbort, { once: true });
1694
+ encoderTransferred = true;
1695
+ if (targetSignal.aborted) {
1696
+ clear(targetSignal.reason);
1697
+ return undefined;
1698
+ }
1699
+ if (profile) {
1700
+ emitSyncProfileEvent(profile, {
1701
+ name: "rateless.dispatchMode",
1702
+ entries: properties.entryCount,
1703
+ symbols: startSyncSymbols.length,
1704
+ targets: 1,
1705
+ syncId,
1706
+ details: { mode: "rateless" },
1707
+ });
1708
+ }
1709
+ const sendStartedAt = syncProfileStart(profile);
1710
+ let sendResult;
1711
+ try {
1712
+ sendResult = Promise.resolve(this.simple.rpc.send(startSync, {
1713
+ mode: new SilentDelivery({
1714
+ to: [target],
1715
+ redundancy: 1,
1716
+ }),
1717
+ priority: SYNC_MESSAGE_PRIORITY,
1718
+ signal: processSignal,
1719
+ }));
1720
+ }
1721
+ catch (error) {
1722
+ sendResult = Promise.reject(error);
1723
+ }
1724
+ void sendResult.then(() => {
1725
+ if (profile) {
1726
+ emitSyncProfileDuration(profile, sendStartedAt, {
1727
+ name: "rateless.sendStartSync",
1728
+ messages: 1,
1729
+ symbols: startSyncSymbols.length,
1730
+ targets: 1,
1731
+ syncId,
1732
+ });
1733
+ }
1734
+ }, () => {
1735
+ clear();
1736
+ if (profile) {
1737
+ emitSyncProfileDuration(profile, sendStartedAt, {
1738
+ name: "rateless.sendStartSync",
1739
+ messages: 1,
1740
+ symbols: startSyncSymbols.length,
1741
+ targets: 1,
1742
+ syncId,
1743
+ details: {
1744
+ rejected: true,
1745
+ cancelled: processSignal.aborted || undefined,
1746
+ },
1747
+ });
1748
+ }
1749
+ });
1750
+ if (processSignal.aborted) {
1751
+ clear();
1752
+ return undefined;
1753
+ }
1754
+ return startSyncSymbols.length;
1755
+ }
1756
+ catch (error) {
1757
+ if (processConstructed) {
1758
+ obj.free(error);
1759
+ }
1760
+ throw error;
1761
+ }
1762
+ finally {
1763
+ if (!encoderTransferred) {
1764
+ freeEncoder();
1765
+ }
1766
+ }
1767
+ }
1768
+ consumeAuthorizedRatelessResponse(hashes, from) {
1769
+ const target = from.hashcode();
1770
+ const process = this.outgoingSyncProcessByTarget.get(target);
1771
+ if (!process ||
1772
+ process.signal.aborted ||
1773
+ !this.isRatelessDispatchLifecycleActive(process.targetLifecycle.lifecycle, target)) {
1774
+ return undefined;
1775
+ }
1776
+ if (this.activeRatelessResponseCount >=
1777
+ MAX_ACTIVE_RATELESS_RESPONSES_GLOBAL ||
1778
+ (this.activeRatelessResponseCountByPeer.get(target) ?? 0) >=
1779
+ MAX_ACTIVE_RATELESS_RESPONSES_PER_PEER) {
1780
+ return undefined;
1781
+ }
1782
+ const authorized = [];
1783
+ const remaining = [];
1784
+ const seen = new Set();
1785
+ for (const hash of hashes) {
1786
+ if (seen.has(hash)) {
1787
+ continue;
1788
+ }
1789
+ seen.add(hash);
1790
+ if (!process.authorizedHashes.has(hash) ||
1791
+ process.consumedResponseHashes.has(hash)) {
1792
+ remaining.push(hash);
1793
+ continue;
1794
+ }
1795
+ process.consumedResponseHashes.add(hash);
1796
+ authorized.push(hash);
1797
+ }
1798
+ if (authorized.length === 0) {
1799
+ return {
1800
+ process,
1801
+ authorized,
1802
+ remaining,
1803
+ signal: process.targetLifecycle.controller.signal,
1804
+ release: () => { },
1805
+ };
1806
+ }
1807
+ const targetLifecycle = process.targetLifecycle;
1808
+ targetLifecycle.responseLeases += 1;
1809
+ this.activeRatelessResponseCount += 1;
1810
+ this.activeRatelessResponseCountByPeer.set(target, (this.activeRatelessResponseCountByPeer.get(target) ?? 0) + 1);
1811
+ let released = false;
1812
+ return {
1813
+ process,
1814
+ authorized,
1815
+ remaining,
1816
+ signal: targetLifecycle.controller.signal,
1817
+ release: (options) => {
1818
+ if (released) {
1819
+ return;
1820
+ }
1821
+ released = true;
1822
+ if (options?.rollback) {
1823
+ for (const hash of authorized) {
1824
+ process.consumedResponseHashes.delete(hash);
1825
+ }
1826
+ }
1827
+ targetLifecycle.responseLeases -= 1;
1828
+ this.activeRatelessResponseCount -= 1;
1829
+ const activeForPeer = (this.activeRatelessResponseCountByPeer.get(target) ?? 1) - 1;
1830
+ if (activeForPeer === 0) {
1831
+ this.activeRatelessResponseCountByPeer.delete(target);
1832
+ }
1833
+ else {
1834
+ this.activeRatelessResponseCountByPeer.set(target, activeForPeer);
1835
+ }
1836
+ this.maybeDisposeRatelessDispatchLifecycle(targetLifecycle.lifecycle);
1837
+ },
1838
+ };
1839
+ }
1840
+ async onMessage(message, context) {
1841
+ const profile = this.properties.sync?.profile;
1842
+ if (message instanceof StartSync) {
1843
+ const from = context.from;
1844
+ const ownershipLifecycleController = this.ratelessDispatchLifecycleController;
1845
+ if (!from ||
1846
+ !this.isIncomingSyncGenerationActive(ownershipLifecycleController)) {
1847
+ return true;
1848
+ }
1849
+ const sender = from.hashcode();
1850
+ const syncId = getSyncIdString(message);
1851
+ const key = this.getIncomingSyncProcessKey(sender, syncId);
1852
+ const completedSynchronizations = this.startedOrCompletedSynchronizations;
1853
+ const admissionSet = this.incomingRatelessProcessAdmissions;
1854
+ if (this.ingoingSyncProcesses.has(key)) {
1855
+ return true;
1856
+ }
1857
+ if (completedSynchronizations.has(key)) {
1858
+ return true;
1859
+ }
1860
+ let admissionsForSender = 0;
1861
+ for (const admission of admissionSet) {
1862
+ if (admission.key === key) {
1863
+ return true;
1864
+ }
1865
+ if (admission.sender === sender) {
1866
+ admissionsForSender += 1;
1867
+ }
1868
+ }
1869
+ if (admissionSet.size >= MAX_INCOMING_RATELESS_PROCESSES ||
1870
+ admissionsForSender >= MAX_INCOMING_RATELESS_PROCESSES_PER_SENDER) {
1871
+ return true;
1872
+ }
1873
+ const admission = { key, sender };
1874
+ admissionSet.add(admission);
1875
+ const controller = new AbortController();
1876
+ let freed = false;
1877
+ let completed = false;
1878
+ let initializationPending = false;
1879
+ let fallbackSendPending = false;
1880
+ let admissionReleased = false;
1881
+ let fallbackGraceTimeout;
1882
+ let onOwnershipAbort;
1883
+ let obj;
1884
+ const releaseAdmissionIfSettled = () => {
1885
+ if (admissionReleased ||
1886
+ !freed ||
1887
+ initializationPending ||
1888
+ fallbackSendPending) {
1889
+ return;
1890
+ }
1891
+ admissionReleased = true;
1892
+ admissionSet.delete(admission);
1893
+ };
1894
+ const free = (reason = new Error("incoming rateless process closed")) => {
1895
+ if (freed) {
1896
+ return;
1897
+ }
1898
+ freed = true;
1899
+ // Abort first: no queued process/send continuation may observe the
1900
+ // object detached while its native decoder is still usable.
1901
+ controller.abort(reason);
1902
+ if (onOwnershipAbort) {
1903
+ ownershipLifecycleController.signal.removeEventListener("abort", onOwnershipAbort);
1904
+ }
1905
+ if (obj.timeout) {
1906
+ clearTimeout(obj.timeout);
1907
+ obj.timeout = undefined;
1908
+ }
1909
+ if (obj.deadlineTimeout) {
1910
+ clearTimeout(obj.deadlineTimeout);
1911
+ obj.deadlineTimeout = undefined;
1912
+ }
1913
+ if (fallbackGraceTimeout) {
1914
+ clearTimeout(fallbackGraceTimeout);
1915
+ fallbackGraceTimeout = undefined;
1916
+ }
1917
+ if (this.ingoingSyncProcesses.get(key) === obj) {
1918
+ this.ingoingSyncProcesses.delete(key);
1919
+ }
1920
+ const ownedDecoder = obj.decoder;
1921
+ obj.decoder = undefined;
1922
+ ownedDecoder?.free();
1923
+ releaseAdmissionIfSettled();
1924
+ };
1925
+ const complete = () => {
1926
+ if (completed || !this.isIncomingSyncProcessActive(obj)) {
1927
+ return;
1928
+ }
1929
+ completed = true;
1930
+ completedSynchronizations.add(key);
1931
+ free(new Error("incoming rateless process completed"));
1932
+ };
1933
+ obj = {
1934
+ key,
1935
+ syncId,
1936
+ sender,
1937
+ from,
1938
+ ownershipLifecycleController,
1939
+ controller,
1940
+ completedSynchronizations,
1941
+ timeout: undefined,
1942
+ deadlineTimeout: undefined,
1943
+ refresh: () => { },
1944
+ lastSeqNo: -1n,
1945
+ processedSymbols: 0,
1946
+ queuedSymbols: 0,
1947
+ symbolBudget: getIncomingRatelessSymbolBudget(message.symbols.length),
1948
+ process: async () => undefined,
1949
+ requestAll: async () => { },
1950
+ fallbackToSimple: async () => { },
1951
+ free,
1952
+ complete,
1953
+ };
1954
+ this.ingoingSyncProcesses.set(key, obj);
1955
+ onOwnershipAbort = () => free(ownershipLifecycleController.signal.reason);
1956
+ ownershipLifecycleController.signal.addEventListener("abort", onOwnershipAbort, { once: true });
1957
+ obj.deadlineTimeout = setTimeout(() => {
1958
+ void obj.fallbackToSimple(new Error("incoming rateless process deadline exceeded"));
1959
+ }, RATELESS_PROCESS_ABSOLUTE_TIMEOUT_MS);
1960
+ obj.deadlineTimeout.unref?.();
1961
+ if (!this.isIncomingSyncProcessActive(obj)) {
1962
+ free(ownershipLifecycleController.signal.reason);
1963
+ return true;
1964
+ }
1965
+ let requestAllPromise;
1966
+ obj.requestAll = () => {
1967
+ if (requestAllPromise) {
1968
+ return requestAllPromise;
1969
+ }
1970
+ requestAllPromise = (async () => {
1971
+ if (!this.isIncomingSyncProcessActive(obj)) {
1972
+ return;
1973
+ }
1974
+ fallbackSendPending = true;
1975
+ const sendStartedAt = syncProfileStart(profile);
1976
+ try {
1977
+ await this.simple.rpc.send(new RequestAll({
1978
+ syncId: message.syncId,
1979
+ }), {
1980
+ mode: new SilentDelivery({
1981
+ to: [obj.sender],
1982
+ redundancy: 1,
1983
+ }),
1984
+ priority: SYNC_MESSAGE_PRIORITY,
1985
+ signal: controller.signal,
1986
+ });
1987
+ if (this.isIncomingSyncProcessActive(obj) && profile) {
1988
+ emitSyncProfileDuration(profile, sendStartedAt, {
1989
+ name: "rateless.sendRequestAll",
1990
+ messages: 1,
1991
+ targets: 1,
1992
+ syncId,
1993
+ });
1994
+ }
1995
+ }
1996
+ finally {
1997
+ fallbackSendPending = false;
1998
+ if (this.isIncomingSyncProcessActive(obj)) {
1999
+ complete();
2000
+ }
2001
+ releaseAdmissionIfSettled();
2002
+ }
2003
+ })();
2004
+ return requestAllPromise;
2005
+ };
2006
+ let boundedFallbackPromise;
2007
+ obj.fallbackToSimple = (reason = new Error("incoming rateless fallback timed out")) => {
2008
+ if (boundedFallbackPromise) {
2009
+ return boundedFallbackPromise;
2010
+ }
2011
+ const requestAll = obj.requestAll();
2012
+ boundedFallbackPromise = new Promise((resolve) => {
2013
+ let settled = false;
2014
+ const settle = () => {
2015
+ if (settled) {
2016
+ return;
2017
+ }
2018
+ settled = true;
2019
+ controller.signal.removeEventListener("abort", onAbort);
2020
+ if (fallbackGraceTimeout) {
2021
+ clearTimeout(fallbackGraceTimeout);
2022
+ fallbackGraceTimeout = undefined;
2023
+ }
2024
+ resolve();
2025
+ };
2026
+ const onAbort = () => settle();
2027
+ controller.signal.addEventListener("abort", onAbort, {
2028
+ once: true,
2029
+ });
2030
+ fallbackGraceTimeout = setTimeout(() => {
2031
+ free(reason);
2032
+ settle();
2033
+ }, INCOMING_RATELESS_FALLBACK_GRACE_MS);
2034
+ fallbackGraceTimeout.unref?.();
2035
+ void requestAll.then(settle, settle);
2036
+ if (controller.signal.aborted) {
2037
+ onAbort();
2038
+ }
2039
+ });
2040
+ return boundedFallbackPromise;
2041
+ };
2042
+ if (message.symbols.length > MAX_MORE_SYMBOLS_BATCH_SIZE) {
2043
+ await obj.fallbackToSimple(new Error("oversized incoming rateless initial batch"));
2044
+ return true;
2045
+ }
2046
+ const wrapped = message.end < message.start;
2047
+ const decoderStartedAt = syncProfileStart(profile);
2048
+ let decoder;
2049
+ initializationPending = true;
2050
+ try {
2051
+ decoder = await this.getLocalDecoderForRange({
2052
+ start1: message.start,
2053
+ end1: wrapped ? this.properties.numbers.maxValue : message.end,
2054
+ start2: 0n,
2055
+ end2: wrapped ? message.end : 0n,
2056
+ }, {
2057
+ ownershipLifecycleController,
2058
+ signal: controller.signal,
2059
+ });
2060
+ }
2061
+ catch (error) {
2062
+ const processAborted = controller.signal.aborted;
2063
+ free(error);
2064
+ if (processAborted ||
2065
+ !this.isIncomingSyncGenerationActive(ownershipLifecycleController)) {
2066
+ return true;
2067
+ }
2068
+ throw error;
2069
+ }
2070
+ finally {
2071
+ initializationPending = false;
2072
+ releaseAdmissionIfSettled();
2073
+ }
2074
+ if (!this.isIncomingSyncProcessActive(obj)) {
2075
+ decoder && decoder.free();
2076
+ free(controller.signal.reason);
2077
+ return true;
2078
+ }
2079
+ if (decoder) {
2080
+ // Transfer the native decoder before invoking diagnostics. A profile
2081
+ // sink is caller code and may throw; from this point process cleanup
2082
+ // owns the decoder on every exit.
2083
+ obj.decoder = decoder;
2084
+ }
2085
+ try {
2086
+ if (profile) {
2087
+ emitSyncProfileDuration(profile, decoderStartedAt, {
2088
+ name: "rateless.getLocalDecoderForRange",
2089
+ syncId,
2090
+ details: { wrapped, found: decoder !== false },
2091
+ });
2092
+ }
2093
+ }
2094
+ catch (error) {
2095
+ free(error);
2096
+ throw error;
2097
+ }
2098
+ if (!decoder) {
2099
+ try {
2100
+ await obj.fallbackToSimple(new Error("incoming rateless decoder unavailable"));
2101
+ }
2102
+ catch (error) {
2103
+ if (controller.signal.aborted) {
2104
+ return true;
1235
2105
  }
1236
- const finalizeIfDecoded = () => {
1237
- if (!decoder.decoded()) {
1238
- return false;
2106
+ throw error;
2107
+ }
2108
+ return true;
2109
+ }
2110
+ const createTimeout = () => {
2111
+ const timeout = setTimeout(() => {
2112
+ void obj.fallbackToSimple(new Error("incoming rateless process timed out"));
2113
+ }, INCOMING_RATELESS_IDLE_TIMEOUT_MS);
2114
+ timeout.unref?.();
2115
+ return timeout;
2116
+ };
2117
+ let messageQueue = [];
2118
+ obj.refresh = () => {
2119
+ if (!this.isIncomingSyncProcessActive(obj)) {
2120
+ return;
2121
+ }
2122
+ if (obj.timeout) {
2123
+ clearTimeout(obj.timeout);
2124
+ }
2125
+ obj.timeout = createTimeout();
2126
+ };
2127
+ obj.process = async (newMessage) => {
2128
+ if (!this.isIncomingSyncProcessActive(obj)) {
2129
+ return undefined;
2130
+ }
2131
+ const symbolCount = CodedSymbolBatch.from(newMessage.symbols).length;
2132
+ if (symbolCount > MAX_MORE_SYMBOLS_BATCH_SIZE) {
2133
+ free(new Error("incoming rateless symbol batch exceeds limit"));
2134
+ return undefined;
2135
+ }
2136
+ if (newMessage.seqNo <= obj.lastSeqNo) {
2137
+ return undefined;
2138
+ }
2139
+ if (newMessage.seqNo >
2140
+ obj.lastSeqNo + MAX_INCOMING_RATELESS_SEQUENCE_GAP) {
2141
+ return undefined;
2142
+ }
2143
+ if (messageQueue.some((queued) => queued.seqNo === newMessage.seqNo)) {
2144
+ return undefined;
2145
+ }
2146
+ if (newMessage.seqNo !== obj.lastSeqNo + 1n &&
2147
+ messageQueue.length >= MAX_INCOMING_RATELESS_QUEUED_BATCHES) {
2148
+ return undefined;
2149
+ }
2150
+ if (obj.processedSymbols + obj.queuedSymbols + symbolCount >
2151
+ obj.symbolBudget) {
2152
+ return "fallback-to-simple";
2153
+ }
2154
+ if (newMessage.seqNo > 0n && symbolCount === 0) {
2155
+ return "fallback-to-simple";
2156
+ }
2157
+ messageQueue.push({ ...newMessage, symbolCount });
2158
+ obj.queuedSymbols += symbolCount;
2159
+ messageQueue.sort((a, b) => Number(a.seqNo - b.seqNo));
2160
+ if (messageQueue[0].seqNo !== obj.lastSeqNo + 1n) {
2161
+ return;
2162
+ }
2163
+ const finalizeIfDecoded = () => {
2164
+ if (!this.isIncomingSyncProcessActive(obj)) {
2165
+ return true;
2166
+ }
2167
+ if (!decoder.decoded()) {
2168
+ return false;
2169
+ }
2170
+ const remoteStartedAt = syncProfileStart(profile);
2171
+ const allMissingSymbolsInRemote = getRemoteSymbolValues(decoder);
2172
+ if (profile) {
2173
+ emitSyncProfileDuration(profile, remoteStartedAt, {
2174
+ name: "rateless.remoteSymbols",
2175
+ entries: allMissingSymbolsInRemote.length,
2176
+ symbols: allMissingSymbolsInRemote.length,
2177
+ syncId,
2178
+ });
2179
+ }
2180
+ // The IBLT decoder is based on a local snapshot. Entries can arrive via
2181
+ // overlapping repair before we issue the follow-up simple request, so
2182
+ // re-check local presence to avoid stale duplicate bounce-back.
2183
+ void this.simple
2184
+ .queueSync(allMissingSymbolsInRemote, obj.from)
2185
+ .catch((error) => logger.error(error));
2186
+ obj.complete();
2187
+ return true;
2188
+ };
2189
+ while (messageQueue.length > 0 &&
2190
+ messageQueue[0].seqNo === obj.lastSeqNo + 1n) {
2191
+ const symbolMessage = messageQueue.shift();
2192
+ if (!symbolMessage) {
2193
+ break;
2194
+ }
2195
+ obj.queuedSymbols -= symbolMessage.symbolCount;
2196
+ obj.lastSeqNo = symbolMessage.seqNo;
2197
+ obj.processedSymbols += symbolMessage.symbolCount;
2198
+ // Only an authenticated, previously unseen contiguous sequence advances
2199
+ // the idle deadline. Replays and speculative future batches do not.
2200
+ obj.refresh();
2201
+ const addBatchAndDecode = decoder
2202
+ .add_coded_symbols_and_try_decode;
2203
+ if (typeof BigUint64Array !== "undefined" && addBatchAndDecode) {
2204
+ const flatStartedAt = syncProfileStart(profile);
2205
+ const flatSymbols = flatFromCodedSymbols(symbolMessage.symbols);
2206
+ if (profile) {
2207
+ emitSyncProfileDuration(profile, flatStartedAt, {
2208
+ name: "rateless.symbolBatchToFlat",
2209
+ symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
2210
+ syncId,
2211
+ });
1239
2212
  }
1240
- const remoteStartedAt = syncProfileStart(profile);
1241
- const allMissingSymbolsInRemote = getRemoteSymbolValues(decoder);
2213
+ const decodeStartedAt = syncProfileStart(profile);
2214
+ const decoded = addBatchAndDecode.call(decoder, flatSymbols);
1242
2215
  if (profile) {
1243
- emitSyncProfileDuration(profile, remoteStartedAt, {
1244
- name: "rateless.remoteSymbols",
1245
- entries: allMissingSymbolsInRemote.length,
1246
- symbols: allMissingSymbolsInRemote.length,
2216
+ emitSyncProfileDuration(profile, decodeStartedAt, {
2217
+ name: "rateless.decodeBatch",
2218
+ symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
1247
2219
  syncId,
2220
+ details: { decoded },
1248
2221
  });
1249
2222
  }
1250
- // The IBLT decoder is based on a local snapshot. Entries can arrive via
1251
- // overlapping repair before we issue the follow-up simple request, so
1252
- // re-check local presence to avoid stale duplicate bounce-back.
1253
- this.simple.queueSync(allMissingSymbolsInRemote, context.from);
1254
- obj.free();
1255
- return true;
1256
- };
1257
- while (messageQueue.length > 0 &&
1258
- messageQueue[0].seqNo === lastSeqNo + 1n) {
1259
- const symbolMessage = messageQueue.shift();
1260
- if (!symbolMessage) {
1261
- break;
2223
+ if (decoded && finalizeIfDecoded()) {
2224
+ return true;
1262
2225
  }
1263
- lastSeqNo = symbolMessage.seqNo;
1264
- const addBatchAndDecode = decoder
1265
- .add_coded_symbols_and_try_decode;
1266
- if (typeof BigUint64Array !== "undefined" && addBatchAndDecode) {
1267
- const flatStartedAt = syncProfileStart(profile);
1268
- const flatSymbols = flatFromCodedSymbols(symbolMessage.symbols);
1269
- if (profile) {
1270
- emitSyncProfileDuration(profile, flatStartedAt, {
1271
- name: "rateless.symbolBatchToFlat",
1272
- symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
1273
- syncId,
1274
- });
1275
- }
1276
- const decodeStartedAt = syncProfileStart(profile);
1277
- const decoded = addBatchAndDecode.call(decoder, flatSymbols);
1278
- if (profile) {
1279
- emitSyncProfileDuration(profile, decodeStartedAt, {
1280
- name: "rateless.decodeBatch",
1281
- symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
1282
- syncId,
1283
- details: { decoded },
1284
- });
1285
- }
1286
- if (decoded && finalizeIfDecoded()) {
2226
+ continue;
2227
+ }
2228
+ const decodeLoopStartedAt = syncProfileStart(profile);
2229
+ let symbolsProcessed = 0;
2230
+ for (const symbol of CodedSymbolBatch.from(symbolMessage.symbols)) {
2231
+ symbolsProcessed += 1;
2232
+ const normalizedSymbol = symbol instanceof SymbolSerialized
2233
+ ? symbol
2234
+ : new SymbolSerialized({
2235
+ count: symbol.count,
2236
+ hash: symbol.hash,
2237
+ symbol: symbol.symbol,
2238
+ });
2239
+ decoder.add_coded_symbol(normalizedSymbol);
2240
+ try {
2241
+ decoder.try_decode();
2242
+ if (finalizeIfDecoded()) {
1287
2243
  return true;
1288
2244
  }
1289
- continue;
1290
2245
  }
1291
- const decodeLoopStartedAt = syncProfileStart(profile);
1292
- let symbolsProcessed = 0;
1293
- for (const symbol of CodedSymbolBatch.from(symbolMessage.symbols)) {
1294
- symbolsProcessed += 1;
1295
- const normalizedSymbol = symbol instanceof SymbolSerialized
1296
- ? symbol
1297
- : new SymbolSerialized({
1298
- count: symbol.count,
1299
- hash: symbol.hash,
1300
- symbol: symbol.symbol,
1301
- });
1302
- decoder.add_coded_symbol(normalizedSymbol);
1303
- try {
1304
- decoder.try_decode();
1305
- if (finalizeIfDecoded()) {
1306
- return true;
1307
- }
2246
+ catch (error) {
2247
+ if (error?.message === "Invalid degree" ||
2248
+ error === "Invalid degree") {
2249
+ logger.trace("Decoder reported invalid degree; waiting for more symbols");
2250
+ continue;
1308
2251
  }
1309
- catch (error) {
1310
- if (error?.message === "Invalid degree" ||
1311
- error === "Invalid degree") {
1312
- logger.trace("Decoder reported invalid degree; waiting for more symbols");
1313
- continue;
1314
- }
1315
- throw error;
1316
- }
1317
- }
1318
- if (profile) {
1319
- emitSyncProfileDuration(profile, decodeLoopStartedAt, {
1320
- name: "rateless.decodeSymbolLoop",
1321
- symbols: symbolsProcessed,
1322
- syncId,
1323
- });
2252
+ throw error;
1324
2253
  }
1325
2254
  }
1326
- return false;
1327
- },
1328
- free: () => {
1329
- decoder.free();
1330
- clearTimeout(this.ingoingSyncProcesses.get(syncId)?.timeout);
1331
- this.ingoingSyncProcesses.delete(syncId);
1332
- },
2255
+ if (profile) {
2256
+ emitSyncProfileDuration(profile, decodeLoopStartedAt, {
2257
+ name: "rateless.decodeSymbolLoop",
2258
+ symbols: symbolsProcessed,
2259
+ syncId,
2260
+ });
2261
+ }
2262
+ }
2263
+ return false;
1333
2264
  };
1334
- this.ingoingSyncProcesses.set(syncId, obj);
1335
- if (await obj.process({ seqNo: 0n, symbols: message.symbols })) {
2265
+ obj.timeout = createTimeout();
2266
+ let initialResult;
2267
+ try {
2268
+ initialResult = await obj.process({
2269
+ seqNo: 0n,
2270
+ symbols: message.symbols,
2271
+ });
2272
+ }
2273
+ catch (error) {
2274
+ const wasActive = this.isIncomingSyncProcessActive(obj);
2275
+ free(error);
2276
+ if (!wasActive) {
2277
+ return true;
2278
+ }
2279
+ throw error;
2280
+ }
2281
+ if (initialResult === true) {
2282
+ return true;
2283
+ }
2284
+ if (initialResult === "fallback-to-simple") {
2285
+ await obj.fallbackToSimple(new Error("incoming rateless symbol budget exhausted"));
2286
+ return true;
2287
+ }
2288
+ if (!this.isIncomingSyncProcessActive(obj)) {
1336
2289
  return true;
1337
2290
  }
1338
2291
  // not done, request more symbols
1339
2292
  const sendStartedAt = syncProfileStart(profile);
1340
- await this.simple.rpc.send(new RequestMoreSymbols({
1341
- lastSeqNo: 0n,
1342
- syncId: message.syncId,
1343
- }), {
1344
- mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
1345
- priority: SYNC_MESSAGE_PRIORITY,
1346
- });
2293
+ try {
2294
+ await this.simple.rpc.send(new RequestMoreSymbols({
2295
+ lastSeqNo: obj.lastSeqNo,
2296
+ syncId: message.syncId,
2297
+ }), {
2298
+ mode: new SilentDelivery({ to: [obj.sender], redundancy: 1 }),
2299
+ priority: SYNC_MESSAGE_PRIORITY,
2300
+ signal: controller.signal,
2301
+ });
2302
+ }
2303
+ catch (error) {
2304
+ if (!this.isIncomingSyncProcessActive(obj)) {
2305
+ return true;
2306
+ }
2307
+ free(error);
2308
+ throw error;
2309
+ }
2310
+ if (!this.isIncomingSyncProcessActive(obj)) {
2311
+ return true;
2312
+ }
1347
2313
  if (profile) {
1348
2314
  emitSyncProfileDuration(profile, sendStartedAt, {
1349
2315
  name: "rateless.sendRequestMoreSymbols",
@@ -1355,40 +2321,84 @@ export class RatelessIBLTSynchronizer {
1355
2321
  return true;
1356
2322
  }
1357
2323
  else if (message instanceof MoreSymbols) {
2324
+ const from = context.from;
2325
+ if (!from) {
2326
+ return true;
2327
+ }
2328
+ const sender = from.hashcode();
1358
2329
  const syncId = getSyncIdString(message);
1359
- const obj = this.ingoingSyncProcesses.get(syncId);
1360
- if (!obj) {
2330
+ const key = this.getIncomingSyncProcessKey(sender, syncId);
2331
+ const obj = this.ingoingSyncProcesses.get(key);
2332
+ if (!obj ||
2333
+ obj.sender !== sender ||
2334
+ !this.isIncomingSyncProcessActive(obj)) {
1361
2335
  return true;
1362
2336
  }
1363
- const outProcess = await obj.process(message);
2337
+ let outProcess;
2338
+ try {
2339
+ outProcess = await obj.process(message);
2340
+ }
2341
+ catch (error) {
2342
+ const wasActive = this.isIncomingSyncProcessActive(obj);
2343
+ obj.free(error);
2344
+ if (!wasActive) {
2345
+ return true;
2346
+ }
2347
+ throw error;
2348
+ }
1364
2349
  if (outProcess === true) {
1365
2350
  return true;
1366
2351
  }
2352
+ else if (outProcess === "fallback-to-simple") {
2353
+ try {
2354
+ await obj.fallbackToSimple(new Error("incoming rateless symbol budget exhausted"));
2355
+ }
2356
+ catch {
2357
+ // The bounded rateless process is already complete or aborted. A
2358
+ // later repair round may retry if the fallback request was lost.
2359
+ }
2360
+ return true;
2361
+ }
1367
2362
  else if (outProcess === undefined) {
1368
2363
  return true; // we don't have enough information, or received information that is redundant
1369
2364
  }
2365
+ if (!this.isIncomingSyncProcessActive(obj)) {
2366
+ return true;
2367
+ }
1370
2368
  // we are not done
1371
2369
  const sendStartedAt = syncProfileStart(profile);
1372
- const sendResult = this.simple.rpc.send(new RequestMoreSymbols({
1373
- lastSeqNo: message.seqNo,
1374
- syncId: message.syncId,
1375
- }), {
1376
- mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
1377
- priority: SYNC_MESSAGE_PRIORITY,
1378
- });
2370
+ try {
2371
+ await this.simple.rpc.send(new RequestMoreSymbols({
2372
+ lastSeqNo: obj.lastSeqNo,
2373
+ syncId: message.syncId,
2374
+ }), {
2375
+ mode: new SilentDelivery({ to: [obj.sender], redundancy: 1 }),
2376
+ priority: SYNC_MESSAGE_PRIORITY,
2377
+ signal: obj.controller.signal,
2378
+ });
2379
+ }
2380
+ catch {
2381
+ if (profile) {
2382
+ emitSyncProfileDuration(profile, sendStartedAt, {
2383
+ name: "rateless.sendRequestMoreSymbols",
2384
+ messages: 1,
2385
+ targets: 1,
2386
+ syncId,
2387
+ details: { rejected: true },
2388
+ });
2389
+ }
2390
+ return true;
2391
+ }
2392
+ if (!this.isIncomingSyncProcessActive(obj)) {
2393
+ return true;
2394
+ }
1379
2395
  if (profile) {
1380
- void Promise.resolve(sendResult).then(() => emitSyncProfileDuration(profile, sendStartedAt, {
1381
- name: "rateless.sendRequestMoreSymbols",
1382
- messages: 1,
1383
- targets: 1,
1384
- syncId,
1385
- }), () => emitSyncProfileDuration(profile, sendStartedAt, {
2396
+ emitSyncProfileDuration(profile, sendStartedAt, {
1386
2397
  name: "rateless.sendRequestMoreSymbols",
1387
2398
  messages: 1,
1388
2399
  targets: 1,
1389
2400
  syncId,
1390
- details: { rejected: true },
1391
- }));
2401
+ });
1392
2402
  }
1393
2403
  return true;
1394
2404
  }
@@ -1398,16 +2408,39 @@ export class RatelessIBLTSynchronizer {
1398
2408
  if (!obj) {
1399
2409
  return true;
1400
2410
  }
1401
- const symbols = obj.next(message);
2411
+ if (context.from?.hashcode() !== obj.target) {
2412
+ return true;
2413
+ }
2414
+ const signal = obj.signal;
2415
+ if (signal.aborted) {
2416
+ return true;
2417
+ }
2418
+ const next = obj.next(message);
2419
+ if (!next) {
2420
+ return true;
2421
+ }
2422
+ const { symbols, exhaustedAfterSend } = next;
2423
+ if (signal.aborted) {
2424
+ return true;
2425
+ }
1402
2426
  const sendStartedAt = syncProfileStart(profile);
1403
- await this.properties.rpc.send(new MoreSymbols({
1404
- lastSeqNo: message.lastSeqNo,
1405
- syncId: message.syncId,
1406
- symbols,
1407
- }), {
1408
- mode: new SilentDelivery({ to: [context.from], redundancy: 1 }),
1409
- priority: SYNC_MESSAGE_PRIORITY,
1410
- });
2427
+ try {
2428
+ await this.properties.rpc.send(new MoreSymbols({
2429
+ lastSeqNo: message.lastSeqNo,
2430
+ syncId: message.syncId,
2431
+ symbols,
2432
+ }), {
2433
+ mode: new SilentDelivery({ to: [obj.target], redundancy: 1 }),
2434
+ priority: SYNC_MESSAGE_PRIORITY,
2435
+ signal,
2436
+ });
2437
+ }
2438
+ catch (error) {
2439
+ if (signal?.aborted) {
2440
+ return true;
2441
+ }
2442
+ throw error;
2443
+ }
1411
2444
  if (profile) {
1412
2445
  emitSyncProfileDuration(profile, sendStartedAt, {
1413
2446
  name: "rateless.sendMoreSymbols",
@@ -1417,6 +2450,13 @@ export class RatelessIBLTSynchronizer {
1417
2450
  syncId,
1418
2451
  });
1419
2452
  }
2453
+ if (exhaustedAfterSend) {
2454
+ // Latch exhaustion in next() before the send begins, then retain this
2455
+ // bounded process long enough for RequestAll. Starting Simple eagerly
2456
+ // also keeps fallback working with older receivers that ignore an empty
2457
+ // terminal symbol batch.
2458
+ void obj.startSimpleFallback().catch((error) => logger.error(error));
2459
+ }
1420
2460
  return true;
1421
2461
  }
1422
2462
  else if (message instanceof RequestAll) {
@@ -1424,12 +2464,134 @@ export class RatelessIBLTSynchronizer {
1424
2464
  if (!p) {
1425
2465
  return true;
1426
2466
  }
1427
- await this.simple.onMaybeMissingHashes({
1428
- hashes: p.outgoingHashes,
1429
- targets: [context.from.hashcode()],
1430
- });
2467
+ if (context.from?.hashcode() !== p.target) {
2468
+ return true;
2469
+ }
2470
+ if (p.signal.aborted) {
2471
+ return true;
2472
+ }
2473
+ // RequestAll ends only this target's rateless attempt. Other target
2474
+ // encoders and response authorizations remain independently owned.
2475
+ const fallback = p.startSimpleFallback();
2476
+ p.free();
2477
+ await fallback;
1431
2478
  return true;
1432
2479
  }
2480
+ else if (message instanceof ResponseMaybeSync ||
2481
+ message instanceof ResponseMaybeSyncCapabilities) {
2482
+ const from = context.from;
2483
+ // Simple authorizations are exact request leases and take precedence
2484
+ // over Rateless' broader process authorization. This also handles a
2485
+ // delayed fallback/prelude response after the Rateless process for the
2486
+ // same target has been replaced.
2487
+ const simpleLeases = this.simple.consumeAuthorizedMaybeSyncResponse(message.hashes, from);
2488
+ const simpleHashes = simpleLeases.flatMap((lease) => lease.hashes);
2489
+ const simpleHashSet = new Set(simpleHashes);
2490
+ const ratelessCandidateHashes = [];
2491
+ let inspected = 0;
2492
+ const iterator = message.hashes[Symbol.iterator]();
2493
+ let exhausted = false;
2494
+ try {
2495
+ while (inspected < MAX_RATELESS_RESPONSE_HASHES_INSPECTED) {
2496
+ const next = iterator.next();
2497
+ if (next.done) {
2498
+ exhausted = true;
2499
+ break;
2500
+ }
2501
+ inspected += 1;
2502
+ if (!simpleHashSet.has(next.value)) {
2503
+ ratelessCandidateHashes.push(next.value);
2504
+ }
2505
+ }
2506
+ }
2507
+ finally {
2508
+ if (!exhausted) {
2509
+ iterator.return?.();
2510
+ }
2511
+ }
2512
+ const response = ratelessCandidateHashes.length > 0
2513
+ ? this.consumeAuthorizedRatelessResponse(ratelessCandidateHashes, from)
2514
+ : undefined;
2515
+ if (simpleLeases.length === 0 &&
2516
+ (!response || response.authorized.length === 0)) {
2517
+ response?.release();
2518
+ return true;
2519
+ }
2520
+ let firstError;
2521
+ let rollbackRatelessAuthorization = false;
2522
+ try {
2523
+ const simpleMessage = message instanceof ResponseMaybeSyncCapabilities
2524
+ ? new ResponseMaybeSyncCapabilities({
2525
+ hashes: simpleHashes,
2526
+ capabilities: message.capabilities,
2527
+ })
2528
+ : new ResponseMaybeSync({ hashes: simpleHashes });
2529
+ if (response && response.authorized.length > 0) {
2530
+ const responseStartedAt = syncProfileStart(profile);
2531
+ let responseShipment = {
2532
+ messages: 0,
2533
+ fused: false,
2534
+ entries: 0,
2535
+ };
2536
+ try {
2537
+ responseShipment =
2538
+ await this.simple.shipAuthorizedMaybeSyncResponse({
2539
+ hashes: response.authorized,
2540
+ from,
2541
+ response: message,
2542
+ signal: response.signal,
2543
+ });
2544
+ }
2545
+ catch (error) {
2546
+ firstError = error;
2547
+ rollbackRatelessAuthorization = true;
2548
+ }
2549
+ if (profile) {
2550
+ try {
2551
+ emitSyncProfileDuration(profile, responseStartedAt, {
2552
+ name: "simple.exchangeHeads",
2553
+ entries: responseShipment.entries,
2554
+ messages: responseShipment.messages,
2555
+ targets: 1,
2556
+ details: {
2557
+ source: "ratelessResponseMaybeSync",
2558
+ fused: responseShipment.fused,
2559
+ },
2560
+ });
2561
+ }
2562
+ catch (error) {
2563
+ firstError ??= error;
2564
+ }
2565
+ }
2566
+ }
2567
+ if (simpleLeases.length > 0) {
2568
+ try {
2569
+ await this.simple.shipAuthorizedMaybeSyncResponseLeases({
2570
+ leases: simpleLeases,
2571
+ from,
2572
+ response: simpleMessage,
2573
+ });
2574
+ }
2575
+ catch (error) {
2576
+ firstError ??= error;
2577
+ }
2578
+ }
2579
+ if (firstError !== undefined) {
2580
+ throw firstError;
2581
+ }
2582
+ return true;
2583
+ }
2584
+ finally {
2585
+ response?.release({ rollback: rollbackRatelessAuthorization });
2586
+ // The Simple helper owns these leases once entered and releases each
2587
+ // one in its own finally. Keep this outer ownership release as the
2588
+ // final safety net for diagnostics or setup failures that happen
2589
+ // before the helper can take responsibility.
2590
+ for (const lease of simpleLeases) {
2591
+ lease.release();
2592
+ }
2593
+ }
2594
+ }
1433
2595
  return this.simple.onMessage(message, context);
1434
2596
  }
1435
2597
  onReceivedEntries(properties) {
@@ -1459,16 +2621,42 @@ export class RatelessIBLTSynchronizer {
1459
2621
  return this.simple.onEntryRemovedHashes(hashes);
1460
2622
  }
1461
2623
  onPeerDisconnected(key) {
1462
- return this.simple.onPeerDisconnected(key);
2624
+ const target = typeof key === "string" ? key : key.hashcode();
2625
+ for (const process of [...this.ingoingSyncProcesses.values()]) {
2626
+ if (process.sender === target) {
2627
+ process.free(new Error("incoming rateless peer disconnected"));
2628
+ }
2629
+ }
2630
+ for (const targetLifecycle of [
2631
+ ...(this.ratelessDispatchTargets.get(target) ?? []),
2632
+ ]) {
2633
+ this.abortRatelessDispatchTarget(targetLifecycle, new Error("rateless sync target disconnected"));
2634
+ }
2635
+ return this.simple.onPeerDisconnected(target);
1463
2636
  }
1464
2637
  open() {
2638
+ const reason = new Error("rateless sync generation replaced");
2639
+ this.cancelRatelessRepairSessions(reason);
2640
+ this.ratelessDispatchLifecycleController.abort(reason);
2641
+ this.ratelessDispatchLifecycleController = new AbortController();
2642
+ this.startedOrCompletedSynchronizations = new Cache({ max: 1e4 });
2643
+ // Admission accounting intentionally spans local generations: native range
2644
+ // initialization and fallback delivery are not guaranteed to stop merely
2645
+ // because their logical process was aborted.
2646
+ this.ratelessClosed = false;
1465
2647
  return this.simple.open();
1466
2648
  }
1467
2649
  close() {
1468
- for (const [, obj] of this.ingoingSyncProcesses) {
2650
+ this.ratelessClosed = true;
2651
+ // Abort ownership first. Process abort listeners then cancel any in-flight
2652
+ // StartSync/MoreSymbols send before they detach or free native state.
2653
+ const reason = new Error("rateless synchronizer closed");
2654
+ this.cancelRatelessRepairSessions(reason);
2655
+ this.ratelessDispatchLifecycleController.abort(reason);
2656
+ for (const obj of [...this.ingoingSyncProcesses.values()]) {
1469
2657
  obj.free();
1470
2658
  }
1471
- for (const [, obj] of this.outgoingSyncProcesses) {
2659
+ for (const obj of [...this.outgoingSyncProcesses.values()]) {
1472
2660
  obj.free();
1473
2661
  }
1474
2662
  this.clearLocalRangeEncoderCache();