@peerbit/shared-log 13.2.12 → 13.2.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -42,6 +42,8 @@ import {
42
42
  syncProfileStart,
43
43
  } from "./profile.js";
44
44
  import {
45
+ ResponseMaybeSync,
46
+ ResponseMaybeSyncCapabilities,
45
47
  SYNC_MESSAGE_PRIORITY,
46
48
  SimpleSyncronizer,
47
49
  } from "./simple.js";
@@ -79,6 +81,7 @@ class SymbolSerialized implements SSymbol {
79
81
  const CODED_SYMBOL_WORDS = 3;
80
82
  const CODED_SYMBOL_WORD_BYTES = 8;
81
83
  const CODED_SYMBOL_BYTES = CODED_SYMBOL_WORDS * CODED_SYMBOL_WORD_BYTES;
84
+ const MAX_CODED_SYMBOL_BATCH_SIZE = 1_024;
82
85
  const BIG_UINT64_ARRAY_IS_LITTLE_ENDIAN =
83
86
  typeof BigUint64Array !== "undefined" &&
84
87
  new Uint8Array(new BigUint64Array([1n]).buffer)[0] === 1;
@@ -224,6 +227,9 @@ const codedSymbolBatchField = {
224
227
  },
225
228
  deserialize: (reader: BinaryReader): CodedSymbolBatch => {
226
229
  const length = reader.u32();
230
+ if (length > MAX_CODED_SYMBOL_BATCH_SIZE) {
231
+ throw new Error("RIBLT coded symbol batch exceeds the receiver limit");
232
+ }
227
233
  const wordLength = length * CODED_SYMBOL_WORDS;
228
234
  const byteLength = length * CODED_SYMBOL_BYTES;
229
235
 
@@ -466,7 +472,47 @@ const DEFAULT_CONVERGENT_REPAIR_TIMEOUT_MS = 30_000;
466
472
  const DEFAULT_CONVERGENT_RETRY_INTERVALS_MS = [0, 1_000, 3_000, 7_000];
467
473
  const DEFAULT_MAX_CONVERGENT_TRACKED_HASHES = 4_096;
468
474
  const MIN_MORE_SYMBOLS_BATCH_SIZE = 64;
469
- const MAX_MORE_SYMBOLS_BATCH_SIZE = 1_024;
475
+ const MAX_MORE_SYMBOLS_BATCH_SIZE = MAX_CODED_SYMBOL_BATCH_SIZE;
476
+ const MAX_INCOMING_RATELESS_PROCESSES = 32;
477
+ const MAX_INCOMING_RATELESS_PROCESSES_PER_SENDER = 4;
478
+ const MAX_INCOMING_RATELESS_QUEUED_BATCHES = 8;
479
+ const MAX_INCOMING_RATELESS_SEQUENCE_GAP = BigInt(
480
+ MAX_INCOMING_RATELESS_QUEUED_BATCHES,
481
+ );
482
+ const MIN_RATELESS_SYMBOL_BUDGET = 4_096;
483
+ const MAX_RATELESS_SYMBOL_BUDGET = 262_144;
484
+ const RATELESS_SYMBOL_BUDGET_MULTIPLIER = 4;
485
+ const INCOMING_RATELESS_IDLE_TIMEOUT_MS = 20_000;
486
+ const OUTGOING_RATELESS_IDLE_TIMEOUT_MS = 10_000;
487
+ const RATELESS_PROCESS_ABSOLUTE_TIMEOUT_MS = 120_000;
488
+ const INCOMING_RATELESS_FALLBACK_GRACE_MS = 5_000;
489
+ const MAX_RATELESS_RESPONSE_HASHES_INSPECTED = 10_000;
490
+ export const MAX_ACTIVE_RATELESS_RESPONSES_PER_PEER = 4;
491
+ export const MAX_ACTIVE_RATELESS_RESPONSES_GLOBAL = 32;
492
+
493
+ const getIncomingRatelessSymbolBudget = (initialSymbols: number): number =>
494
+ Math.min(
495
+ MAX_RATELESS_SYMBOL_BUDGET,
496
+ Math.max(
497
+ MIN_RATELESS_SYMBOL_BUDGET,
498
+ initialSymbols * initialSymbols * RATELESS_SYMBOL_BUDGET_MULTIPLIER,
499
+ ),
500
+ );
501
+
502
+ const getOutgoingRatelessSymbolBudget = (
503
+ entries: number,
504
+ initialSymbols: number,
505
+ ): number =>
506
+ Math.min(
507
+ MAX_RATELESS_SYMBOL_BUDGET,
508
+ Math.max(
509
+ MIN_RATELESS_SYMBOL_BUDGET,
510
+ initialSymbols,
511
+ entries * RATELESS_SYMBOL_BUDGET_MULTIPLIER,
512
+ ),
513
+ );
514
+
515
+ type IncomingRatelessProcessResult = boolean | undefined | "fallback-to-simple";
470
516
 
471
517
  @variant([3, 0])
472
518
  export class StartSync extends TransportMessage {
@@ -607,81 +653,177 @@ const buildEncoderOrDecoderFromRange = async <
607
653
  await ribltReady;
608
654
  const encoder =
609
655
  type === "encoder" ? new EncoderWrapper() : new DecoderWrapper();
656
+ let transferred = false;
657
+ try {
658
+ const rangeQueryStartedAt = syncProfileStart(profile);
659
+ let hashNumbers: Array<bigint | number> | BigUint64Array | undefined;
660
+ let source = "index";
661
+ const resolved = await resolveHashNumbersInRange?.({
662
+ end1: ranges.end1,
663
+ start1: ranges.start1,
664
+ end2: ranges.end2,
665
+ start2: ranges.start2,
666
+ });
667
+ if (resolved) {
668
+ source = "native";
669
+ hashNumbers =
670
+ typeof BigUint64Array !== "undefined" &&
671
+ resolved instanceof BigUint64Array
672
+ ? resolved
673
+ : [...resolved];
674
+ } else {
675
+ const entries = await entryIndex
676
+ .iterate(
677
+ {
678
+ // Range sync for IBLT is done in hashNumber space.
679
+ query: matchEntriesByHashNumberInRangeQuery({
680
+ end1: ranges.end1,
681
+ start1: ranges.start1,
682
+ end2: ranges.end2,
683
+ start2: ranges.start2,
684
+ }),
685
+ },
686
+ {
687
+ shape: {
688
+ hash: true,
689
+ hashNumber: true,
690
+ },
691
+ },
692
+ )
693
+ .all();
694
+ hashNumbers = entries.map((entry) => entry.value.hashNumber);
695
+ }
696
+ if (profile) {
697
+ emitSyncProfileDuration(profile, rangeQueryStartedAt, {
698
+ name: "rateless.rangeQuery",
699
+ entries: hashNumbers.length,
700
+ details: { type, source },
701
+ });
702
+ }
703
+
704
+ if (hashNumbers.length === 0) {
705
+ return false;
706
+ }
610
707
 
611
- const rangeQueryStartedAt = syncProfileStart(profile);
612
- let hashNumbers: Array<bigint | number> | BigUint64Array | undefined;
613
- let source = "index";
614
- const resolved = await resolveHashNumbersInRange?.({
615
- end1: ranges.end1,
616
- start1: ranges.start1,
617
- end2: ranges.end2,
618
- start2: ranges.start2,
619
- });
620
- if (resolved) {
621
- source = "native";
622
- hashNumbers =
708
+ const addSymbolsStartedAt = syncProfileStart(profile);
709
+ if (
623
710
  typeof BigUint64Array !== "undefined" &&
624
- resolved instanceof BigUint64Array
625
- ? resolved
626
- : [...resolved];
627
- } else {
628
- const entries = await entryIndex
629
- .iterate(
630
- {
631
- // Range sync for IBLT is done in hashNumber space.
632
- query: matchEntriesByHashNumberInRangeQuery({
633
- end1: ranges.end1,
634
- start1: ranges.start1,
635
- end2: ranges.end2,
636
- start2: ranges.start2,
637
- }),
638
- },
639
- {
640
- shape: {
641
- hash: true,
642
- hashNumber: true,
643
- },
644
- },
645
- )
646
- .all();
647
- hashNumbers = entries.map((entry) => entry.value.hashNumber);
648
- }
649
- if (profile) {
650
- emitSyncProfileDuration(profile, rangeQueryStartedAt, {
651
- name: "rateless.rangeQuery",
652
- entries: hashNumbers.length,
653
- details: { type, source },
654
- });
711
+ typeof (encoder as RibltSymbolAdder).add_symbols === "function"
712
+ ) {
713
+ const symbols =
714
+ hashNumbers instanceof BigUint64Array
715
+ ? hashNumbers
716
+ : BigUint64Array.from(hashNumbers, coerceBigInt);
717
+ addSymbolsToRiblt(encoder as RibltSymbolAdder, symbols);
718
+ } else {
719
+ for (const hashNumber of hashNumbers) {
720
+ encoder.add_symbol(coerceBigInt(hashNumber));
721
+ }
722
+ }
723
+ if (profile) {
724
+ emitSyncProfileDuration(profile, addSymbolsStartedAt, {
725
+ name: "rateless.rangeAddSymbols",
726
+ entries: hashNumbers.length,
727
+ symbols: hashNumbers.length,
728
+ details: { type },
729
+ });
730
+ }
731
+ transferred = true;
732
+ return encoder as E;
733
+ } finally {
734
+ if (!transferred) {
735
+ encoder.free();
736
+ }
655
737
  }
738
+ };
656
739
 
657
- if (hashNumbers.length === 0) {
658
- return false;
659
- }
740
+ type RatelessDispatchLifecycle = {
741
+ ownershipLifecycleController: AbortController;
742
+ callerSignal?: AbortSignal;
743
+ controller: AbortController;
744
+ targets: Map<string, RatelessDispatchTargetLifecycle>;
745
+ onOwnerOrCallerAbort: () => void;
746
+ dispatchFinished: boolean;
747
+ disposed: boolean;
748
+ };
660
749
 
661
- const addSymbolsStartedAt = syncProfileStart(profile);
662
- if (
663
- typeof BigUint64Array !== "undefined" &&
664
- typeof (encoder as RibltSymbolAdder).add_symbols === "function"
665
- ) {
666
- const symbols =
667
- hashNumbers instanceof BigUint64Array
668
- ? hashNumbers
669
- : BigUint64Array.from(hashNumbers, coerceBigInt);
670
- addSymbolsToRiblt(encoder as RibltSymbolAdder, symbols);
671
- } else {
672
- for (const hashNumber of hashNumbers) {
673
- encoder.add_symbol(coerceBigInt(hashNumber));
674
- }
675
- }
676
- if (profile) {
677
- emitSyncProfileDuration(profile, addSymbolsStartedAt, {
678
- name: "rateless.rangeAddSymbols",
679
- entries: hashNumbers.length,
680
- symbols: hashNumbers.length,
681
- details: { type },
682
- });
683
- }
684
- return encoder as E;
750
+ type RatelessDispatchTargetLifecycle = {
751
+ lifecycle: RatelessDispatchLifecycle;
752
+ target: string;
753
+ controller: AbortController;
754
+ retainedByProcess: boolean;
755
+ responseLeases: number;
756
+ };
757
+
758
+ type OutgoingRatelessSyncProcess = {
759
+ target: string;
760
+ targetLifecycle: RatelessDispatchTargetLifecycle;
761
+ outgoingHashes: string[];
762
+ authorizedHashes: ReadonlySet<string>;
763
+ consumedResponseHashes: Set<string>;
764
+ encoder: EncoderWrapper;
765
+ timeout?: ReturnType<typeof setTimeout>;
766
+ deadlineTimeout?: ReturnType<typeof setTimeout>;
767
+ refresh: () => void;
768
+ next: (message: {
769
+ lastSeqNo: bigint;
770
+ }) => { symbols: CodedSymbolBatch; exhaustedAfterSend: boolean } | undefined;
771
+ startSimpleFallback: () => Promise<void>;
772
+ simpleFallbackStarted: boolean;
773
+ free: (reason?: unknown) => void;
774
+ processController: AbortController;
775
+ signal: AbortSignal;
776
+ callerSignal?: AbortSignal;
777
+ };
778
+
779
+ type AuthorizedRatelessResponseLease = {
780
+ process: OutgoingRatelessSyncProcess;
781
+ authorized: string[];
782
+ remaining: string[];
783
+ signal: AbortSignal;
784
+ release: (options?: { rollback?: boolean }) => void;
785
+ };
786
+
787
+ type IncomingRatelessSyncProcess = {
788
+ key: string;
789
+ syncId: string;
790
+ sender: string;
791
+ from: PublicSignKey;
792
+ ownershipLifecycleController: AbortController;
793
+ controller: AbortController;
794
+ decoder?: DecoderWrapper;
795
+ completedSynchronizations: Cache<string>;
796
+ timeout?: ReturnType<typeof setTimeout>;
797
+ deadlineTimeout?: ReturnType<typeof setTimeout>;
798
+ refresh: () => void;
799
+ lastSeqNo: bigint;
800
+ processedSymbols: number;
801
+ queuedSymbols: number;
802
+ symbolBudget: number;
803
+ process: (message: {
804
+ seqNo: bigint;
805
+ symbols: CodedSymbolInput;
806
+ }) => Promise<IncomingRatelessProcessResult>;
807
+ requestAll: () => Promise<void>;
808
+ fallbackToSimple: (reason?: unknown) => Promise<void>;
809
+ free: (reason?: unknown) => void;
810
+ complete: () => void;
811
+ };
812
+
813
+ type IncomingRatelessProcessAdmission = {
814
+ key: string;
815
+ sender: string;
816
+ };
817
+
818
+ type RatelessRepairSessionLifecycle = {
819
+ id: string;
820
+ ownershipLifecycleController: AbortController;
821
+ controller: AbortController;
822
+ trackedSession: RepairSession;
823
+ onOwnershipAbort: () => void;
824
+ cancelled: boolean;
825
+ settled: boolean;
826
+ deadlineTimer?: ReturnType<typeof setTimeout>;
685
827
  };
686
828
 
687
829
  export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
@@ -689,6 +831,7 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
689
831
  {
690
832
  simple: SimpleSyncronizer<D>;
691
833
  private repairSessionCounter: number;
834
+ private ratelessRepairSessions: Map<string, RatelessRepairSessionLifecycle>;
692
835
 
693
836
  startedOrCompletedSynchronizations: Cache<string>;
694
837
  private localRangeEncoderCacheVersion = 0;
@@ -698,44 +841,40 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
698
841
  > = new Map();
699
842
  private localRangeEncoderCacheMax = 2;
700
843
 
701
- ingoingSyncProcesses: Map<
702
- string,
703
- {
704
- decoder: DecoderWrapper;
705
- timeout: ReturnType<typeof setTimeout>;
706
- refresh: () => void;
707
- process: (message: {
708
- seqNo: bigint;
709
- symbols: CodedSymbolInput;
710
- }) => Promise<boolean | undefined>;
711
- free: () => void;
712
- }
713
- >;
844
+ ingoingSyncProcesses: Map<string, IncomingRatelessSyncProcess>;
845
+ private incomingRatelessProcessAdmissions: Set<IncomingRatelessProcessAdmission>;
714
846
 
715
- outgoingSyncProcesses: Map<
847
+ outgoingSyncProcesses: Map<string, OutgoingRatelessSyncProcess>;
848
+ private outgoingSyncProcessByTarget: Map<string, OutgoingRatelessSyncProcess>;
849
+ private ratelessDispatchLifecycleController: AbortController;
850
+ private ratelessDispatchTargets: Map<
716
851
  string,
717
- {
718
- outgoingHashes: string[];
719
- encoder: EncoderWrapper;
720
- timeout: ReturnType<typeof setTimeout>;
721
- refresh: () => void;
722
- next: (message: { lastSeqNo: bigint }) => CodedSymbolBatch;
723
- free: () => void;
724
- }
852
+ Set<RatelessDispatchTargetLifecycle>
725
853
  >;
854
+ private activeRatelessResponseCount: number;
855
+ private activeRatelessResponseCountByPeer: Map<string, number>;
856
+ private ratelessClosed: boolean;
726
857
 
727
858
  constructor(readonly properties: SynchronizerComponents<D>) {
728
859
  this.simple = new SimpleSyncronizer(properties);
729
860
  this.repairSessionCounter = 0;
861
+ this.ratelessRepairSessions = new Map();
730
862
  this.outgoingSyncProcesses = new Map();
863
+ this.outgoingSyncProcessByTarget = new Map();
864
+ this.ratelessDispatchLifecycleController = new AbortController();
865
+ this.ratelessDispatchTargets = new Map();
866
+ this.activeRatelessResponseCount = 0;
867
+ this.activeRatelessResponseCountByPeer = new Map();
868
+ this.ratelessClosed = false;
731
869
  this.ingoingSyncProcesses = new Map();
870
+ this.incomingRatelessProcessAdmissions = new Set();
732
871
  this.startedOrCompletedSynchronizations = new Cache({ max: 1e4 });
733
872
  }
734
873
 
735
874
  private get maxConvergentTrackedHashes() {
736
875
  const value = this.properties.sync?.maxConvergentTrackedHashes;
737
876
  return value && Number.isFinite(value) && value > 0
738
- ? Math.floor(value)
877
+ ? Math.max(1, Math.floor(value))
739
878
  : DEFAULT_MAX_CONVERGENT_TRACKED_HASHES;
740
879
  }
741
880
 
@@ -775,6 +914,121 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
775
914
  return scored.map((x) => x.entry);
776
915
  }
777
916
 
917
+ private isRatelessRepairSessionActive(
918
+ session: RatelessRepairSessionLifecycle,
919
+ ): boolean {
920
+ return (
921
+ !this.ratelessClosed &&
922
+ !session.cancelled &&
923
+ !session.controller.signal.aborted &&
924
+ !session.ownershipLifecycleController.signal.aborted &&
925
+ session.ownershipLifecycleController ===
926
+ this.ratelessDispatchLifecycleController &&
927
+ this.ratelessRepairSessions.get(session.id) === session
928
+ );
929
+ }
930
+
931
+ private cancelRatelessRepairSession(
932
+ session: RatelessRepairSessionLifecycle,
933
+ reason: unknown = new Error("rateless repair session cancelled"),
934
+ ): void {
935
+ if (session.cancelled) {
936
+ return;
937
+ }
938
+ session.cancelled = true;
939
+ if (session.deadlineTimer !== undefined) {
940
+ clearTimeout(session.deadlineTimer);
941
+ session.deadlineTimer = undefined;
942
+ }
943
+ session.controller.abort(reason);
944
+ session.trackedSession.cancel();
945
+ }
946
+
947
+ private disposeRatelessRepairSession(
948
+ session: RatelessRepairSessionLifecycle,
949
+ ): void {
950
+ if (session.settled) {
951
+ return;
952
+ }
953
+ session.settled = true;
954
+ if (session.deadlineTimer !== undefined) {
955
+ clearTimeout(session.deadlineTimer);
956
+ session.deadlineTimer = undefined;
957
+ }
958
+ session.ownershipLifecycleController.signal.removeEventListener(
959
+ "abort",
960
+ session.onOwnershipAbort,
961
+ );
962
+ if (this.ratelessRepairSessions.get(session.id) === session) {
963
+ this.ratelessRepairSessions.delete(session.id);
964
+ }
965
+ }
966
+
967
+ private cancelRatelessRepairSessions(reason: unknown): void {
968
+ for (const session of [...this.ratelessRepairSessions.values()]) {
969
+ this.cancelRatelessRepairSession(session, reason);
970
+ }
971
+ }
972
+
973
+ private waitForRatelessRepairRetry(
974
+ delayMs: number,
975
+ signal: AbortSignal,
976
+ ): Promise<boolean> {
977
+ if (delayMs <= 0) {
978
+ return Promise.resolve(!signal.aborted);
979
+ }
980
+ if (signal.aborted) {
981
+ return Promise.resolve(false);
982
+ }
983
+ return new Promise<boolean>((resolve) => {
984
+ let settled = false;
985
+ let timer: ReturnType<typeof setTimeout>;
986
+ const settle = (elapsed: boolean) => {
987
+ if (settled) {
988
+ return;
989
+ }
990
+ settled = true;
991
+ clearTimeout(timer);
992
+ signal.removeEventListener("abort", onAbort);
993
+ resolve(elapsed);
994
+ };
995
+ const onAbort = () => settle(false);
996
+ timer = setTimeout(() => settle(true), delayMs);
997
+ timer.unref?.();
998
+ signal.addEventListener("abort", onAbort, { once: true });
999
+ if (signal.aborted) {
1000
+ onAbort();
1001
+ }
1002
+ });
1003
+ }
1004
+
1005
+ private waitForRatelessRepairDispatch(
1006
+ dispatch: Promise<void> | void,
1007
+ signal: AbortSignal,
1008
+ ): Promise<boolean> {
1009
+ const dispatchPromise = Promise.resolve(dispatch);
1010
+ return new Promise<boolean>((resolve) => {
1011
+ let settled = false;
1012
+ const settle = (completed: boolean) => {
1013
+ if (settled) {
1014
+ return;
1015
+ }
1016
+ settled = true;
1017
+ signal.removeEventListener("abort", onAbort);
1018
+ resolve(completed);
1019
+ };
1020
+ const onAbort = () => settle(false);
1021
+ signal.addEventListener("abort", onAbort, { once: true });
1022
+ void dispatchPromise.then(
1023
+ () => settle(true),
1024
+ () => settle(true),
1025
+ );
1026
+ if (signal.aborted) {
1027
+ onAbort();
1028
+ }
1029
+ });
1030
+ }
1031
+
778
1032
  startRepairSession(properties: {
779
1033
  entries: Map<string, SyncEntryCoordinates<D>>;
780
1034
  targets: string[];
@@ -814,7 +1068,6 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
814
1068
  trackedEntries.set(entry.hash, entry);
815
1069
  }
816
1070
 
817
- let cancelled = false;
818
1071
  const trackedSession = this.simple.startRepairSession({
819
1072
  entries: trackedEntries,
820
1073
  targets,
@@ -822,33 +1075,91 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
822
1075
  timeoutMs,
823
1076
  retryIntervalsMs,
824
1077
  });
1078
+ const ownershipLifecycleController =
1079
+ this.ratelessDispatchLifecycleController;
1080
+ let session!: RatelessRepairSessionLifecycle;
1081
+ const cancel = (
1082
+ reason: unknown = new Error("rateless repair session cancelled"),
1083
+ ) => this.cancelRatelessRepairSession(session, reason);
1084
+ session = {
1085
+ id,
1086
+ ownershipLifecycleController,
1087
+ controller: new AbortController(),
1088
+ trackedSession,
1089
+ onOwnershipAbort: () =>
1090
+ cancel(
1091
+ ownershipLifecycleController.signal.reason ??
1092
+ new Error("rateless repair generation ended"),
1093
+ ),
1094
+ cancelled: false,
1095
+ settled: false,
1096
+ deadlineTimer: undefined,
1097
+ };
1098
+ this.ratelessRepairSessions.set(id, session);
1099
+ session.deadlineTimer = setTimeout(
1100
+ () => cancel(new Error("rateless convergent repair session timed out")),
1101
+ Math.max(0, timeoutMs - (Date.now() - startedAt)),
1102
+ );
1103
+ session.deadlineTimer.unref?.();
1104
+ ownershipLifecycleController.signal.addEventListener(
1105
+ "abort",
1106
+ session.onOwnershipAbort,
1107
+ { once: true },
1108
+ );
1109
+ if (
1110
+ this.ratelessClosed ||
1111
+ ownershipLifecycleController.signal.aborted ||
1112
+ ownershipLifecycleController !==
1113
+ this.ratelessDispatchLifecycleController
1114
+ ) {
1115
+ cancel(
1116
+ ownershipLifecycleController.signal.reason ??
1117
+ new Error("rateless repair generation is not active"),
1118
+ );
1119
+ }
825
1120
 
826
1121
  const runDispatchSchedule = async () => {
827
1122
  let previousDelay = 0;
828
1123
  for (const delayMs of retryIntervalsMs) {
829
- if (cancelled) {
1124
+ if (!this.isRatelessRepairSessionActive(session)) {
830
1125
  return;
831
1126
  }
832
1127
  const elapsed = Date.now() - startedAt;
833
1128
  if (elapsed >= timeoutMs) {
834
1129
  return;
835
1130
  }
836
- const waitMs = Math.max(0, delayMs - previousDelay);
1131
+ const waitMs = Math.min(
1132
+ Math.max(0, delayMs - previousDelay),
1133
+ timeoutMs - elapsed,
1134
+ );
837
1135
  previousDelay = delayMs;
838
- if (waitMs > 0) {
839
- await new Promise<void>((resolve) => {
840
- const timer = setTimeout(resolve, waitMs);
841
- timer.unref?.();
842
- });
1136
+ if (
1137
+ !(await this.waitForRatelessRepairRetry(
1138
+ waitMs,
1139
+ session.controller.signal,
1140
+ ))
1141
+ ) {
1142
+ return;
843
1143
  }
844
- if (cancelled) {
1144
+ if (
1145
+ !this.isRatelessRepairSessionActive(session) ||
1146
+ Date.now() - startedAt >= timeoutMs
1147
+ ) {
845
1148
  return;
846
1149
  }
847
1150
  try {
848
- await this.onMaybeMissingEntries({
849
- entries: properties.entries,
850
- targets,
851
- });
1151
+ if (
1152
+ !(await this.waitForRatelessRepairDispatch(
1153
+ this.onMaybeMissingEntries({
1154
+ entries: properties.entries,
1155
+ targets,
1156
+ signal: session.controller.signal,
1157
+ }),
1158
+ session.controller.signal,
1159
+ ))
1160
+ ) {
1161
+ return;
1162
+ }
852
1163
  } catch {
853
1164
  // Best-effort schedule: tracked session timeout/result decides completion.
854
1165
  }
@@ -856,22 +1167,23 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
856
1167
  };
857
1168
 
858
1169
  const done = (async (): Promise<RepairSessionResult[]> => {
859
- await runDispatchSchedule();
860
- const trackedResults = await trackedSession.done;
861
- return trackedResults.map((result) => ({
862
- ...result,
863
- requestedTotal: requestedHashes.length,
864
- truncated: true,
865
- }));
1170
+ try {
1171
+ await runDispatchSchedule();
1172
+ const trackedResults = await trackedSession.done;
1173
+ return trackedResults.map((result) => ({
1174
+ ...result,
1175
+ requestedTotal: requestedHashes.length,
1176
+ truncated: true,
1177
+ }));
1178
+ } finally {
1179
+ this.disposeRatelessRepairSession(session);
1180
+ }
866
1181
  })();
867
1182
 
868
1183
  return {
869
1184
  id,
870
1185
  done,
871
- cancel: () => {
872
- cancelled = true;
873
- trackedSession.cancel();
874
- },
1186
+ cancel: () => cancel(),
875
1187
  };
876
1188
  }
877
1189
 
@@ -927,38 +1239,77 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
927
1239
  )}:${String(ranges.end2)}`;
928
1240
  }
929
1241
 
930
- private decoderFromCachedEncoder(encoder: EncoderWrapper): DecoderWrapper {
1242
+ private decoderFromCachedEncoder(
1243
+ encoder: EncoderWrapper,
1244
+ beforeDecoderTransfer?: () => void,
1245
+ ): DecoderWrapper {
931
1246
  const clone = encoder.clone();
932
- const decoder = clone.to_decoder();
933
- clone.free();
934
- return decoder;
1247
+ let cloneFreed = false;
1248
+ let decoder: DecoderWrapper | undefined;
1249
+ let decoderTransferred = false;
1250
+ const freeClone = () => {
1251
+ if (cloneFreed) {
1252
+ return;
1253
+ }
1254
+ cloneFreed = true;
1255
+ clone.free();
1256
+ };
1257
+ try {
1258
+ decoder = clone.to_decoder();
1259
+ beforeDecoderTransfer?.();
1260
+ freeClone();
1261
+ decoderTransferred = true;
1262
+ return decoder;
1263
+ } finally {
1264
+ try {
1265
+ freeClone();
1266
+ } finally {
1267
+ if (!decoderTransferred) {
1268
+ decoder?.free();
1269
+ }
1270
+ }
1271
+ }
935
1272
  }
936
1273
 
937
- private async getLocalDecoderForRange(ranges: {
938
- start1: NumberOrBigint;
939
- end1: NumberOrBigint;
940
- start2: NumberOrBigint;
941
- end2: NumberOrBigint;
942
- }): Promise<DecoderWrapper | false> {
1274
+ private async getLocalDecoderForRange(
1275
+ ranges: {
1276
+ start1: NumberOrBigint;
1277
+ end1: NumberOrBigint;
1278
+ start2: NumberOrBigint;
1279
+ end2: NumberOrBigint;
1280
+ },
1281
+ options?: {
1282
+ ownershipLifecycleController?: AbortController;
1283
+ signal?: AbortSignal;
1284
+ },
1285
+ ): Promise<DecoderWrapper | false> {
1286
+ const isActive = () =>
1287
+ options?.signal?.aborted !== true &&
1288
+ (options?.ownershipLifecycleController === undefined ||
1289
+ this.isIncomingSyncGenerationActive(
1290
+ options.ownershipLifecycleController,
1291
+ ));
1292
+ if (!isActive()) {
1293
+ return false;
1294
+ }
943
1295
  const profile = this.properties.sync?.profile;
944
1296
  const key = this.localRangeEncoderCacheKey(ranges);
945
1297
  const cached = this.localRangeEncoderCache.get(key);
946
1298
  if (cached && cached.version === this.localRangeEncoderCacheVersion) {
947
1299
  const startedAt = syncProfileStart(profile);
948
1300
  cached.lastUsed = Date.now();
949
- try {
950
- return this.decoderFromCachedEncoder(cached.encoder);
951
- } finally {
1301
+ return this.decoderFromCachedEncoder(cached.encoder, () => {
952
1302
  if (profile) {
953
1303
  emitSyncProfileDuration(profile, startedAt, {
954
1304
  name: "rateless.localDecoder",
955
1305
  cacheHit: true,
956
1306
  });
957
1307
  }
958
- }
1308
+ });
959
1309
  }
960
1310
 
961
1311
  const startedAt = syncProfileStart(profile);
1312
+ const cacheVersion = this.localRangeEncoderCacheVersion;
962
1313
  const encoder = (await buildEncoderOrDecoderFromRange(
963
1314
  ranges,
964
1315
  this.properties.entryIndex,
@@ -976,6 +1327,25 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
976
1327
  }
977
1328
  return false;
978
1329
  }
1330
+ if (!isActive()) {
1331
+ encoder.free();
1332
+ return false;
1333
+ }
1334
+ if (cacheVersion !== this.localRangeEncoderCacheVersion) {
1335
+ try {
1336
+ return this.decoderFromCachedEncoder(encoder, () => {
1337
+ if (profile) {
1338
+ emitSyncProfileDuration(profile, startedAt, {
1339
+ name: "rateless.localDecoder",
1340
+ cacheHit: false,
1341
+ details: { cacheInvalidated: true },
1342
+ });
1343
+ }
1344
+ });
1345
+ } finally {
1346
+ encoder.free();
1347
+ }
1348
+ }
979
1349
 
980
1350
  const now = Date.now();
981
1351
  const existing = this.localRangeEncoderCache.get(key);
@@ -1007,24 +1377,256 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
1007
1377
  this.localRangeEncoderCache.delete(oldestKey);
1008
1378
  }
1009
1379
 
1010
- try {
1011
- return this.decoderFromCachedEncoder(encoder);
1012
- } finally {
1380
+ return this.decoderFromCachedEncoder(encoder, () => {
1013
1381
  if (profile) {
1014
1382
  emitSyncProfileDuration(profile, startedAt, {
1015
1383
  name: "rateless.localDecoder",
1016
1384
  cacheHit: false,
1017
1385
  });
1018
1386
  }
1387
+ });
1388
+ }
1389
+
1390
+ private captureRatelessDispatchLifecycle(
1391
+ targets: string[],
1392
+ callerSignal?: AbortSignal,
1393
+ ): RatelessDispatchLifecycle {
1394
+ const ownershipLifecycleController =
1395
+ this.ratelessDispatchLifecycleController;
1396
+ const lifecycle = {
1397
+ ownershipLifecycleController,
1398
+ callerSignal,
1399
+ controller: new AbortController(),
1400
+ targets: new Map<string, RatelessDispatchTargetLifecycle>(),
1401
+ onOwnerOrCallerAbort: () => {
1402
+ const reason =
1403
+ callerSignal?.aborted === true
1404
+ ? callerSignal.reason
1405
+ : ownershipLifecycleController.signal.reason;
1406
+ this.abortRatelessDispatchLifecycle(lifecycle, reason);
1407
+ },
1408
+ dispatchFinished: false,
1409
+ disposed: false,
1410
+ } satisfies RatelessDispatchLifecycle;
1411
+
1412
+ for (const target of [...new Set(targets)]) {
1413
+ const targetLifecycle: RatelessDispatchTargetLifecycle = {
1414
+ lifecycle,
1415
+ target,
1416
+ controller: new AbortController(),
1417
+ retainedByProcess: false,
1418
+ responseLeases: 0,
1419
+ };
1420
+ lifecycle.targets.set(target, targetLifecycle);
1421
+ let activeForTarget = this.ratelessDispatchTargets.get(target);
1422
+ if (!activeForTarget) {
1423
+ activeForTarget = new Set();
1424
+ this.ratelessDispatchTargets.set(target, activeForTarget);
1425
+ }
1426
+ activeForTarget.add(targetLifecycle);
1427
+ }
1428
+
1429
+ ownershipLifecycleController.signal.addEventListener(
1430
+ "abort",
1431
+ lifecycle.onOwnerOrCallerAbort,
1432
+ { once: true },
1433
+ );
1434
+ if (callerSignal && callerSignal !== ownershipLifecycleController.signal) {
1435
+ callerSignal.addEventListener("abort", lifecycle.onOwnerOrCallerAbort, {
1436
+ once: true,
1437
+ });
1438
+ }
1439
+ if (
1440
+ this.ratelessClosed ||
1441
+ ownershipLifecycleController !==
1442
+ this.ratelessDispatchLifecycleController ||
1443
+ ownershipLifecycleController.signal.aborted ||
1444
+ callerSignal?.aborted
1445
+ ) {
1446
+ lifecycle.onOwnerOrCallerAbort();
1447
+ }
1448
+ return lifecycle;
1449
+ }
1450
+
1451
+ private abortRatelessDispatchTarget(
1452
+ targetLifecycle: RatelessDispatchTargetLifecycle,
1453
+ reason?: unknown,
1454
+ ): void {
1455
+ if (!targetLifecycle.controller.signal.aborted) {
1456
+ targetLifecycle.controller.abort(reason);
1457
+ }
1458
+ this.maybeDisposeRatelessDispatchLifecycle(targetLifecycle.lifecycle);
1459
+ }
1460
+
1461
+ private abortRatelessDispatchLifecycle(
1462
+ lifecycle: RatelessDispatchLifecycle,
1463
+ reason?: unknown,
1464
+ ): void {
1465
+ if (!lifecycle.controller.signal.aborted) {
1466
+ lifecycle.controller.abort(reason);
1467
+ }
1468
+ for (const targetLifecycle of lifecycle.targets.values()) {
1469
+ this.abortRatelessDispatchTarget(targetLifecycle, reason);
1470
+ }
1471
+ this.maybeDisposeRatelessDispatchLifecycle(lifecycle);
1472
+ }
1473
+
1474
+ private finishRatelessDispatchLifecycle(
1475
+ lifecycle: RatelessDispatchLifecycle,
1476
+ ): void {
1477
+ lifecycle.dispatchFinished = true;
1478
+ this.maybeDisposeRatelessDispatchLifecycle(lifecycle);
1479
+ }
1480
+
1481
+ private maybeDisposeRatelessDispatchLifecycle(
1482
+ lifecycle: RatelessDispatchLifecycle,
1483
+ ): void {
1484
+ if (
1485
+ lifecycle.disposed ||
1486
+ !lifecycle.dispatchFinished ||
1487
+ [...lifecycle.targets.values()].some(
1488
+ (target) => target.retainedByProcess || target.responseLeases > 0,
1489
+ )
1490
+ ) {
1491
+ return;
1492
+ }
1493
+ lifecycle.disposed = true;
1494
+ lifecycle.ownershipLifecycleController.signal.removeEventListener(
1495
+ "abort",
1496
+ lifecycle.onOwnerOrCallerAbort,
1497
+ );
1498
+ if (
1499
+ lifecycle.callerSignal &&
1500
+ lifecycle.callerSignal !== lifecycle.ownershipLifecycleController.signal
1501
+ ) {
1502
+ lifecycle.callerSignal.removeEventListener(
1503
+ "abort",
1504
+ lifecycle.onOwnerOrCallerAbort,
1505
+ );
1506
+ }
1507
+ for (const targetLifecycle of lifecycle.targets.values()) {
1508
+ const activeForTarget = this.ratelessDispatchTargets.get(
1509
+ targetLifecycle.target,
1510
+ );
1511
+ activeForTarget?.delete(targetLifecycle);
1512
+ if (activeForTarget?.size === 0) {
1513
+ this.ratelessDispatchTargets.delete(targetLifecycle.target);
1514
+ }
1515
+ }
1516
+ }
1517
+
1518
+ private isRatelessDispatchLifecycleActive(
1519
+ lifecycle: RatelessDispatchLifecycle,
1520
+ target?: string,
1521
+ ): boolean {
1522
+ if (
1523
+ this.ratelessClosed ||
1524
+ lifecycle.disposed ||
1525
+ lifecycle.ownershipLifecycleController !==
1526
+ this.ratelessDispatchLifecycleController ||
1527
+ lifecycle.ownershipLifecycleController.signal.aborted ||
1528
+ lifecycle.callerSignal?.aborted ||
1529
+ lifecycle.controller.signal.aborted
1530
+ ) {
1531
+ return false;
1019
1532
  }
1533
+ if (target === undefined) {
1534
+ return true;
1535
+ }
1536
+ const targetLifecycle = lifecycle.targets.get(target);
1537
+ return (
1538
+ targetLifecycle !== undefined &&
1539
+ !targetLifecycle.controller.signal.aborted &&
1540
+ this.ratelessDispatchTargets.get(target)?.has(targetLifecycle) === true
1541
+ );
1542
+ }
1543
+
1544
+ private getIncomingSyncProcessKey(sender: string, syncId: string): string {
1545
+ return `${sender.length}:${sender}${syncId}`;
1546
+ }
1547
+
1548
+ private isIncomingSyncGenerationActive(
1549
+ ownershipLifecycleController: AbortController,
1550
+ ): boolean {
1551
+ return (
1552
+ !this.ratelessClosed &&
1553
+ ownershipLifecycleController ===
1554
+ this.ratelessDispatchLifecycleController &&
1555
+ !ownershipLifecycleController.signal.aborted
1556
+ );
1557
+ }
1558
+
1559
+ private isIncomingSyncProcessActive(
1560
+ process: IncomingRatelessSyncProcess,
1561
+ ): boolean {
1562
+ return (
1563
+ this.isIncomingSyncGenerationActive(
1564
+ process.ownershipLifecycleController,
1565
+ ) &&
1566
+ !process.controller.signal.aborted &&
1567
+ this.ingoingSyncProcesses.get(process.key) === process
1568
+ );
1020
1569
  }
1021
1570
 
1022
1571
  async onMaybeMissingEntries(properties: {
1023
1572
  entries: Map<string, SyncEntryCoordinates<D>>;
1024
1573
  targets: string[];
1574
+ signal?: AbortSignal;
1025
1575
  }): Promise<void> {
1576
+ // Capture this rateless open generation synchronously, before any await.
1577
+ // In particular, signal-less internal repair work must not resume after
1578
+ // close/open and borrow the newly opened Simple generation.
1579
+ const lifecycle = this.captureRatelessDispatchLifecycle(
1580
+ properties.targets,
1581
+ properties.signal,
1582
+ );
1583
+ try {
1584
+ await this.onMaybeMissingEntriesWithLifecycle(properties, lifecycle);
1585
+ } finally {
1586
+ this.finishRatelessDispatchLifecycle(lifecycle);
1587
+ }
1588
+ }
1589
+
1590
+ private async onMaybeMissingEntriesWithLifecycle(
1591
+ properties: {
1592
+ entries: Map<string, SyncEntryCoordinates<D>>;
1593
+ targets: string[];
1594
+ signal?: AbortSignal;
1595
+ },
1596
+ lifecycle: RatelessDispatchLifecycle,
1597
+ ): Promise<void> {
1598
+ const signal = lifecycle.controller.signal;
1026
1599
  const profile = this.properties.sync?.profile;
1027
1600
  const startedAt = syncProfileStart(profile);
1601
+ let topLevelProfileEmitted = false;
1602
+ const emitTopLevelProfile = (
1603
+ details: Record<string, string | number | boolean | undefined>,
1604
+ measurements: { messages?: number; symbols?: number } = {},
1605
+ ) => {
1606
+ if (!profile || topLevelProfileEmitted) {
1607
+ return;
1608
+ }
1609
+ topLevelProfileEmitted = true;
1610
+ emitSyncProfileDuration(profile, startedAt, {
1611
+ name: "rateless.onMaybeMissingEntries",
1612
+ entries: properties.entries.size,
1613
+ targets: properties.targets.length,
1614
+ ...measurements,
1615
+ details,
1616
+ });
1617
+ };
1618
+ const emitCancelledTopLevelProfile = (
1619
+ phase: string,
1620
+ mode: "simple-small" | "rateless" = "rateless",
1621
+ ) =>
1622
+ emitTopLevelProfile(
1623
+ {
1624
+ mode,
1625
+ phase,
1626
+ cancelled: true,
1627
+ },
1628
+ { messages: 0, symbols: 0 },
1629
+ );
1028
1630
  // NOTE: this method is best-effort dispatch, not a per-hash convergence API.
1029
1631
  // It may require follow-up repair rounds under churn/loss to fully close all gaps.
1030
1632
  // Strategy:
@@ -1035,6 +1637,15 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
1035
1637
 
1036
1638
  let minSyncIbltSize = 333; // TODO: make configurable
1037
1639
  let maxSyncWithSimpleMethod = 1e3;
1640
+ if (signal?.aborted) {
1641
+ emitCancelledTopLevelProfile(
1642
+ "before-dispatch",
1643
+ properties.entries.size <= minSyncIbltSize
1644
+ ? "simple-small"
1645
+ : "rateless",
1646
+ );
1647
+ return;
1648
+ }
1038
1649
  const priorityFn = this.properties.sync?.priority;
1039
1650
  const maxSimpleEntries = this.properties.sync?.maxSimpleEntries;
1040
1651
 
@@ -1052,16 +1663,16 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
1052
1663
  await this.simple.onMaybeMissingEntries({
1053
1664
  entries: properties.entries,
1054
1665
  targets: properties.targets,
1666
+ signal,
1055
1667
  });
1056
- } finally {
1057
- if (profile) {
1058
- emitSyncProfileDuration(profile, startedAt, {
1059
- name: "rateless.onMaybeMissingEntries",
1060
- entries: properties.entries.size,
1061
- targets: properties.targets.length,
1062
- details: { mode: "simple-small" },
1063
- });
1668
+ if (signal?.aborted) {
1669
+ return;
1064
1670
  }
1671
+ } finally {
1672
+ emitTopLevelProfile({
1673
+ mode: "simple-small",
1674
+ cancelled: signal?.aborted || undefined,
1675
+ });
1065
1676
  }
1066
1677
  return;
1067
1678
  }
@@ -1143,32 +1754,44 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
1143
1754
  }
1144
1755
  }
1145
1756
 
1146
- if (naiveHashes.length > 0) {
1757
+ const useAllCoordinatesForIblt =
1758
+ allCoordinatesToSyncWithIblt.length === 0 ||
1759
+ naiveHashes.length > maxSyncWithSimpleMethod;
1760
+ if (useAllCoordinatesForIblt) {
1761
+ // If every entry is a range-boundary special case, or the special-case
1762
+ // set itself is too large for the Simple prelude, use one bounded
1763
+ // Rateless process for the full set. Sending the oversized Simple
1764
+ // prelude first can complete or abort the repair lifecycle before
1765
+ // StartSync is ever dispatched.
1766
+ allCoordinatesToSyncWithIblt = [];
1767
+ for (const entry of properties.entries.values()) {
1768
+ allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
1769
+ }
1770
+ } else if (naiveHashes.length > 0) {
1147
1771
  // If there are special-case entries, sync them simply in parallel
1148
1772
  if (priorityFn && naiveEntriesForPriority) {
1149
1773
  await this.simple.onMaybeMissingEntries({
1150
1774
  entries: naiveEntriesForPriority,
1151
1775
  targets: properties.targets,
1776
+ signal,
1152
1777
  });
1153
1778
  } else {
1154
1779
  await this.simple.onMaybeMissingHashes({
1155
1780
  hashes: naiveHashes,
1156
1781
  targets: properties.targets,
1782
+ signal,
1157
1783
  });
1158
1784
  }
1159
- }
1160
-
1161
- if (
1162
- allCoordinatesToSyncWithIblt.length === 0 ||
1163
- naiveHashes.length > maxSyncWithSimpleMethod
1164
- ) {
1165
- // Fallback: if nothing left for IBLT (or simple set is too large), include all in IBLT
1166
- allCoordinatesToSyncWithIblt = [];
1167
- for (const entry of properties.entries.values()) {
1168
- allCoordinatesToSyncWithIblt.push(coerceBigInt(entry.hashNumber));
1785
+ if (signal?.aborted) {
1786
+ emitCancelledTopLevelProfile("simple-prelude");
1787
+ return;
1169
1788
  }
1170
1789
  }
1171
1790
 
1791
+ const simplePreludeEntries = useAllCoordinatesForIblt
1792
+ ? 0
1793
+ : naiveHashes.length;
1794
+
1172
1795
  if (profile) {
1173
1796
  emitSyncProfileDuration(profile, selectStartedAt, {
1174
1797
  name: "rateless.selectEntries",
@@ -1176,11 +1799,15 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
1176
1799
  symbols: allCoordinatesToSyncWithIblt.length,
1177
1800
  targets: properties.targets.length,
1178
1801
  details: {
1179
- naiveEntries: naiveHashes.length,
1802
+ naiveEntries: simplePreludeEntries,
1180
1803
  priority: priorityFn != null,
1181
1804
  },
1182
1805
  });
1183
1806
  }
1807
+ if (signal?.aborted) {
1808
+ emitCancelledTopLevelProfile("entry-selection");
1809
+ return;
1810
+ }
1184
1811
 
1185
1812
  if (allCoordinatesToSyncWithIblt.length === 0) {
1186
1813
  if (profile) {
@@ -1190,13 +1817,8 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
1190
1817
  targets: properties.targets.length,
1191
1818
  details: { mode: "simple-only" },
1192
1819
  });
1193
- emitSyncProfileDuration(profile, startedAt, {
1194
- name: "rateless.onMaybeMissingEntries",
1195
- entries: properties.entries.size,
1196
- targets: properties.targets.length,
1197
- details: { mode: "simple-only" },
1198
- });
1199
1820
  }
1821
+ emitTopLevelProfile({ mode: "simple-only" });
1200
1822
  return;
1201
1823
  }
1202
1824
 
@@ -1207,6 +1829,10 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
1207
1829
  (entry) => entry.hash,
1208
1830
  );
1209
1831
  await ribltReady;
1832
+ if (signal?.aborted) {
1833
+ emitCancelledTopLevelProfile("riblt-ready");
1834
+ return;
1835
+ }
1210
1836
 
1211
1837
  // For smaller sets, the original `sqrt(n)` heuristic can occasionally under-provision
1212
1838
  // low-degree symbols early, causing an unnecessary `MoreSymbols` round-trip. Use a
@@ -1214,147 +1840,508 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
1214
1840
  let initialSymbolCount = Math.round(
1215
1841
  Math.sqrt(allCoordinatesToSyncWithIblt.length),
1216
1842
  ); // TODO choose better
1217
- initialSymbolCount = Math.max(64, initialSymbolCount);
1218
- const prepareStartedAt = syncProfileStart(profile);
1219
- const { encoder, start, end, initialSymbols } = prepareStartSyncEncoder(
1220
- allCoordinatesToSyncWithIblt,
1221
- this.properties.numbers.maxValue,
1222
- initialSymbolCount,
1843
+ initialSymbolCount = Math.min(
1844
+ MAX_MORE_SYMBOLS_BATCH_SIZE,
1845
+ Math.max(64, initialSymbolCount),
1223
1846
  );
1224
- if (profile) {
1225
- emitSyncProfileDuration(profile, prepareStartedAt, {
1226
- name: "rateless.prepareStartSyncEncoder",
1227
- entries: allCoordinatesToSyncWithIblt.length,
1228
- symbols: initialSymbols?.length,
1229
- details: {
1230
- initialSymbolCount,
1231
- includesInitialSymbols: initialSymbols != null,
1232
- },
1233
- });
1847
+ if (signal?.aborted) {
1848
+ emitCancelledTopLevelProfile("before-encoder-prepare");
1849
+ return;
1234
1850
  }
1235
-
1236
- let startSyncSymbols = initialSymbols;
1237
- if (!startSyncSymbols) {
1238
- const produceStartedAt = syncProfileStart(profile);
1239
- startSyncSymbols = produceNextCodedSymbols(encoder, initialSymbolCount);
1240
- if (profile) {
1241
- emitSyncProfileDuration(profile, produceStartedAt, {
1242
- name: "rateless.produceStartSyncSymbols",
1243
- symbols: startSyncSymbols.length,
1244
- });
1851
+ const authorizedHashes = new Set(outgoingHashes);
1852
+ let messages = 0;
1853
+ let symbols = 0;
1854
+ for (const [target, targetLifecycle] of lifecycle.targets) {
1855
+ if (!this.isRatelessDispatchLifecycleActive(lifecycle)) {
1856
+ break;
1857
+ }
1858
+ if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
1859
+ continue;
1860
+ }
1861
+ const activeProcess = this.outgoingSyncProcessByTarget.get(target);
1862
+ if (
1863
+ activeProcess &&
1864
+ !activeProcess.signal.aborted &&
1865
+ this.isRatelessDispatchLifecycleActive(
1866
+ activeProcess.targetLifecycle.lifecycle,
1867
+ target,
1868
+ )
1869
+ ) {
1870
+ // A repair sweep can flush adjacent batches for the same target
1871
+ // faster than its first StartSync reaches the transport. Replacing
1872
+ // that process here aborts the in-flight send; repeated large
1873
+ // batches can therefore starve Rateless sync entirely. Keep the
1874
+ // unambiguous active process and use bounded Simple sync only for
1875
+ // hashes that it does not already authorize.
1876
+ const uncoveredHashes = outgoingHashes.filter(
1877
+ (hash) => !activeProcess.authorizedHashes.has(hash),
1878
+ );
1879
+ if (uncoveredHashes.length > 0) {
1880
+ await this.simple.onMaybeMissingHashes({
1881
+ hashes: uncoveredHashes,
1882
+ targets: [target],
1883
+ signal,
1884
+ });
1885
+ }
1886
+ continue;
1887
+ }
1888
+ const startedSymbols = this.startOutgoingRatelessSyncForTarget({
1889
+ targetLifecycle,
1890
+ coordinates: allCoordinatesToSyncWithIblt,
1891
+ outgoingHashes,
1892
+ authorizedHashes,
1893
+ initialSymbolCount,
1894
+ entryCount: properties.entries.size,
1895
+ });
1896
+ if (startedSymbols !== undefined) {
1897
+ messages += 1;
1898
+ symbols += startedSymbols;
1245
1899
  }
1246
1900
  }
1901
+ if (signal.aborted) {
1902
+ emitTopLevelProfile(
1903
+ {
1904
+ mode: "rateless",
1905
+ phase: "start-sync-send",
1906
+ cancelled: true,
1907
+ ibltEntries: allCoordinatesToSyncWithIblt.length,
1908
+ naiveEntries: simplePreludeEntries,
1909
+ },
1910
+ { messages, symbols },
1911
+ );
1912
+ return;
1913
+ }
1914
+ emitTopLevelProfile(
1915
+ {
1916
+ mode: "rateless",
1917
+ ibltEntries: allCoordinatesToSyncWithIblt.length,
1918
+ naiveEntries: simplePreludeEntries,
1919
+ },
1920
+ { messages, symbols },
1921
+ );
1922
+ }
1247
1923
 
1248
- const startSync = new StartSync({
1249
- from: start,
1250
- to: end,
1251
- symbols: startSyncSymbols,
1252
- });
1253
- const syncId = getSyncIdString(startSync);
1924
+ private startOutgoingRatelessSyncForTarget(properties: {
1925
+ targetLifecycle: RatelessDispatchTargetLifecycle;
1926
+ coordinates: bigint[];
1927
+ outgoingHashes: string[];
1928
+ authorizedHashes: ReadonlySet<string>;
1929
+ initialSymbolCount: number;
1930
+ entryCount: number;
1931
+ }): number | undefined {
1932
+ const lifecycle = properties.targetLifecycle.lifecycle;
1933
+ const target = properties.targetLifecycle.target;
1934
+ const profile = this.properties.sync?.profile;
1935
+ if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
1936
+ return undefined;
1937
+ }
1254
1938
 
1255
- const clear = () => {
1939
+ const prepareStartedAt = syncProfileStart(profile);
1940
+ const { encoder, start, end, initialSymbols } = prepareStartSyncEncoder(
1941
+ properties.coordinates,
1942
+ this.properties.numbers.maxValue,
1943
+ properties.initialSymbolCount,
1944
+ );
1945
+ let encoderFreed = false;
1946
+ let encoderTransferred = false;
1947
+ let obj!: OutgoingRatelessSyncProcess;
1948
+ let processConstructed = false;
1949
+ const freeEncoder = () => {
1950
+ if (encoderFreed) {
1951
+ return;
1952
+ }
1953
+ encoderFreed = true;
1256
1954
  encoder.free();
1257
- clearTimeout(this.outgoingSyncProcesses.get(syncId)?.timeout);
1258
- this.outgoingSyncProcesses.delete(syncId);
1259
- };
1260
- const createTimeout = () => {
1261
- return setTimeout(clear, 1e4); // TODO arg
1262
1955
  };
1956
+ try {
1957
+ if (profile) {
1958
+ emitSyncProfileDuration(profile, prepareStartedAt, {
1959
+ name: "rateless.prepareStartSyncEncoder",
1960
+ entries: properties.coordinates.length,
1961
+ symbols: initialSymbols?.length,
1962
+ targets: 1,
1963
+ details: {
1964
+ initialSymbolCount: properties.initialSymbolCount,
1965
+ includesInitialSymbols: initialSymbols != null,
1966
+ },
1967
+ });
1968
+ }
1969
+ if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
1970
+ freeEncoder();
1971
+ return undefined;
1972
+ }
1263
1973
 
1264
- let lastSeqNo = -1n;
1265
- // Keep follow-up symbol payloads bounded. Each symbol is serialized as an
1266
- // object with three bigint fields, so very large batches can dominate heap under
1267
- // concurrent churn even though the native RIBLT encoder itself is compact.
1268
- const nextBatch = Math.max(
1269
- MIN_MORE_SYMBOLS_BATCH_SIZE,
1270
- Math.min(
1271
- MAX_MORE_SYMBOLS_BATCH_SIZE,
1272
- Math.ceil(allCoordinatesToSyncWithIblt.length / 4),
1273
- ),
1274
- );
1275
- const obj = {
1276
- encoder,
1277
- timeout: createTimeout(),
1278
- refresh: () => {
1279
- let prevTimeout = obj.timeout;
1280
- if (prevTimeout) {
1281
- clearTimeout(prevTimeout);
1282
- }
1283
- obj.timeout = createTimeout();
1284
- },
1285
- next: (properties: { lastSeqNo: bigint }): CodedSymbolBatch => {
1286
- if (properties.lastSeqNo <= lastSeqNo) {
1287
- return CodedSymbolBatch.fromSymbols([]);
1288
- }
1289
- lastSeqNo++;
1290
- obj.refresh(); // TODO use timestamp instead and collective pruning/refresh
1291
-
1974
+ let startSyncSymbols = initialSymbols;
1975
+ if (!startSyncSymbols) {
1292
1976
  const produceStartedAt = syncProfileStart(profile);
1293
- const symbols = produceNextCodedSymbols(encoder, nextBatch);
1977
+ startSyncSymbols = produceNextCodedSymbols(
1978
+ encoder,
1979
+ properties.initialSymbolCount,
1980
+ );
1294
1981
  if (profile) {
1295
1982
  emitSyncProfileDuration(profile, produceStartedAt, {
1296
- name: "rateless.produceMoreSymbols",
1297
- syncId,
1298
- symbols: symbols.length,
1983
+ name: "rateless.produceStartSyncSymbols",
1984
+ symbols: startSyncSymbols.length,
1985
+ targets: 1,
1299
1986
  });
1300
1987
  }
1301
- return symbols;
1302
- },
1303
- free: clear,
1304
- outgoingHashes,
1305
- };
1988
+ }
1989
+ if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
1990
+ freeEncoder();
1991
+ return undefined;
1992
+ }
1306
1993
 
1307
- this.outgoingSyncProcesses.set(syncId, obj);
1308
- if (profile) {
1309
- emitSyncProfileEvent(profile, {
1310
- name: "rateless.dispatchMode",
1311
- entries: properties.entries.size,
1312
- symbols: startSyncSymbols.length,
1313
- targets: properties.targets.length,
1314
- syncId,
1315
- details: { mode: "rateless" },
1994
+ const startSync = new StartSync({
1995
+ from: start,
1996
+ to: end,
1997
+ symbols: startSyncSymbols,
1316
1998
  });
1317
- }
1318
- const sendStartedAt = syncProfileStart(profile);
1319
- const sendResult = this.simple.rpc.send(startSync, {
1320
- mode: new SilentDelivery({ to: properties.targets, redundancy: 1 }),
1321
- priority: SYNC_MESSAGE_PRIORITY,
1322
- });
1323
- if (profile) {
1324
- void Promise.resolve(sendResult).then(
1325
- () =>
1326
- emitSyncProfileDuration(profile, sendStartedAt, {
1327
- name: "rateless.sendStartSync",
1328
- messages: 1,
1329
- symbols: startSyncSymbols.length,
1330
- targets: properties.targets.length,
1331
- syncId,
1332
- }),
1333
- () =>
1334
- emitSyncProfileDuration(profile, sendStartedAt, {
1335
- name: "rateless.sendStartSync",
1336
- messages: 1,
1337
- symbols: startSyncSymbols.length,
1338
- targets: properties.targets.length,
1339
- syncId,
1340
- details: { rejected: true },
1999
+ const syncId = getSyncIdString(startSync);
2000
+ const targetSignal = properties.targetLifecycle.controller.signal;
2001
+ const processController = new AbortController();
2002
+ const processSignal = processController.signal;
2003
+ let cleared = false;
2004
+ let lastSeqNo = -1n;
2005
+ let symbolsProduced = startSyncSymbols.length;
2006
+ let symbolBudgetExhausted = false;
2007
+ let simpleFallbackPromise: Promise<void> | undefined;
2008
+ const symbolBudget = getOutgoingRatelessSymbolBudget(
2009
+ properties.coordinates.length,
2010
+ startSyncSymbols.length,
2011
+ );
2012
+ let onTargetAbort: (() => void) | undefined;
2013
+ const clear = (
2014
+ reason: unknown = new Error("rateless outgoing process closed"),
2015
+ ) => {
2016
+ if (cleared) {
2017
+ return;
2018
+ }
2019
+ cleared = true;
2020
+
2021
+ // Timeout/replacement owns only the encoder process. Response leases
2022
+ // retain the original open/caller/target lifecycle independently, so a
2023
+ // response already accepted for delivery can finish after this process
2024
+ // expires while close, caller abort, and disconnect still cancel it.
2025
+ processController.abort(reason);
2026
+ if (onTargetAbort) {
2027
+ targetSignal.removeEventListener("abort", onTargetAbort);
2028
+ }
2029
+ if (obj.timeout) {
2030
+ clearTimeout(obj.timeout);
2031
+ }
2032
+ if (obj.deadlineTimeout) {
2033
+ clearTimeout(obj.deadlineTimeout);
2034
+ }
2035
+ if (this.outgoingSyncProcesses.get(syncId) === obj) {
2036
+ this.outgoingSyncProcesses.delete(syncId);
2037
+ }
2038
+ if (this.outgoingSyncProcessByTarget.get(target) === obj) {
2039
+ this.outgoingSyncProcessByTarget.delete(target);
2040
+ }
2041
+ properties.targetLifecycle.retainedByProcess = false;
2042
+ freeEncoder();
2043
+ this.maybeDisposeRatelessDispatchLifecycle(lifecycle);
2044
+ };
2045
+ const startSimpleFallback = () => {
2046
+ if (!simpleFallbackPromise) {
2047
+ // From this point overlapping ResponseMaybeSync authorization
2048
+ // belongs to Simple. Keeping Rateless first would ship the response
2049
+ // while leaving the duplicate Simple lease charged until its TTL.
2050
+ obj.simpleFallbackStarted = true;
2051
+ simpleFallbackPromise = Promise.resolve(
2052
+ this.simple.onMaybeMissingHashes({
2053
+ hashes: properties.outgoingHashes,
2054
+ targets: [target],
2055
+ signal: lifecycle.callerSignal,
2056
+ }),
2057
+ );
2058
+ }
2059
+ return simpleFallbackPromise;
2060
+ };
2061
+ const fallbackAndClear = (reason: Error) => {
2062
+ void startSimpleFallback().catch((error) => logger.error(error));
2063
+ clear(reason);
2064
+ };
2065
+ const createTimeout = () => {
2066
+ const timeout = setTimeout(
2067
+ () =>
2068
+ fallbackAndClear(new Error("rateless outgoing process timed out")),
2069
+ OUTGOING_RATELESS_IDLE_TIMEOUT_MS,
2070
+ );
2071
+ timeout.unref?.();
2072
+ return timeout;
2073
+ };
2074
+ const createDeadlineTimeout = () => {
2075
+ const timeout = setTimeout(
2076
+ () =>
2077
+ fallbackAndClear(
2078
+ new Error("rateless outgoing process deadline exceeded"),
2079
+ ),
2080
+ RATELESS_PROCESS_ABSOLUTE_TIMEOUT_MS,
2081
+ );
2082
+ timeout.unref?.();
2083
+ return timeout;
2084
+ };
2085
+ onTargetAbort = () => clear(targetSignal.reason);
2086
+
2087
+ // Keep follow-up symbol payloads bounded. Each symbol is serialized as an
2088
+ // object with three bigint fields, so very large batches can dominate heap under
2089
+ // concurrent churn even though the native RIBLT encoder itself is compact.
2090
+ const nextBatch = Math.max(
2091
+ MIN_MORE_SYMBOLS_BATCH_SIZE,
2092
+ Math.min(
2093
+ MAX_MORE_SYMBOLS_BATCH_SIZE,
2094
+ Math.ceil(properties.coordinates.length / 4),
2095
+ ),
2096
+ );
2097
+ obj = {
2098
+ target,
2099
+ targetLifecycle: properties.targetLifecycle,
2100
+ encoder,
2101
+ timeout: undefined,
2102
+ deadlineTimeout: undefined,
2103
+ refresh: () => {
2104
+ if (obj.timeout) {
2105
+ clearTimeout(obj.timeout);
2106
+ }
2107
+ obj.timeout = createTimeout();
2108
+ },
2109
+ next: (message: { lastSeqNo: bigint }) => {
2110
+ if (processSignal.aborted || symbolBudgetExhausted) {
2111
+ return undefined;
2112
+ }
2113
+ if (message.lastSeqNo !== lastSeqNo + 1n) {
2114
+ return undefined;
2115
+ }
2116
+ lastSeqNo = message.lastSeqNo;
2117
+
2118
+ const remainingBudget = symbolBudget - symbolsProduced;
2119
+ if (remainingBudget <= 0) {
2120
+ symbolBudgetExhausted = true;
2121
+ return {
2122
+ symbols: CodedSymbolBatch.fromSymbols([]),
2123
+ exhaustedAfterSend: true,
2124
+ };
2125
+ }
2126
+
2127
+ const produceStartedAt = syncProfileStart(profile);
2128
+ const symbols = produceNextCodedSymbols(
2129
+ encoder,
2130
+ Math.min(nextBatch, remainingBudget),
2131
+ );
2132
+ symbolsProduced += symbols.length;
2133
+ const exhaustedAfterSend = symbols.length === 0;
2134
+ if (exhaustedAfterSend) {
2135
+ symbolBudgetExhausted = true;
2136
+ } else {
2137
+ obj.refresh();
2138
+ }
2139
+ if (profile) {
2140
+ emitSyncProfileDuration(profile, produceStartedAt, {
2141
+ name: "rateless.produceMoreSymbols",
2142
+ syncId,
2143
+ symbols: symbols.length,
2144
+ targets: 1,
2145
+ });
2146
+ }
2147
+ return { symbols, exhaustedAfterSend };
2148
+ },
2149
+ startSimpleFallback,
2150
+ simpleFallbackStarted: false,
2151
+ free: clear,
2152
+ outgoingHashes: properties.outgoingHashes,
2153
+ authorizedHashes: properties.authorizedHashes,
2154
+ consumedResponseHashes: new Set(),
2155
+ processController,
2156
+ signal: processSignal,
2157
+ callerSignal: lifecycle.callerSignal,
2158
+ };
2159
+ processConstructed = true;
2160
+
2161
+ if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
2162
+ clear();
2163
+ return undefined;
2164
+ }
2165
+ // Without a wire process id on ResponseMaybeSync, retaining at most one
2166
+ // process per target keeps sender/hash authorization unambiguous.
2167
+ this.outgoingSyncProcessByTarget
2168
+ .get(target)
2169
+ ?.free(new Error("rateless outgoing process replaced"));
2170
+ if (!this.isRatelessDispatchLifecycleActive(lifecycle, target)) {
2171
+ clear();
2172
+ return undefined;
2173
+ }
2174
+ properties.targetLifecycle.retainedByProcess = true;
2175
+ this.outgoingSyncProcesses.set(syncId, obj);
2176
+ this.outgoingSyncProcessByTarget.set(target, obj);
2177
+ obj.timeout = createTimeout();
2178
+ obj.deadlineTimeout = createDeadlineTimeout();
2179
+ targetSignal.addEventListener("abort", onTargetAbort, { once: true });
2180
+ encoderTransferred = true;
2181
+ if (targetSignal.aborted) {
2182
+ clear(targetSignal.reason);
2183
+ return undefined;
2184
+ }
2185
+ if (profile) {
2186
+ emitSyncProfileEvent(profile, {
2187
+ name: "rateless.dispatchMode",
2188
+ entries: properties.entryCount,
2189
+ symbols: startSyncSymbols.length,
2190
+ targets: 1,
2191
+ syncId,
2192
+ details: { mode: "rateless" },
2193
+ });
2194
+ }
2195
+
2196
+ const sendStartedAt = syncProfileStart(profile);
2197
+ let sendResult: Promise<unknown>;
2198
+ try {
2199
+ sendResult = Promise.resolve(
2200
+ this.simple.rpc.send(startSync, {
2201
+ mode: new SilentDelivery({
2202
+ to: [target],
2203
+ redundancy: 1,
2204
+ }),
2205
+ priority: SYNC_MESSAGE_PRIORITY,
2206
+ signal: processSignal,
1341
2207
  }),
2208
+ );
2209
+ } catch (error) {
2210
+ sendResult = Promise.reject(error);
2211
+ }
2212
+ void sendResult.then(
2213
+ () => {
2214
+ if (profile) {
2215
+ emitSyncProfileDuration(profile, sendStartedAt, {
2216
+ name: "rateless.sendStartSync",
2217
+ messages: 1,
2218
+ symbols: startSyncSymbols.length,
2219
+ targets: 1,
2220
+ syncId,
2221
+ });
2222
+ }
2223
+ },
2224
+ () => {
2225
+ clear();
2226
+ if (profile) {
2227
+ emitSyncProfileDuration(profile, sendStartedAt, {
2228
+ name: "rateless.sendStartSync",
2229
+ messages: 1,
2230
+ symbols: startSyncSymbols.length,
2231
+ targets: 1,
2232
+ syncId,
2233
+ details: {
2234
+ rejected: true,
2235
+ cancelled: processSignal.aborted || undefined,
2236
+ },
2237
+ });
2238
+ }
2239
+ },
1342
2240
  );
2241
+ if (processSignal.aborted) {
2242
+ clear();
2243
+ return undefined;
2244
+ }
2245
+ return startSyncSymbols.length;
2246
+ } catch (error) {
2247
+ if (processConstructed) {
2248
+ obj.free(error);
2249
+ }
2250
+ throw error;
2251
+ } finally {
2252
+ if (!encoderTransferred) {
2253
+ freeEncoder();
2254
+ }
1343
2255
  }
1344
- if (profile) {
1345
- emitSyncProfileDuration(profile, startedAt, {
1346
- name: "rateless.onMaybeMissingEntries",
1347
- entries: properties.entries.size,
1348
- messages: 1,
1349
- symbols: startSyncSymbols.length,
1350
- targets: properties.targets.length,
1351
- details: {
1352
- mode: "rateless",
1353
- ibltEntries: allCoordinatesToSyncWithIblt.length,
1354
- naiveEntries: naiveHashes.length,
1355
- },
1356
- });
2256
+ }
2257
+
2258
+ private consumeAuthorizedRatelessResponse(
2259
+ hashes: Iterable<string>,
2260
+ from: PublicSignKey,
2261
+ ): AuthorizedRatelessResponseLease | undefined {
2262
+ const target = from.hashcode();
2263
+ const process = this.outgoingSyncProcessByTarget.get(target);
2264
+ if (
2265
+ !process ||
2266
+ process.signal.aborted ||
2267
+ !this.isRatelessDispatchLifecycleActive(
2268
+ process.targetLifecycle.lifecycle,
2269
+ target,
2270
+ )
2271
+ ) {
2272
+ return undefined;
2273
+ }
2274
+ if (
2275
+ this.activeRatelessResponseCount >=
2276
+ MAX_ACTIVE_RATELESS_RESPONSES_GLOBAL ||
2277
+ (this.activeRatelessResponseCountByPeer.get(target) ?? 0) >=
2278
+ MAX_ACTIVE_RATELESS_RESPONSES_PER_PEER
2279
+ ) {
2280
+ return undefined;
1357
2281
  }
2282
+ const authorized: string[] = [];
2283
+ const remaining: string[] = [];
2284
+ const seen = new Set<string>();
2285
+ for (const hash of hashes) {
2286
+ if (seen.has(hash)) {
2287
+ continue;
2288
+ }
2289
+ seen.add(hash);
2290
+ if (
2291
+ !process.authorizedHashes.has(hash) ||
2292
+ process.consumedResponseHashes.has(hash)
2293
+ ) {
2294
+ remaining.push(hash);
2295
+ continue;
2296
+ }
2297
+ process.consumedResponseHashes.add(hash);
2298
+ authorized.push(hash);
2299
+ }
2300
+ if (authorized.length === 0) {
2301
+ return {
2302
+ process,
2303
+ authorized,
2304
+ remaining,
2305
+ signal: process.targetLifecycle.controller.signal,
2306
+ release: () => {},
2307
+ };
2308
+ }
2309
+
2310
+ const targetLifecycle = process.targetLifecycle;
2311
+ targetLifecycle.responseLeases += 1;
2312
+ this.activeRatelessResponseCount += 1;
2313
+ this.activeRatelessResponseCountByPeer.set(
2314
+ target,
2315
+ (this.activeRatelessResponseCountByPeer.get(target) ?? 0) + 1,
2316
+ );
2317
+ let released = false;
2318
+ return {
2319
+ process,
2320
+ authorized,
2321
+ remaining,
2322
+ signal: targetLifecycle.controller.signal,
2323
+ release: (options) => {
2324
+ if (released) {
2325
+ return;
2326
+ }
2327
+ released = true;
2328
+ if (options?.rollback) {
2329
+ for (const hash of authorized) {
2330
+ process.consumedResponseHashes.delete(hash);
2331
+ }
2332
+ }
2333
+ targetLifecycle.responseLeases -= 1;
2334
+ this.activeRatelessResponseCount -= 1;
2335
+ const activeForPeer =
2336
+ (this.activeRatelessResponseCountByPeer.get(target) ?? 1) - 1;
2337
+ if (activeForPeer === 0) {
2338
+ this.activeRatelessResponseCountByPeer.delete(target);
2339
+ } else {
2340
+ this.activeRatelessResponseCountByPeer.set(target, activeForPeer);
2341
+ }
2342
+ this.maybeDisposeRatelessDispatchLifecycle(targetLifecycle.lifecycle);
2343
+ },
2344
+ };
1358
2345
  }
1359
2346
 
1360
2347
  async onMessage(
@@ -1363,226 +2350,550 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
1363
2350
  ): Promise<boolean> {
1364
2351
  const profile = this.properties.sync?.profile;
1365
2352
  if (message instanceof StartSync) {
2353
+ const from = context.from;
2354
+ const ownershipLifecycleController =
2355
+ this.ratelessDispatchLifecycleController;
2356
+ if (
2357
+ !from ||
2358
+ !this.isIncomingSyncGenerationActive(ownershipLifecycleController)
2359
+ ) {
2360
+ return true;
2361
+ }
2362
+ const sender = from.hashcode();
1366
2363
  const syncId = getSyncIdString(message);
1367
- if (this.ingoingSyncProcesses.has(syncId)) {
2364
+ const key = this.getIncomingSyncProcessKey(sender, syncId);
2365
+ const completedSynchronizations = this.startedOrCompletedSynchronizations;
2366
+ const admissionSet = this.incomingRatelessProcessAdmissions;
2367
+ if (this.ingoingSyncProcesses.has(key)) {
1368
2368
  return true;
1369
2369
  }
1370
2370
 
1371
- if (this.startedOrCompletedSynchronizations.has(syncId)) {
2371
+ if (completedSynchronizations.has(key)) {
1372
2372
  return true;
1373
2373
  }
1374
2374
 
1375
- this.startedOrCompletedSynchronizations.add(syncId);
2375
+ let admissionsForSender = 0;
2376
+ for (const admission of admissionSet) {
2377
+ if (admission.key === key) {
2378
+ return true;
2379
+ }
2380
+ if (admission.sender === sender) {
2381
+ admissionsForSender += 1;
2382
+ }
2383
+ }
2384
+ if (
2385
+ admissionSet.size >= MAX_INCOMING_RATELESS_PROCESSES ||
2386
+ admissionsForSender >= MAX_INCOMING_RATELESS_PROCESSES_PER_SENDER
2387
+ ) {
2388
+ return true;
2389
+ }
2390
+ const admission = { key, sender };
2391
+ admissionSet.add(admission);
2392
+
2393
+ const controller = new AbortController();
2394
+ let freed = false;
2395
+ let completed = false;
2396
+ let initializationPending = false;
2397
+ let fallbackSendPending = false;
2398
+ let admissionReleased = false;
2399
+ let fallbackGraceTimeout: ReturnType<typeof setTimeout> | undefined;
2400
+ let onOwnershipAbort: (() => void) | undefined;
2401
+ let obj!: IncomingRatelessSyncProcess;
2402
+ const releaseAdmissionIfSettled = () => {
2403
+ if (
2404
+ admissionReleased ||
2405
+ !freed ||
2406
+ initializationPending ||
2407
+ fallbackSendPending
2408
+ ) {
2409
+ return;
2410
+ }
2411
+ admissionReleased = true;
2412
+ admissionSet.delete(admission);
2413
+ };
2414
+ const free = (
2415
+ reason: unknown = new Error("incoming rateless process closed"),
2416
+ ) => {
2417
+ if (freed) {
2418
+ return;
2419
+ }
2420
+ freed = true;
2421
+ // Abort first: no queued process/send continuation may observe the
2422
+ // object detached while its native decoder is still usable.
2423
+ controller.abort(reason);
2424
+ if (onOwnershipAbort) {
2425
+ ownershipLifecycleController.signal.removeEventListener(
2426
+ "abort",
2427
+ onOwnershipAbort,
2428
+ );
2429
+ }
2430
+ if (obj.timeout) {
2431
+ clearTimeout(obj.timeout);
2432
+ obj.timeout = undefined;
2433
+ }
2434
+ if (obj.deadlineTimeout) {
2435
+ clearTimeout(obj.deadlineTimeout);
2436
+ obj.deadlineTimeout = undefined;
2437
+ }
2438
+ if (fallbackGraceTimeout) {
2439
+ clearTimeout(fallbackGraceTimeout);
2440
+ fallbackGraceTimeout = undefined;
2441
+ }
2442
+ if (this.ingoingSyncProcesses.get(key) === obj) {
2443
+ this.ingoingSyncProcesses.delete(key);
2444
+ }
2445
+ const ownedDecoder = obj.decoder;
2446
+ obj.decoder = undefined;
2447
+ ownedDecoder?.free();
2448
+ releaseAdmissionIfSettled();
2449
+ };
2450
+ const complete = () => {
2451
+ if (completed || !this.isIncomingSyncProcessActive(obj)) {
2452
+ return;
2453
+ }
2454
+ completed = true;
2455
+ completedSynchronizations.add(key);
2456
+ free(new Error("incoming rateless process completed"));
2457
+ };
2458
+ obj = {
2459
+ key,
2460
+ syncId,
2461
+ sender,
2462
+ from,
2463
+ ownershipLifecycleController,
2464
+ controller,
2465
+ completedSynchronizations,
2466
+ timeout: undefined,
2467
+ deadlineTimeout: undefined,
2468
+ refresh: () => {},
2469
+ lastSeqNo: -1n,
2470
+ processedSymbols: 0,
2471
+ queuedSymbols: 0,
2472
+ symbolBudget: getIncomingRatelessSymbolBudget(message.symbols.length),
2473
+ process: async () => undefined,
2474
+ requestAll: async () => {},
2475
+ fallbackToSimple: async () => {},
2476
+ free,
2477
+ complete,
2478
+ };
2479
+ this.ingoingSyncProcesses.set(key, obj);
2480
+ onOwnershipAbort = () => free(ownershipLifecycleController.signal.reason);
2481
+ ownershipLifecycleController.signal.addEventListener(
2482
+ "abort",
2483
+ onOwnershipAbort,
2484
+ { once: true },
2485
+ );
2486
+ obj.deadlineTimeout = setTimeout(() => {
2487
+ void obj.fallbackToSimple(
2488
+ new Error("incoming rateless process deadline exceeded"),
2489
+ );
2490
+ }, RATELESS_PROCESS_ABSOLUTE_TIMEOUT_MS);
2491
+ obj.deadlineTimeout.unref?.();
2492
+ if (!this.isIncomingSyncProcessActive(obj)) {
2493
+ free(ownershipLifecycleController.signal.reason);
2494
+ return true;
2495
+ }
1376
2496
 
1377
- const wrapped = message.end < message.start;
1378
- const decoderStartedAt = syncProfileStart(profile);
1379
- const decoder = await this.getLocalDecoderForRange({
1380
- start1: message.start,
1381
- end1: wrapped ? this.properties.numbers.maxValue : message.end,
1382
- start2: 0n,
1383
- end2: wrapped ? message.end : 0n,
1384
- });
1385
- if (profile) {
1386
- emitSyncProfileDuration(profile, decoderStartedAt, {
1387
- name: "rateless.getLocalDecoderForRange",
1388
- syncId,
1389
- details: { wrapped, found: decoder !== false },
2497
+ let requestAllPromise: Promise<void> | undefined;
2498
+ obj.requestAll = () => {
2499
+ if (requestAllPromise) {
2500
+ return requestAllPromise;
2501
+ }
2502
+ requestAllPromise = (async () => {
2503
+ if (!this.isIncomingSyncProcessActive(obj)) {
2504
+ return;
2505
+ }
2506
+ fallbackSendPending = true;
2507
+ const sendStartedAt = syncProfileStart(profile);
2508
+ try {
2509
+ await this.simple.rpc.send(
2510
+ new RequestAll({
2511
+ syncId: message.syncId,
2512
+ }),
2513
+ {
2514
+ mode: new SilentDelivery({
2515
+ to: [obj.sender],
2516
+ redundancy: 1,
2517
+ }),
2518
+ priority: SYNC_MESSAGE_PRIORITY,
2519
+ signal: controller.signal,
2520
+ },
2521
+ );
2522
+ if (this.isIncomingSyncProcessActive(obj) && profile) {
2523
+ emitSyncProfileDuration(profile, sendStartedAt, {
2524
+ name: "rateless.sendRequestAll",
2525
+ messages: 1,
2526
+ targets: 1,
2527
+ syncId,
2528
+ });
2529
+ }
2530
+ } finally {
2531
+ fallbackSendPending = false;
2532
+ if (this.isIncomingSyncProcessActive(obj)) {
2533
+ complete();
2534
+ }
2535
+ releaseAdmissionIfSettled();
2536
+ }
2537
+ })();
2538
+ return requestAllPromise;
2539
+ };
2540
+ let boundedFallbackPromise: Promise<void> | undefined;
2541
+ obj.fallbackToSimple = (
2542
+ reason: unknown = new Error("incoming rateless fallback timed out"),
2543
+ ) => {
2544
+ if (boundedFallbackPromise) {
2545
+ return boundedFallbackPromise;
2546
+ }
2547
+ const requestAll = obj.requestAll();
2548
+ boundedFallbackPromise = new Promise<void>((resolve) => {
2549
+ let settled = false;
2550
+ const settle = () => {
2551
+ if (settled) {
2552
+ return;
2553
+ }
2554
+ settled = true;
2555
+ controller.signal.removeEventListener("abort", onAbort);
2556
+ if (fallbackGraceTimeout) {
2557
+ clearTimeout(fallbackGraceTimeout);
2558
+ fallbackGraceTimeout = undefined;
2559
+ }
2560
+ resolve();
2561
+ };
2562
+ const onAbort = () => settle();
2563
+ controller.signal.addEventListener("abort", onAbort, {
2564
+ once: true,
2565
+ });
2566
+ fallbackGraceTimeout = setTimeout(() => {
2567
+ free(reason);
2568
+ settle();
2569
+ }, INCOMING_RATELESS_FALLBACK_GRACE_MS);
2570
+ fallbackGraceTimeout.unref?.();
2571
+ void requestAll.then(settle, settle);
2572
+ if (controller.signal.aborted) {
2573
+ onAbort();
2574
+ }
1390
2575
  });
2576
+ return boundedFallbackPromise;
2577
+ };
2578
+
2579
+ if (message.symbols.length > MAX_MORE_SYMBOLS_BATCH_SIZE) {
2580
+ await obj.fallbackToSimple(
2581
+ new Error("oversized incoming rateless initial batch"),
2582
+ );
2583
+ return true;
1391
2584
  }
1392
2585
 
1393
- if (!decoder) {
1394
- const sendStartedAt = syncProfileStart(profile);
1395
- await this.simple.rpc.send(
1396
- new RequestAll({
1397
- syncId: message.syncId,
1398
- }),
2586
+ const wrapped = message.end < message.start;
2587
+ const decoderStartedAt = syncProfileStart(profile);
2588
+ let decoder: DecoderWrapper | false;
2589
+ initializationPending = true;
2590
+ try {
2591
+ decoder = await this.getLocalDecoderForRange(
1399
2592
  {
1400
- mode: new SilentDelivery({ to: [context.from!], redundancy: 1 }),
1401
- priority: SYNC_MESSAGE_PRIORITY,
2593
+ start1: message.start,
2594
+ end1: wrapped ? this.properties.numbers.maxValue : message.end,
2595
+ start2: 0n,
2596
+ end2: wrapped ? message.end : 0n,
2597
+ },
2598
+ {
2599
+ ownershipLifecycleController,
2600
+ signal: controller.signal,
1402
2601
  },
1403
2602
  );
2603
+ } catch (error) {
2604
+ const processAborted = controller.signal.aborted;
2605
+ free(error);
2606
+ if (
2607
+ processAborted ||
2608
+ !this.isIncomingSyncGenerationActive(ownershipLifecycleController)
2609
+ ) {
2610
+ return true;
2611
+ }
2612
+ throw error;
2613
+ } finally {
2614
+ initializationPending = false;
2615
+ releaseAdmissionIfSettled();
2616
+ }
2617
+ if (!this.isIncomingSyncProcessActive(obj)) {
2618
+ decoder && decoder.free();
2619
+ free(controller.signal.reason);
2620
+ return true;
2621
+ }
2622
+ if (decoder) {
2623
+ // Transfer the native decoder before invoking diagnostics. A profile
2624
+ // sink is caller code and may throw; from this point process cleanup
2625
+ // owns the decoder on every exit.
2626
+ obj.decoder = decoder;
2627
+ }
2628
+ try {
1404
2629
  if (profile) {
1405
- emitSyncProfileDuration(profile, sendStartedAt, {
1406
- name: "rateless.sendRequestAll",
1407
- messages: 1,
1408
- targets: 1,
2630
+ emitSyncProfileDuration(profile, decoderStartedAt, {
2631
+ name: "rateless.getLocalDecoderForRange",
1409
2632
  syncId,
2633
+ details: { wrapped, found: decoder !== false },
1410
2634
  });
1411
2635
  }
2636
+ } catch (error) {
2637
+ free(error);
2638
+ throw error;
2639
+ }
2640
+
2641
+ if (!decoder) {
2642
+ try {
2643
+ await obj.fallbackToSimple(
2644
+ new Error("incoming rateless decoder unavailable"),
2645
+ );
2646
+ } catch (error) {
2647
+ if (controller.signal.aborted) {
2648
+ return true;
2649
+ }
2650
+ throw error;
2651
+ }
1412
2652
  return true;
1413
2653
  }
1414
2654
 
1415
2655
  const createTimeout = () => {
1416
- return setTimeout(() => {
1417
- decoder.free();
1418
- this.ingoingSyncProcesses.delete(syncId);
1419
- }, 2e4); // TODO arg
2656
+ const timeout = setTimeout(() => {
2657
+ void obj.fallbackToSimple(
2658
+ new Error("incoming rateless process timed out"),
2659
+ );
2660
+ }, INCOMING_RATELESS_IDLE_TIMEOUT_MS);
2661
+ timeout.unref?.();
2662
+ return timeout;
1420
2663
  };
1421
2664
 
1422
2665
  let messageQueue: {
1423
2666
  seqNo: bigint;
1424
2667
  symbols: CodedSymbolInput;
2668
+ symbolCount: number;
1425
2669
  }[] = [];
1426
- let lastSeqNo = -1n;
1427
- const obj = {
1428
- decoder,
1429
- timeout: createTimeout(),
1430
- refresh: () => {
1431
- let prevTimeout = obj.timeout;
1432
- if (prevTimeout) {
1433
- clearTimeout(prevTimeout);
2670
+ obj.refresh = () => {
2671
+ if (!this.isIncomingSyncProcessActive(obj)) {
2672
+ return;
2673
+ }
2674
+ if (obj.timeout) {
2675
+ clearTimeout(obj.timeout);
2676
+ }
2677
+ obj.timeout = createTimeout();
2678
+ };
2679
+ obj.process = async (newMessage: {
2680
+ seqNo: bigint;
2681
+ symbols: CodedSymbolInput;
2682
+ }): Promise<IncomingRatelessProcessResult> => {
2683
+ if (!this.isIncomingSyncProcessActive(obj)) {
2684
+ return undefined;
2685
+ }
2686
+
2687
+ const symbolCount = CodedSymbolBatch.from(newMessage.symbols).length;
2688
+ if (symbolCount > MAX_MORE_SYMBOLS_BATCH_SIZE) {
2689
+ free(new Error("incoming rateless symbol batch exceeds limit"));
2690
+ return undefined;
2691
+ }
2692
+ if (newMessage.seqNo <= obj.lastSeqNo) {
2693
+ return undefined;
2694
+ }
2695
+ if (
2696
+ newMessage.seqNo >
2697
+ obj.lastSeqNo + MAX_INCOMING_RATELESS_SEQUENCE_GAP
2698
+ ) {
2699
+ return undefined;
2700
+ }
2701
+ if (messageQueue.some((queued) => queued.seqNo === newMessage.seqNo)) {
2702
+ return undefined;
2703
+ }
2704
+ if (
2705
+ newMessage.seqNo !== obj.lastSeqNo + 1n &&
2706
+ messageQueue.length >= MAX_INCOMING_RATELESS_QUEUED_BATCHES
2707
+ ) {
2708
+ return undefined;
2709
+ }
2710
+ if (
2711
+ obj.processedSymbols + obj.queuedSymbols + symbolCount >
2712
+ obj.symbolBudget
2713
+ ) {
2714
+ return "fallback-to-simple";
2715
+ }
2716
+ if (newMessage.seqNo > 0n && symbolCount === 0) {
2717
+ return "fallback-to-simple";
2718
+ }
2719
+
2720
+ messageQueue.push({ ...newMessage, symbolCount });
2721
+ obj.queuedSymbols += symbolCount;
2722
+ messageQueue.sort((a, b) => Number(a.seqNo - b.seqNo));
2723
+ if (messageQueue[0].seqNo !== obj.lastSeqNo + 1n) {
2724
+ return;
2725
+ }
2726
+
2727
+ const finalizeIfDecoded = (): boolean => {
2728
+ if (!this.isIncomingSyncProcessActive(obj)) {
2729
+ return true;
2730
+ }
2731
+ if (!decoder.decoded()) {
2732
+ return false;
1434
2733
  }
1435
- obj.timeout = createTimeout();
1436
- },
1437
- process: async (newMessage: {
1438
- seqNo: bigint;
1439
- symbols: CodedSymbolInput;
1440
- }): Promise<boolean | undefined> => {
1441
- obj.refresh(); // TODO use timestamp instead and collective pruning/refresh
1442
2734
 
1443
- if (newMessage.seqNo <= lastSeqNo) {
1444
- return undefined;
2735
+ const remoteStartedAt = syncProfileStart(profile);
2736
+ const allMissingSymbolsInRemote = getRemoteSymbolValues(decoder);
2737
+ if (profile) {
2738
+ emitSyncProfileDuration(profile, remoteStartedAt, {
2739
+ name: "rateless.remoteSymbols",
2740
+ entries: allMissingSymbolsInRemote.length,
2741
+ symbols: allMissingSymbolsInRemote.length,
2742
+ syncId,
2743
+ });
1445
2744
  }
1446
2745
 
1447
- messageQueue.push(newMessage);
1448
- messageQueue.sort((a, b) => Number(a.seqNo - b.seqNo));
1449
- if (messageQueue[0].seqNo !== lastSeqNo + 1n) {
1450
- return;
2746
+ // The IBLT decoder is based on a local snapshot. Entries can arrive via
2747
+ // overlapping repair before we issue the follow-up simple request, so
2748
+ // re-check local presence to avoid stale duplicate bounce-back.
2749
+ void this.simple
2750
+ .queueSync(allMissingSymbolsInRemote, obj.from)
2751
+ .catch((error) => logger.error(error));
2752
+ obj.complete();
2753
+ return true;
2754
+ };
2755
+
2756
+ while (
2757
+ messageQueue.length > 0 &&
2758
+ messageQueue[0].seqNo === obj.lastSeqNo + 1n
2759
+ ) {
2760
+ const symbolMessage = messageQueue.shift();
2761
+ if (!symbolMessage) {
2762
+ break;
1451
2763
  }
1452
2764
 
1453
- const finalizeIfDecoded = (): boolean => {
1454
- if (!decoder.decoded()) {
1455
- return false;
2765
+ obj.queuedSymbols -= symbolMessage.symbolCount;
2766
+ obj.lastSeqNo = symbolMessage.seqNo;
2767
+ obj.processedSymbols += symbolMessage.symbolCount;
2768
+ // Only an authenticated, previously unseen contiguous sequence advances
2769
+ // the idle deadline. Replays and speculative future batches do not.
2770
+ obj.refresh();
2771
+
2772
+ const addBatchAndDecode:
2773
+ | ((symbols: BigUint64Array) => boolean)
2774
+ | undefined = (decoder as BatchDecoderWrapper)
2775
+ .add_coded_symbols_and_try_decode;
2776
+ if (typeof BigUint64Array !== "undefined" && addBatchAndDecode) {
2777
+ const flatStartedAt = syncProfileStart(profile);
2778
+ const flatSymbols = flatFromCodedSymbols(symbolMessage.symbols);
2779
+ if (profile) {
2780
+ emitSyncProfileDuration(profile, flatStartedAt, {
2781
+ name: "rateless.symbolBatchToFlat",
2782
+ symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
2783
+ syncId,
2784
+ });
1456
2785
  }
1457
2786
 
1458
- const remoteStartedAt = syncProfileStart(profile);
1459
- const allMissingSymbolsInRemote = getRemoteSymbolValues(decoder);
2787
+ const decodeStartedAt = syncProfileStart(profile);
2788
+ const decoded = addBatchAndDecode.call(decoder, flatSymbols);
1460
2789
  if (profile) {
1461
- emitSyncProfileDuration(profile, remoteStartedAt, {
1462
- name: "rateless.remoteSymbols",
1463
- entries: allMissingSymbolsInRemote.length,
1464
- symbols: allMissingSymbolsInRemote.length,
2790
+ emitSyncProfileDuration(profile, decodeStartedAt, {
2791
+ name: "rateless.decodeBatch",
2792
+ symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
1465
2793
  syncId,
2794
+ details: { decoded },
1466
2795
  });
1467
2796
  }
1468
-
1469
- // The IBLT decoder is based on a local snapshot. Entries can arrive via
1470
- // overlapping repair before we issue the follow-up simple request, so
1471
- // re-check local presence to avoid stale duplicate bounce-back.
1472
- this.simple.queueSync(allMissingSymbolsInRemote, context.from!);
1473
- obj.free();
1474
- return true;
1475
- };
1476
-
1477
- while (
1478
- messageQueue.length > 0 &&
1479
- messageQueue[0].seqNo === lastSeqNo + 1n
1480
- ) {
1481
- const symbolMessage = messageQueue.shift();
1482
- if (!symbolMessage) {
1483
- break;
2797
+ if (decoded && finalizeIfDecoded()) {
2798
+ return true;
1484
2799
  }
2800
+ continue;
2801
+ }
1485
2802
 
1486
- lastSeqNo = symbolMessage.seqNo;
1487
-
1488
- const addBatchAndDecode:
1489
- | ((symbols: BigUint64Array) => boolean)
1490
- | undefined = (decoder as BatchDecoderWrapper)
1491
- .add_coded_symbols_and_try_decode;
1492
- if (typeof BigUint64Array !== "undefined" && addBatchAndDecode) {
1493
- const flatStartedAt = syncProfileStart(profile);
1494
- const flatSymbols = flatFromCodedSymbols(symbolMessage.symbols);
1495
- if (profile) {
1496
- emitSyncProfileDuration(profile, flatStartedAt, {
1497
- name: "rateless.symbolBatchToFlat",
1498
- symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
1499
- syncId,
1500
- });
1501
- }
1502
-
1503
- const decodeStartedAt = syncProfileStart(profile);
1504
- const decoded = addBatchAndDecode.call(decoder, flatSymbols);
1505
- if (profile) {
1506
- emitSyncProfileDuration(profile, decodeStartedAt, {
1507
- name: "rateless.decodeBatch",
1508
- symbols: flatSymbols.length / CODED_SYMBOL_WORDS,
1509
- syncId,
1510
- details: { decoded },
1511
- });
1512
- }
1513
- if (decoded && finalizeIfDecoded()) {
2803
+ const decodeLoopStartedAt = syncProfileStart(profile);
2804
+ let symbolsProcessed = 0;
2805
+ for (const symbol of CodedSymbolBatch.from(symbolMessage.symbols)) {
2806
+ symbolsProcessed += 1;
2807
+ const normalizedSymbol =
2808
+ symbol instanceof SymbolSerialized
2809
+ ? symbol
2810
+ : new SymbolSerialized({
2811
+ count: symbol.count,
2812
+ hash: symbol.hash,
2813
+ symbol: symbol.symbol,
2814
+ });
2815
+
2816
+ decoder.add_coded_symbol(normalizedSymbol);
2817
+ try {
2818
+ decoder.try_decode();
2819
+ if (finalizeIfDecoded()) {
1514
2820
  return true;
1515
2821
  }
1516
- continue;
1517
- }
1518
-
1519
- const decodeLoopStartedAt = syncProfileStart(profile);
1520
- let symbolsProcessed = 0;
1521
- for (const symbol of CodedSymbolBatch.from(symbolMessage.symbols)) {
1522
- symbolsProcessed += 1;
1523
- const normalizedSymbol =
1524
- symbol instanceof SymbolSerialized
1525
- ? symbol
1526
- : new SymbolSerialized({
1527
- count: symbol.count,
1528
- hash: symbol.hash,
1529
- symbol: symbol.symbol,
1530
- });
1531
-
1532
- decoder.add_coded_symbol(normalizedSymbol);
1533
- try {
1534
- decoder.try_decode();
1535
- if (finalizeIfDecoded()) {
1536
- return true;
1537
- }
1538
- } catch (error: any) {
1539
- if (
1540
- error?.message === "Invalid degree" ||
1541
- error === "Invalid degree"
1542
- ) {
1543
- logger.trace(
1544
- "Decoder reported invalid degree; waiting for more symbols",
1545
- );
1546
- continue;
1547
- }
1548
- throw error;
2822
+ } catch (error: any) {
2823
+ if (
2824
+ error?.message === "Invalid degree" ||
2825
+ error === "Invalid degree"
2826
+ ) {
2827
+ logger.trace(
2828
+ "Decoder reported invalid degree; waiting for more symbols",
2829
+ );
2830
+ continue;
1549
2831
  }
1550
- }
1551
- if (profile) {
1552
- emitSyncProfileDuration(profile, decodeLoopStartedAt, {
1553
- name: "rateless.decodeSymbolLoop",
1554
- symbols: symbolsProcessed,
1555
- syncId,
1556
- });
2832
+ throw error;
1557
2833
  }
1558
2834
  }
1559
- return false;
1560
- },
1561
- free: () => {
1562
- decoder.free();
1563
- clearTimeout(this.ingoingSyncProcesses.get(syncId)?.timeout);
1564
- this.ingoingSyncProcesses.delete(syncId);
1565
- },
2835
+ if (profile) {
2836
+ emitSyncProfileDuration(profile, decodeLoopStartedAt, {
2837
+ name: "rateless.decodeSymbolLoop",
2838
+ symbols: symbolsProcessed,
2839
+ syncId,
2840
+ });
2841
+ }
2842
+ }
2843
+ return false;
1566
2844
  };
1567
-
1568
- this.ingoingSyncProcesses.set(syncId, obj);
1569
-
1570
- if (await obj.process({ seqNo: 0n, symbols: message.symbols })) {
2845
+ obj.timeout = createTimeout();
2846
+ let initialResult: IncomingRatelessProcessResult;
2847
+ try {
2848
+ initialResult = await obj.process({
2849
+ seqNo: 0n,
2850
+ symbols: message.symbols,
2851
+ });
2852
+ } catch (error) {
2853
+ const wasActive = this.isIncomingSyncProcessActive(obj);
2854
+ free(error);
2855
+ if (!wasActive) {
2856
+ return true;
2857
+ }
2858
+ throw error;
2859
+ }
2860
+ if (initialResult === true) {
2861
+ return true;
2862
+ }
2863
+ if (initialResult === "fallback-to-simple") {
2864
+ await obj.fallbackToSimple(
2865
+ new Error("incoming rateless symbol budget exhausted"),
2866
+ );
2867
+ return true;
2868
+ }
2869
+ if (!this.isIncomingSyncProcessActive(obj)) {
1571
2870
  return true;
1572
2871
  }
1573
2872
 
1574
2873
  // not done, request more symbols
1575
2874
  const sendStartedAt = syncProfileStart(profile);
1576
- await this.simple.rpc.send(
1577
- new RequestMoreSymbols({
1578
- lastSeqNo: 0n,
1579
- syncId: message.syncId,
1580
- }),
1581
- {
1582
- mode: new SilentDelivery({ to: [context.from!], redundancy: 1 }),
1583
- priority: SYNC_MESSAGE_PRIORITY,
1584
- },
1585
- );
2875
+ try {
2876
+ await this.simple.rpc.send(
2877
+ new RequestMoreSymbols({
2878
+ lastSeqNo: obj.lastSeqNo,
2879
+ syncId: message.syncId,
2880
+ }),
2881
+ {
2882
+ mode: new SilentDelivery({ to: [obj.sender], redundancy: 1 }),
2883
+ priority: SYNC_MESSAGE_PRIORITY,
2884
+ signal: controller.signal,
2885
+ },
2886
+ );
2887
+ } catch (error) {
2888
+ if (!this.isIncomingSyncProcessActive(obj)) {
2889
+ return true;
2890
+ }
2891
+ free(error);
2892
+ throw error;
2893
+ }
2894
+ if (!this.isIncomingSyncProcessActive(obj)) {
2895
+ return true;
2896
+ }
1586
2897
  if (profile) {
1587
2898
  emitSyncProfileDuration(profile, sendStartedAt, {
1588
2899
  name: "rateless.sendRequestMoreSymbols",
@@ -1594,50 +2905,89 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
1594
2905
 
1595
2906
  return true;
1596
2907
  } else if (message instanceof MoreSymbols) {
2908
+ const from = context.from;
2909
+ if (!from) {
2910
+ return true;
2911
+ }
2912
+ const sender = from.hashcode();
1597
2913
  const syncId = getSyncIdString(message);
1598
- const obj = this.ingoingSyncProcesses.get(syncId);
1599
- if (!obj) {
2914
+ const key = this.getIncomingSyncProcessKey(sender, syncId);
2915
+ const obj = this.ingoingSyncProcesses.get(key);
2916
+ if (
2917
+ !obj ||
2918
+ obj.sender !== sender ||
2919
+ !this.isIncomingSyncProcessActive(obj)
2920
+ ) {
1600
2921
  return true;
1601
2922
  }
1602
- const outProcess = await obj.process(message);
2923
+ let outProcess: IncomingRatelessProcessResult;
2924
+ try {
2925
+ outProcess = await obj.process(message);
2926
+ } catch (error) {
2927
+ const wasActive = this.isIncomingSyncProcessActive(obj);
2928
+ obj.free(error);
2929
+ if (!wasActive) {
2930
+ return true;
2931
+ }
2932
+ throw error;
2933
+ }
1603
2934
 
1604
2935
  if (outProcess === true) {
1605
2936
  return true;
2937
+ } else if (outProcess === "fallback-to-simple") {
2938
+ try {
2939
+ await obj.fallbackToSimple(
2940
+ new Error("incoming rateless symbol budget exhausted"),
2941
+ );
2942
+ } catch {
2943
+ // The bounded rateless process is already complete or aborted. A
2944
+ // later repair round may retry if the fallback request was lost.
2945
+ }
2946
+ return true;
1606
2947
  } else if (outProcess === undefined) {
1607
2948
  return true; // we don't have enough information, or received information that is redundant
1608
2949
  }
2950
+ if (!this.isIncomingSyncProcessActive(obj)) {
2951
+ return true;
2952
+ }
1609
2953
 
1610
2954
  // we are not done
1611
2955
 
1612
2956
  const sendStartedAt = syncProfileStart(profile);
1613
- const sendResult = this.simple.rpc.send(
1614
- new RequestMoreSymbols({
1615
- lastSeqNo: message.seqNo,
1616
- syncId: message.syncId,
1617
- }),
1618
- {
1619
- mode: new SilentDelivery({ to: [context.from!], redundancy: 1 }),
1620
- priority: SYNC_MESSAGE_PRIORITY,
1621
- },
1622
- );
1623
- if (profile) {
1624
- void Promise.resolve(sendResult).then(
1625
- () =>
1626
- emitSyncProfileDuration(profile, sendStartedAt, {
1627
- name: "rateless.sendRequestMoreSymbols",
1628
- messages: 1,
1629
- targets: 1,
1630
- syncId,
1631
- }),
1632
- () =>
1633
- emitSyncProfileDuration(profile, sendStartedAt, {
1634
- name: "rateless.sendRequestMoreSymbols",
1635
- messages: 1,
1636
- targets: 1,
1637
- syncId,
1638
- details: { rejected: true },
1639
- }),
2957
+ try {
2958
+ await this.simple.rpc.send(
2959
+ new RequestMoreSymbols({
2960
+ lastSeqNo: obj.lastSeqNo,
2961
+ syncId: message.syncId,
2962
+ }),
2963
+ {
2964
+ mode: new SilentDelivery({ to: [obj.sender], redundancy: 1 }),
2965
+ priority: SYNC_MESSAGE_PRIORITY,
2966
+ signal: obj.controller.signal,
2967
+ },
1640
2968
  );
2969
+ } catch {
2970
+ if (profile) {
2971
+ emitSyncProfileDuration(profile, sendStartedAt, {
2972
+ name: "rateless.sendRequestMoreSymbols",
2973
+ messages: 1,
2974
+ targets: 1,
2975
+ syncId,
2976
+ details: { rejected: true },
2977
+ });
2978
+ }
2979
+ return true;
2980
+ }
2981
+ if (!this.isIncomingSyncProcessActive(obj)) {
2982
+ return true;
2983
+ }
2984
+ if (profile) {
2985
+ emitSyncProfileDuration(profile, sendStartedAt, {
2986
+ name: "rateless.sendRequestMoreSymbols",
2987
+ messages: 1,
2988
+ targets: 1,
2989
+ syncId,
2990
+ });
1641
2991
  }
1642
2992
 
1643
2993
  return true;
@@ -1647,19 +2997,41 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
1647
2997
  if (!obj) {
1648
2998
  return true;
1649
2999
  }
1650
- const symbols = obj.next(message);
3000
+ if (context.from?.hashcode() !== obj.target) {
3001
+ return true;
3002
+ }
3003
+ const signal = obj.signal;
3004
+ if (signal.aborted) {
3005
+ return true;
3006
+ }
3007
+ const next = obj.next(message);
3008
+ if (!next) {
3009
+ return true;
3010
+ }
3011
+ const { symbols, exhaustedAfterSend } = next;
3012
+ if (signal.aborted) {
3013
+ return true;
3014
+ }
1651
3015
  const sendStartedAt = syncProfileStart(profile);
1652
- await this.properties.rpc.send(
1653
- new MoreSymbols({
1654
- lastSeqNo: message.lastSeqNo,
1655
- syncId: message.syncId,
1656
- symbols,
1657
- }),
1658
- {
1659
- mode: new SilentDelivery({ to: [context.from!], redundancy: 1 }),
1660
- priority: SYNC_MESSAGE_PRIORITY,
1661
- },
1662
- );
3016
+ try {
3017
+ await this.properties.rpc.send(
3018
+ new MoreSymbols({
3019
+ lastSeqNo: message.lastSeqNo,
3020
+ syncId: message.syncId,
3021
+ symbols,
3022
+ }),
3023
+ {
3024
+ mode: new SilentDelivery({ to: [obj.target], redundancy: 1 }),
3025
+ priority: SYNC_MESSAGE_PRIORITY,
3026
+ signal,
3027
+ },
3028
+ );
3029
+ } catch (error) {
3030
+ if (signal?.aborted) {
3031
+ return true;
3032
+ }
3033
+ throw error;
3034
+ }
1663
3035
  if (profile) {
1664
3036
  emitSyncProfileDuration(profile, sendStartedAt, {
1665
3037
  name: "rateless.sendMoreSymbols",
@@ -1669,17 +3041,154 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
1669
3041
  syncId,
1670
3042
  });
1671
3043
  }
3044
+ if (exhaustedAfterSend) {
3045
+ // Latch exhaustion in next() before the send begins, then retain this
3046
+ // bounded process long enough for RequestAll. Starting Simple eagerly
3047
+ // also keeps fallback working with older receivers that ignore an empty
3048
+ // terminal symbol batch.
3049
+ void obj.startSimpleFallback().catch((error) => logger.error(error));
3050
+ }
1672
3051
  return true;
1673
3052
  } else if (message instanceof RequestAll) {
1674
3053
  const p = this.outgoingSyncProcesses.get(getSyncIdString(message));
1675
3054
  if (!p) {
1676
3055
  return true;
1677
3056
  }
1678
- await this.simple.onMaybeMissingHashes({
1679
- hashes: p.outgoingHashes,
1680
- targets: [context.from!.hashcode()],
1681
- });
3057
+ if (context.from?.hashcode() !== p.target) {
3058
+ return true;
3059
+ }
3060
+ if (p.signal.aborted) {
3061
+ return true;
3062
+ }
3063
+ // RequestAll ends only this target's rateless attempt. Other target
3064
+ // encoders and response authorizations remain independently owned.
3065
+ const fallback = p.startSimpleFallback();
3066
+ p.free();
3067
+ await fallback;
1682
3068
  return true;
3069
+ } else if (
3070
+ message instanceof ResponseMaybeSync ||
3071
+ message instanceof ResponseMaybeSyncCapabilities
3072
+ ) {
3073
+ const from = context.from!;
3074
+ // Simple authorizations are exact request leases and take precedence
3075
+ // over Rateless' broader process authorization. This also handles a
3076
+ // delayed fallback/prelude response after the Rateless process for the
3077
+ // same target has been replaced.
3078
+ const simpleLeases = this.simple.consumeAuthorizedMaybeSyncResponse(
3079
+ message.hashes,
3080
+ from,
3081
+ );
3082
+ const simpleHashes = simpleLeases.flatMap((lease) => lease.hashes);
3083
+ const simpleHashSet = new Set(simpleHashes);
3084
+ const ratelessCandidateHashes: string[] = [];
3085
+ let inspected = 0;
3086
+ const iterator = message.hashes[Symbol.iterator]();
3087
+ let exhausted = false;
3088
+ try {
3089
+ while (inspected < MAX_RATELESS_RESPONSE_HASHES_INSPECTED) {
3090
+ const next = iterator.next();
3091
+ if (next.done) {
3092
+ exhausted = true;
3093
+ break;
3094
+ }
3095
+ inspected += 1;
3096
+ if (!simpleHashSet.has(next.value)) {
3097
+ ratelessCandidateHashes.push(next.value);
3098
+ }
3099
+ }
3100
+ } finally {
3101
+ if (!exhausted) {
3102
+ iterator.return?.();
3103
+ }
3104
+ }
3105
+ const response =
3106
+ ratelessCandidateHashes.length > 0
3107
+ ? this.consumeAuthorizedRatelessResponse(
3108
+ ratelessCandidateHashes,
3109
+ from,
3110
+ )
3111
+ : undefined;
3112
+ if (
3113
+ simpleLeases.length === 0 &&
3114
+ (!response || response.authorized.length === 0)
3115
+ ) {
3116
+ response?.release();
3117
+ return true;
3118
+ }
3119
+
3120
+ let firstError: unknown;
3121
+ let rollbackRatelessAuthorization = false;
3122
+ try {
3123
+ const simpleMessage =
3124
+ message instanceof ResponseMaybeSyncCapabilities
3125
+ ? new ResponseMaybeSyncCapabilities({
3126
+ hashes: simpleHashes,
3127
+ capabilities: message.capabilities,
3128
+ })
3129
+ : new ResponseMaybeSync({ hashes: simpleHashes });
3130
+ if (response && response.authorized.length > 0) {
3131
+ const responseStartedAt = syncProfileStart(profile);
3132
+ let responseShipment = {
3133
+ messages: 0,
3134
+ fused: false,
3135
+ entries: 0,
3136
+ };
3137
+ try {
3138
+ responseShipment =
3139
+ await this.simple.shipAuthorizedMaybeSyncResponse({
3140
+ hashes: response.authorized,
3141
+ from,
3142
+ response: message,
3143
+ signal: response.signal,
3144
+ });
3145
+ } catch (error) {
3146
+ firstError = error;
3147
+ rollbackRatelessAuthorization = true;
3148
+ }
3149
+ if (profile) {
3150
+ try {
3151
+ emitSyncProfileDuration(profile, responseStartedAt, {
3152
+ name: "simple.exchangeHeads",
3153
+ entries: responseShipment.entries,
3154
+ messages: responseShipment.messages,
3155
+ targets: 1,
3156
+ details: {
3157
+ source: "ratelessResponseMaybeSync",
3158
+ fused: responseShipment.fused,
3159
+ },
3160
+ });
3161
+ } catch (error) {
3162
+ firstError ??= error;
3163
+ }
3164
+ }
3165
+ }
3166
+
3167
+ if (simpleLeases.length > 0) {
3168
+ try {
3169
+ await this.simple.shipAuthorizedMaybeSyncResponseLeases({
3170
+ leases: simpleLeases,
3171
+ from,
3172
+ response: simpleMessage,
3173
+ });
3174
+ } catch (error) {
3175
+ firstError ??= error;
3176
+ }
3177
+ }
3178
+ if (firstError !== undefined) {
3179
+ throw firstError;
3180
+ }
3181
+ return true;
3182
+ } finally {
3183
+ response?.release({ rollback: rollbackRatelessAuthorization });
3184
+ // The Simple helper owns these leases once entered and releases each
3185
+ // one in its own finally. Keep this outer ownership release as the
3186
+ // final safety net for diagnostics or setup failures that happen
3187
+ // before the helper can take responsibility.
3188
+ for (const lease of simpleLeases) {
3189
+ lease.release();
3190
+ }
3191
+ }
1683
3192
  }
1684
3193
  return this.simple.onMessage(message, context);
1685
3194
  }
@@ -1724,18 +3233,47 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
1724
3233
  }
1725
3234
 
1726
3235
  onPeerDisconnected(key: PublicSignKey | string) {
1727
- return this.simple.onPeerDisconnected(key);
3236
+ const target = typeof key === "string" ? key : key.hashcode();
3237
+ for (const process of [...this.ingoingSyncProcesses.values()]) {
3238
+ if (process.sender === target) {
3239
+ process.free(new Error("incoming rateless peer disconnected"));
3240
+ }
3241
+ }
3242
+ for (const targetLifecycle of [
3243
+ ...(this.ratelessDispatchTargets.get(target) ?? []),
3244
+ ]) {
3245
+ this.abortRatelessDispatchTarget(
3246
+ targetLifecycle,
3247
+ new Error("rateless sync target disconnected"),
3248
+ );
3249
+ }
3250
+ return this.simple.onPeerDisconnected(target);
1728
3251
  }
1729
3252
 
1730
3253
  open(): Promise<void> | void {
3254
+ const reason = new Error("rateless sync generation replaced");
3255
+ this.cancelRatelessRepairSessions(reason);
3256
+ this.ratelessDispatchLifecycleController.abort(reason);
3257
+ this.ratelessDispatchLifecycleController = new AbortController();
3258
+ this.startedOrCompletedSynchronizations = new Cache({ max: 1e4 });
3259
+ // Admission accounting intentionally spans local generations: native range
3260
+ // initialization and fallback delivery are not guaranteed to stop merely
3261
+ // because their logical process was aborted.
3262
+ this.ratelessClosed = false;
1731
3263
  return this.simple.open();
1732
3264
  }
1733
3265
 
1734
3266
  close(): Promise<void> | void {
1735
- for (const [, obj] of this.ingoingSyncProcesses) {
3267
+ this.ratelessClosed = true;
3268
+ // Abort ownership first. Process abort listeners then cancel any in-flight
3269
+ // StartSync/MoreSymbols send before they detach or free native state.
3270
+ const reason = new Error("rateless synchronizer closed");
3271
+ this.cancelRatelessRepairSessions(reason);
3272
+ this.ratelessDispatchLifecycleController.abort(reason);
3273
+ for (const obj of [...this.ingoingSyncProcesses.values()]) {
1736
3274
  obj.free();
1737
3275
  }
1738
- for (const [, obj] of this.outgoingSyncProcesses) {
3276
+ for (const obj of [...this.outgoingSyncProcesses.values()]) {
1739
3277
  obj.free();
1740
3278
  }
1741
3279
  this.clearLocalRangeEncoderCache();