@peerbit/shared-log 12.3.4 → 12.3.5-3f16953

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -1,10 +1,11 @@
1
- import { BorshError, field, variant } from "@dao-xyz/borsh";
1
+ import { BorshError, deserialize, field, serialize, variant } from "@dao-xyz/borsh";
2
2
  import { AnyBlockStore, RemoteBlocks } from "@peerbit/blocks";
3
3
  import { cidifyString } from "@peerbit/blocks-interface";
4
4
  import { Cache } from "@peerbit/cache";
5
5
  import {
6
6
  AccessError,
7
7
  PublicSignKey,
8
+ getPublicKeyFromPeerId,
8
9
  sha256Base64Sync,
9
10
  sha256Sync,
10
11
  } from "@peerbit/crypto";
@@ -31,7 +32,16 @@ import {
31
32
  } from "@peerbit/log";
32
33
  import { logger as loggerFn } from "@peerbit/logger";
33
34
  import { ClosedError, Program, type ProgramEvents } from "@peerbit/program";
34
- import { waitForSubscribers } from "@peerbit/pubsub";
35
+ import {
36
+ FanoutChannel,
37
+ type FanoutProviderHandle,
38
+ type FanoutTree,
39
+ type FanoutTreeChannelOptions,
40
+ type FanoutTreeDataEvent,
41
+ type FanoutTreeUnicastEvent,
42
+ type FanoutTreeJoinOptions,
43
+ waitForSubscribers,
44
+ } from "@peerbit/pubsub";
35
45
  import {
36
46
  SubscriptionEvent,
37
47
  UnsubcriptionEvent,
@@ -40,10 +50,10 @@ import { RPC, type RequestContext } from "@peerbit/rpc";
40
50
  import {
41
51
  AcknowledgeDelivery,
42
52
  AnyWhere,
53
+ DataMessage,
54
+ MessageHeader,
43
55
  NotStartedError,
44
- SeekDelivery,
45
56
  SilentDelivery,
46
- type WithMode,
47
57
  } from "@peerbit/stream-interface";
48
58
  import {
49
59
  AbortError,
@@ -69,6 +79,7 @@ import {
69
79
  ResponseIPrune,
70
80
  createExchangeHeadsMessages,
71
81
  } from "./exchange-heads.js";
82
+ import { FanoutEnvelope } from "./fanout-envelope.js";
72
83
  import {
73
84
  MAX_U32,
74
85
  MAX_U64,
@@ -189,6 +200,36 @@ const getLatestEntry = (
189
200
  return latest;
190
201
  };
191
202
 
203
+ const hashToSeed32 = (str: string) => {
204
+ // FNV-1a 32-bit, fast and deterministic.
205
+ let hash = 0x811c9dc5;
206
+ for (let i = 0; i < str.length; i++) {
207
+ hash ^= str.charCodeAt(i);
208
+ hash = Math.imul(hash, 0x01000193);
209
+ }
210
+ return hash >>> 0;
211
+ };
212
+
213
+ const pickDeterministicSubset = (peers: string[], seed: number, max: number) => {
214
+ if (peers.length <= max) return peers;
215
+
216
+ const subset: string[] = [];
217
+ const used = new Set<string>();
218
+ let x = seed || 1;
219
+ while (subset.length < max) {
220
+ // xorshift32
221
+ x ^= x << 13;
222
+ x ^= x >>> 17;
223
+ x ^= x << 5;
224
+ const peer = peers[(x >>> 0) % peers.length];
225
+ if (!used.has(peer)) {
226
+ used.add(peer);
227
+ subset.push(peer);
228
+ }
229
+ }
230
+ return subset;
231
+ };
232
+
192
233
  export type ReplicationLimitsOptions =
193
234
  | Partial<ReplicationLimits>
194
235
  | { min?: number; max?: number };
@@ -373,6 +414,7 @@ export type SharedLogOptions<
373
414
  compatibility?: number;
374
415
  domain?: ReplicationDomainConstructor<D>;
375
416
  eagerBlocks?: boolean | { cacheSize?: number };
417
+ fanout?: SharedLogFanoutOptions;
376
418
  };
377
419
 
378
420
  export const DEFAULT_MIN_REPLICAS = 2;
@@ -385,6 +427,10 @@ export const WAIT_FOR_REPLICATOR_REQUEST_MIN_ATTEMPTS = 3;
385
427
  // Prefer making pruning robust without timing-based heuristics.
386
428
  export const WAIT_FOR_PRUNE_DELAY = 0;
387
429
  const PRUNE_DEBOUNCE_INTERVAL = 500;
430
+ const CHECKED_PRUNE_RESEND_INTERVAL_MIN_MS = 250;
431
+ const CHECKED_PRUNE_RESEND_INTERVAL_MAX_MS = 5_000;
432
+ const CHECKED_PRUNE_RETRY_MAX_ATTEMPTS = 3;
433
+ const CHECKED_PRUNE_RETRY_MAX_DELAY_MS = 30_000;
388
434
 
389
435
  // DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
390
436
  const RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL = 1000;
@@ -395,6 +441,17 @@ const RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR = 1e-3;
395
441
 
396
442
  const DEFAULT_DISTRIBUTION_DEBOUNCE_TIME = 500;
397
443
 
444
+ const DEFAULT_SHARED_LOG_FANOUT_CHANNEL_OPTIONS: Omit<
445
+ FanoutTreeChannelOptions,
446
+ "role"
447
+ > = {
448
+ msgRate: 30,
449
+ msgSize: 1024,
450
+ uploadLimitBps: 5_000_000,
451
+ maxChildren: 24,
452
+ repair: true,
453
+ };
454
+
398
455
  const getIdForDynamicRange = (publicKey: PublicSignKey) => {
399
456
  return sha256Sync(
400
457
  concat([publicKey.bytes, new TextEncoder().encode("dynamic")]),
@@ -424,13 +481,29 @@ export type DeliveryOptions = {
424
481
  signal?: AbortSignal;
425
482
  };
426
483
 
427
- export type SharedAppendOptions<T> = AppendOptions<T> & {
484
+ export type SharedLogFanoutOptions = {
485
+ root?: string;
486
+ channel?: Partial<Omit<FanoutTreeChannelOptions, "role">>;
487
+ join?: FanoutTreeJoinOptions;
488
+ };
489
+
490
+ type SharedAppendBaseOptions<T> = AppendOptions<T> & {
428
491
  replicas?: AbsoluteReplicas | number;
429
492
  replicate?: boolean;
430
- target?: "all" | "replicators" | "none";
431
- delivery?: false | true | DeliveryOptions;
432
493
  };
433
494
 
495
+ export type SharedAppendOptions<T> =
496
+ | (SharedAppendBaseOptions<T> & {
497
+ target?: "replicators" | "none";
498
+ delivery?: false | true | DeliveryOptions;
499
+ })
500
+ | (SharedAppendBaseOptions<T> & {
501
+ // target=all uses the fanout data plane and intentionally does not expose
502
+ // per-recipient settle semantics from RPC delivery options.
503
+ target: "all";
504
+ delivery?: false | undefined;
505
+ });
506
+
434
507
  export type ReplicatorJoinEvent = { publicKey: PublicSignKey };
435
508
  export type ReplicatorLeaveEvent = { publicKey: PublicSignKey };
436
509
  export type ReplicationChangeEvent = { publicKey: PublicSignKey };
@@ -463,11 +536,12 @@ export class SharedLog<
463
536
 
464
537
  private _replicationRangeIndex!: Index<ReplicationRangeIndexable<R>>;
465
538
  private _entryCoordinatesIndex!: Index<EntryReplicated<R>>;
466
- private coordinateToHash!: Cache<string>;
467
- private recentlyRebalanced!: Cache<string>;
539
+ private coordinateToHash!: Cache<string>;
540
+ private recentlyRebalanced!: Cache<string>;
468
541
 
469
- uniqueReplicators!: Set<string>;
470
- private _replicatorsReconciled!: boolean;
542
+ uniqueReplicators!: Set<string>;
543
+ private _replicatorJoinEmitted!: Set<string>;
544
+ private _replicatorsReconciled!: boolean;
471
545
 
472
546
  /* private _totalParticipation!: number; */
473
547
 
@@ -476,6 +550,10 @@ export class SharedLog<
476
550
 
477
551
  private _onSubscriptionFn!: (arg: any) => any;
478
552
  private _onUnsubscriptionFn!: (arg: any) => any;
553
+ private _onFanoutDataFn?: (arg: any) => void;
554
+ private _onFanoutUnicastFn?: (arg: any) => void;
555
+ private _fanoutChannel?: FanoutChannel;
556
+ private _providerHandle?: FanoutProviderHandle;
479
557
 
480
558
  private _isTrustedReplicator?: (
481
559
  publicKey: PublicSignKey,
@@ -519,6 +597,15 @@ export class SharedLog<
519
597
  >; // map of peerId to timeout
520
598
 
521
599
  private latestReplicationInfoMessage!: Map<string, bigint>;
600
+ // Peers that have unsubscribed from this log's topic. We ignore replication-info
601
+ // messages from them until we see a new subscription, to avoid re-introducing
602
+ // stale membership state during close/unsubscribe races.
603
+ private _replicationInfoBlockedPeers!: Set<string>;
604
+ private _replicationInfoRequestByPeer!: Map<
605
+ string,
606
+ { attempts: number; timer?: ReturnType<typeof setTimeout> }
607
+ >;
608
+ private _replicationInfoApplyQueueByPeer!: Map<string, Promise<void>>;
522
609
 
523
610
  private remoteBlocks!: RemoteBlocks;
524
611
 
@@ -552,6 +639,10 @@ export class SharedLog<
552
639
 
553
640
  private _requestIPruneSent!: Map<string, Set<string>>; // tracks entry hash to peer hash for requesting I prune messages
554
641
  private _requestIPruneResponseReplicatorSet!: Map<string, Set<string>>; // tracks entry hash to peer hash
642
+ private _checkedPruneRetries!: Map<
643
+ string,
644
+ { attempts: number; timer?: ReturnType<typeof setTimeout> }
645
+ >;
555
646
 
556
647
  private replicationChangeDebounceFn!: ReturnType<
557
648
  typeof debounceAggregationChanges<ReplicationRangeIndexable<R>>
@@ -597,6 +688,590 @@ export class SharedLog<
597
688
  return (this.compatibility ?? Number.MAX_VALUE) < 9;
598
689
  }
599
690
 
691
+ private getFanoutChannelOptions(
692
+ options?: SharedLogFanoutOptions,
693
+ ): Omit<FanoutTreeChannelOptions, "role"> {
694
+ return {
695
+ ...DEFAULT_SHARED_LOG_FANOUT_CHANNEL_OPTIONS,
696
+ ...(options?.channel ?? {}),
697
+ };
698
+ }
699
+
700
+ private async _openFanoutChannel(options?: SharedLogFanoutOptions) {
701
+ this._closeFanoutChannel();
702
+ if (!options) {
703
+ return;
704
+ }
705
+
706
+ const fanoutService = (this.node.services as any).fanout;
707
+ if (!fanoutService) {
708
+ throw new Error(
709
+ `Fanout is configured for shared-log topic ${this.topic}, but no fanout service is available on this client`,
710
+ );
711
+ }
712
+
713
+ const resolvedRoot =
714
+ options.root ??
715
+ (await (fanoutService as any)?.topicRootControlPlane?.resolveTopicRoot?.(
716
+ this.topic,
717
+ ));
718
+ if (!resolvedRoot) {
719
+ throw new Error(
720
+ `Fanout is configured for shared-log topic ${this.topic}, but no fanout root was provided and none could be resolved`,
721
+ );
722
+ }
723
+
724
+ const channel = new FanoutChannel(fanoutService, {
725
+ topic: this.topic,
726
+ root: resolvedRoot,
727
+ });
728
+ this._fanoutChannel = channel;
729
+
730
+ this._onFanoutDataFn =
731
+ this._onFanoutDataFn ||
732
+ ((evt: any) => {
733
+ const detail = (evt as CustomEvent<FanoutTreeDataEvent>)?.detail;
734
+ if (!detail) {
735
+ return;
736
+ }
737
+ void this._onFanoutData(detail).catch((error) => logger.error(error));
738
+ });
739
+ channel.addEventListener("data", this._onFanoutDataFn);
740
+
741
+ this._onFanoutUnicastFn =
742
+ this._onFanoutUnicastFn ||
743
+ ((evt: any) => {
744
+ const detail = (evt as CustomEvent<FanoutTreeUnicastEvent>)?.detail;
745
+ if (!detail) {
746
+ return;
747
+ }
748
+ void this._onFanoutUnicast(detail).catch((error) => logger.error(error));
749
+ });
750
+ channel.addEventListener("unicast", this._onFanoutUnicastFn);
751
+
752
+ try {
753
+ const channelOptions = this.getFanoutChannelOptions(options);
754
+ if (resolvedRoot === fanoutService.publicKeyHash) {
755
+ await channel.openAsRoot(channelOptions);
756
+ return;
757
+ }
758
+ await channel.join(channelOptions, options.join);
759
+ } catch (error) {
760
+ this._closeFanoutChannel();
761
+ throw error;
762
+ }
763
+ }
764
+
765
+ private _closeFanoutChannel() {
766
+ if (this._fanoutChannel) {
767
+ if (this._onFanoutDataFn) {
768
+ this._fanoutChannel.removeEventListener("data", this._onFanoutDataFn);
769
+ }
770
+ if (this._onFanoutUnicastFn) {
771
+ this._fanoutChannel.removeEventListener(
772
+ "unicast",
773
+ this._onFanoutUnicastFn,
774
+ );
775
+ }
776
+ this._fanoutChannel.close();
777
+ }
778
+ this._fanoutChannel = undefined;
779
+ }
780
+
781
+ private async _onFanoutData(detail: FanoutTreeDataEvent) {
782
+ let envelope: FanoutEnvelope;
783
+ try {
784
+ envelope = deserialize(detail.payload, FanoutEnvelope);
785
+ } catch (error) {
786
+ if (error instanceof BorshError) {
787
+ return;
788
+ }
789
+ throw error;
790
+ }
791
+
792
+ let message: TransportMessage;
793
+ try {
794
+ message = deserialize(envelope.payload, TransportMessage);
795
+ } catch (error) {
796
+ if (error instanceof BorshError) {
797
+ return;
798
+ }
799
+ throw error;
800
+ }
801
+
802
+ if (!(message instanceof ExchangeHeadsMessage)) {
803
+ return;
804
+ }
805
+
806
+ const from =
807
+ (await this._resolvePublicKeyFromHash(envelope.from)) ??
808
+ ({ hashcode: () => envelope.from } as PublicSignKey);
809
+
810
+ const contextMessage = new DataMessage({
811
+ header: new MessageHeader({
812
+ session: 0,
813
+ mode: new AnyWhere(),
814
+ priority: 0,
815
+ }),
816
+ });
817
+ contextMessage.header.timestamp = envelope.timestamp;
818
+
819
+ await this.onMessage(message, {
820
+ from,
821
+ message: contextMessage,
822
+ });
823
+ }
824
+
825
+ private async _onFanoutUnicast(detail: FanoutTreeUnicastEvent) {
826
+ let message: TransportMessage;
827
+ try {
828
+ message = deserialize(detail.payload, TransportMessage);
829
+ } catch (error) {
830
+ if (error instanceof BorshError) {
831
+ return;
832
+ }
833
+ throw error;
834
+ }
835
+
836
+ const fromHash = detail.origin || detail.from;
837
+ const from =
838
+ (await this._resolvePublicKeyFromHash(fromHash)) ??
839
+ ({ hashcode: () => fromHash } as PublicSignKey);
840
+
841
+ const contextMessage = new DataMessage({
842
+ header: new MessageHeader({
843
+ session: 0,
844
+ mode: new AnyWhere(),
845
+ priority: 0,
846
+ }),
847
+ });
848
+ contextMessage.header.timestamp = detail.timestamp;
849
+
850
+ await this.onMessage(message, {
851
+ from,
852
+ message: contextMessage,
853
+ });
854
+ }
855
+
856
+ private async _publishExchangeHeadsViaFanout(
857
+ message: ExchangeHeadsMessage<any>,
858
+ ): Promise<void> {
859
+ if (!this._fanoutChannel) {
860
+ throw new Error(
861
+ `No fanout channel configured for shared-log topic ${this.topic}`,
862
+ );
863
+ }
864
+ const envelope = new FanoutEnvelope({
865
+ from: this.node.identity.publicKey.hashcode(),
866
+ timestamp: BigInt(Date.now()),
867
+ payload: serialize(message),
868
+ });
869
+ await this._fanoutChannel.publish(serialize(envelope));
870
+ }
871
+
872
+ private _parseDeliveryOptions(
873
+ deliveryArg: false | true | DeliveryOptions | undefined,
874
+ ): {
875
+ delivery?: DeliveryOptions;
876
+ requireRecipients: boolean;
877
+ settleMin?: number;
878
+ wrap?: (promise: Promise<void>) => Promise<void>;
879
+ } {
880
+ const delivery: DeliveryOptions | undefined =
881
+ deliveryArg === undefined || deliveryArg === false
882
+ ? undefined
883
+ : deliveryArg === true
884
+ ? {}
885
+ : deliveryArg;
886
+ if (!delivery) {
887
+ return {
888
+ delivery: undefined,
889
+ requireRecipients: false,
890
+ settleMin: undefined,
891
+ wrap: undefined,
892
+ };
893
+ }
894
+
895
+ const deliverySettle = delivery.settle ?? true;
896
+ const deliveryTimeout = delivery.timeout;
897
+ const deliverySignal = delivery.signal;
898
+ const requireRecipients = delivery.requireRecipients === true;
899
+ const settleMin =
900
+ typeof deliverySettle === "object" && Number.isFinite(deliverySettle.min)
901
+ ? Math.max(0, Math.floor(deliverySettle.min))
902
+ : undefined;
903
+
904
+ const wrap =
905
+ deliveryTimeout == null && deliverySignal == null
906
+ ? undefined
907
+ : (promise: Promise<void>) =>
908
+ new Promise<void>((resolve, reject) => {
909
+ let settled = false;
910
+ let timer: ReturnType<typeof setTimeout> | undefined = undefined;
911
+ const onAbort = () => {
912
+ if (settled) {
913
+ return;
914
+ }
915
+ settled = true;
916
+ promise.catch(() => {});
917
+ cleanup();
918
+ reject(new AbortError());
919
+ };
920
+
921
+ const cleanup = () => {
922
+ if (timer != null) {
923
+ clearTimeout(timer);
924
+ timer = undefined;
925
+ }
926
+ deliverySignal?.removeEventListener("abort", onAbort);
927
+ };
928
+
929
+ if (deliverySignal) {
930
+ if (deliverySignal.aborted) {
931
+ onAbort();
932
+ return;
933
+ }
934
+ deliverySignal.addEventListener("abort", onAbort);
935
+ }
936
+
937
+ if (deliveryTimeout != null) {
938
+ timer = setTimeout(() => {
939
+ if (settled) {
940
+ return;
941
+ }
942
+ settled = true;
943
+ promise.catch(() => {});
944
+ cleanup();
945
+ reject(new TimeoutError(`Timeout waiting for delivery`));
946
+ }, deliveryTimeout);
947
+ }
948
+
949
+ promise
950
+ .then(() => {
951
+ if (settled) {
952
+ return;
953
+ }
954
+ settled = true;
955
+ cleanup();
956
+ resolve();
957
+ })
958
+ .catch((error) => {
959
+ if (settled) {
960
+ return;
961
+ }
962
+ settled = true;
963
+ cleanup();
964
+ reject(error);
965
+ });
966
+ });
967
+
968
+ return {
969
+ delivery,
970
+ requireRecipients,
971
+ settleMin,
972
+ wrap,
973
+ };
974
+ }
975
+
976
+ private async _appendDeliverToReplicators(
977
+ entry: Entry<T>,
978
+ minReplicasValue: number,
979
+ leaders: Map<string, any>,
980
+ selfHash: string,
981
+ isLeader: boolean,
982
+ deliveryArg: false | true | DeliveryOptions | undefined,
983
+ ) {
984
+ const { delivery, requireRecipients, settleMin, wrap } =
985
+ this._parseDeliveryOptions(deliveryArg);
986
+ const pending: Promise<void>[] = [];
987
+ const track = (promise: Promise<void>) => {
988
+ pending.push(wrap ? wrap(promise) : promise);
989
+ };
990
+ const fanoutUnicastOptions =
991
+ delivery?.timeout != null || delivery?.signal != null
992
+ ? { timeoutMs: delivery.timeout, signal: delivery.signal }
993
+ : undefined;
994
+
995
+ for await (const message of createExchangeHeadsMessages(this.log, [entry])) {
996
+ await this._mergeLeadersFromGidReferences(message, minReplicasValue, leaders);
997
+ const leadersForDelivery = delivery ? new Set(leaders.keys()) : undefined;
998
+
999
+ const set = this.addPeersToGidPeerHistory(entry.meta.gid, leaders.keys());
1000
+ const hasRemotePeers = set.has(selfHash) ? set.size > 1 : set.size > 0;
1001
+ if (!hasRemotePeers) {
1002
+ if (requireRecipients) {
1003
+ throw new NoPeersError(this.rpc.topic);
1004
+ }
1005
+ continue;
1006
+ }
1007
+
1008
+ if (!delivery) {
1009
+ this.rpc
1010
+ .send(message, {
1011
+ mode: isLeader
1012
+ ? new SilentDelivery({ redundancy: 1, to: set })
1013
+ : new AcknowledgeDelivery({ redundancy: 1, to: set }),
1014
+ })
1015
+ .catch((error) => logger.error(error));
1016
+ continue;
1017
+ }
1018
+
1019
+ const orderedRemoteRecipients: string[] = [];
1020
+ for (const peer of leadersForDelivery!) {
1021
+ if (peer === selfHash) {
1022
+ continue;
1023
+ }
1024
+ orderedRemoteRecipients.push(peer);
1025
+ }
1026
+ for (const peer of set) {
1027
+ if (peer === selfHash) {
1028
+ continue;
1029
+ }
1030
+ if (leadersForDelivery!.has(peer)) {
1031
+ continue;
1032
+ }
1033
+ orderedRemoteRecipients.push(peer);
1034
+ }
1035
+
1036
+ const ackTo: string[] = [];
1037
+ let silentTo: string[] | undefined;
1038
+ // Default delivery semantics: require enough remote ACKs to reach the requested
1039
+ // replication degree (local append counts as 1).
1040
+ const ackLimit =
1041
+ settleMin == null ? Math.max(0, minReplicasValue - 1) : settleMin;
1042
+
1043
+ for (const peer of orderedRemoteRecipients) {
1044
+ if (ackTo.length < ackLimit) {
1045
+ ackTo.push(peer);
1046
+ } else {
1047
+ silentTo ||= [];
1048
+ silentTo.push(peer);
1049
+ }
1050
+ }
1051
+
1052
+ if (requireRecipients && orderedRemoteRecipients.length === 0) {
1053
+ throw new NoPeersError(this.rpc.topic);
1054
+ }
1055
+ if (requireRecipients && ackTo.length + (silentTo?.length || 0) === 0) {
1056
+ throw new NoPeersError(this.rpc.topic);
1057
+ }
1058
+
1059
+ if (ackTo.length > 0) {
1060
+ const payload = serialize(message);
1061
+ for (const peer of ackTo) {
1062
+ track(
1063
+ (async () => {
1064
+ // Unified decision point:
1065
+ // - If we can prove a cheap direct path (connected or routed), use it.
1066
+ // - Otherwise, fall back to the fanout unicast ACK path (bounded overlay routing).
1067
+ // - If that fails, fall back to pubsub/RPC routing which may flood to discover routes.
1068
+ const pubsub: any = this.node.services.pubsub as any;
1069
+ const canDirectFast =
1070
+ Boolean(pubsub?.peers?.get?.(peer)?.isWritable) ||
1071
+ Boolean(
1072
+ pubsub?.routes?.isReachable?.(
1073
+ pubsub?.publicKeyHash,
1074
+ peer,
1075
+ 0,
1076
+ ),
1077
+ );
1078
+
1079
+ if (canDirectFast) {
1080
+ await this.rpc.send(message, {
1081
+ mode: new AcknowledgeDelivery({
1082
+ redundancy: 1,
1083
+ to: [peer],
1084
+ }),
1085
+ });
1086
+ return;
1087
+ }
1088
+
1089
+ if (this._fanoutChannel) {
1090
+ try {
1091
+ await this._fanoutChannel.unicastToAck(
1092
+ peer,
1093
+ payload,
1094
+ fanoutUnicastOptions,
1095
+ );
1096
+ return;
1097
+ } catch {
1098
+ // fall back below
1099
+ }
1100
+ }
1101
+ await this.rpc.send(message, {
1102
+ mode: new AcknowledgeDelivery({
1103
+ redundancy: 1,
1104
+ to: [peer],
1105
+ }),
1106
+ });
1107
+ })(),
1108
+ );
1109
+ }
1110
+ }
1111
+
1112
+ if (silentTo?.length) {
1113
+ this.rpc
1114
+ .send(message, {
1115
+ mode: new SilentDelivery({ redundancy: 1, to: silentTo }),
1116
+ })
1117
+ .catch((error) => logger.error(error));
1118
+ }
1119
+ }
1120
+
1121
+ if (pending.length > 0) {
1122
+ await Promise.all(pending);
1123
+ }
1124
+ }
1125
+
1126
+ private async _mergeLeadersFromGidReferences(
1127
+ message: ExchangeHeadsMessage<any>,
1128
+ minReplicasValue: number,
1129
+ leaders: Map<string, any>,
1130
+ ) {
1131
+ const gidReferences = message.heads[0]?.gidRefrences;
1132
+ if (!gidReferences || gidReferences.length === 0) {
1133
+ return;
1134
+ }
1135
+
1136
+ for (const gidReference of gidReferences) {
1137
+ const entryFromGid = this.log.entryIndex.getHeads(gidReference, false);
1138
+ for (const gidEntry of await entryFromGid.all()) {
1139
+ let coordinates = await this.getCoordinates(gidEntry);
1140
+ if (coordinates == null) {
1141
+ coordinates = await this.createCoordinates(gidEntry, minReplicasValue);
1142
+ }
1143
+
1144
+ const found = await this._findLeaders(coordinates);
1145
+ for (const [key, value] of found) {
1146
+ leaders.set(key, value);
1147
+ }
1148
+ }
1149
+ }
1150
+ }
1151
+
1152
+ private async _appendDeliverToAllFanout(entry: Entry<T>) {
1153
+ for await (const message of createExchangeHeadsMessages(this.log, [entry])) {
1154
+ await this._publishExchangeHeadsViaFanout(message);
1155
+ }
1156
+ }
1157
+
1158
+ private async _resolvePublicKeyFromHash(
1159
+ hash: string,
1160
+ ): Promise<PublicSignKey | undefined> {
1161
+ const fanoutService = (this.node.services as any).fanout;
1162
+ return (
1163
+ fanoutService?.getPublicKey?.(hash) ??
1164
+ this.node.services.pubsub.getPublicKey(hash)
1165
+ );
1166
+ }
1167
+
1168
+ private async _getTopicSubscribers(
1169
+ topic: string,
1170
+ ): Promise<PublicSignKey[] | undefined> {
1171
+ const maxPeers = 64;
1172
+
1173
+ // Prefer the bounded peer set we already know from the fanout overlay.
1174
+ if (this._fanoutChannel && (topic === this.topic || topic === this.rpc.topic)) {
1175
+ const hashes = this._fanoutChannel
1176
+ .getPeerHashes({ includeSelf: false })
1177
+ .slice(0, maxPeers);
1178
+ if (hashes.length === 0) return [];
1179
+
1180
+ const keys = await Promise.all(
1181
+ hashes.map((hash) => this._resolvePublicKeyFromHash(hash)),
1182
+ );
1183
+ const uniqueKeys: PublicSignKey[] = [];
1184
+ const seen = new Set<string>();
1185
+ const selfHash = this.node.identity.publicKey.hashcode();
1186
+ for (const key of keys) {
1187
+ if (!key) continue;
1188
+ const hash = key.hashcode();
1189
+ if (hash === selfHash) continue;
1190
+ if (seen.has(hash)) continue;
1191
+ seen.add(hash);
1192
+ uniqueKeys.push(key);
1193
+ }
1194
+ return uniqueKeys;
1195
+ }
1196
+
1197
+ const selfHash = this.node.identity.publicKey.hashcode();
1198
+ const hashes: string[] = [];
1199
+
1200
+ // Best-effort provider discovery (bounded). This requires bootstrap trackers.
1201
+ try {
1202
+ const fanoutService = (this.node.services as any).fanout;
1203
+ if (fanoutService?.queryProviders) {
1204
+ const ns = `shared-log|${this.topic}`;
1205
+ const seed = hashToSeed32(topic);
1206
+ const providers: string[] = await fanoutService.queryProviders(ns, {
1207
+ want: maxPeers,
1208
+ seed,
1209
+ });
1210
+ for (const h of providers ?? []) {
1211
+ if (!h || h === selfHash) continue;
1212
+ hashes.push(h);
1213
+ if (hashes.length >= maxPeers) break;
1214
+ }
1215
+ }
1216
+ } catch {
1217
+ // Best-effort only.
1218
+ }
1219
+
1220
+ // Next, use already-connected peer streams (bounded and cheap).
1221
+ const peerMap: Map<string, unknown> | undefined = (this.node.services.pubsub as any)
1222
+ ?.peers;
1223
+ if (peerMap?.keys) {
1224
+ for (const h of peerMap.keys()) {
1225
+ if (!h || h === selfHash) continue;
1226
+ hashes.push(h);
1227
+ if (hashes.length >= maxPeers) break;
1228
+ }
1229
+ }
1230
+
1231
+ // Finally, fall back to libp2p connections (e.g. bootstrap peers) without requiring
1232
+ // any global topic membership view.
1233
+ if (hashes.length < maxPeers) {
1234
+ const connectionManager = (this.node.services.pubsub as any)?.components
1235
+ ?.connectionManager;
1236
+ const connections = connectionManager?.getConnections?.() ?? [];
1237
+ for (const conn of connections) {
1238
+ const peerId = conn?.remotePeer;
1239
+ if (!peerId) continue;
1240
+ try {
1241
+ const h = getPublicKeyFromPeerId(peerId).hashcode();
1242
+ if (!h || h === selfHash) continue;
1243
+ hashes.push(h);
1244
+ if (hashes.length >= maxPeers) break;
1245
+ } catch {
1246
+ // Best-effort only.
1247
+ }
1248
+ }
1249
+ }
1250
+
1251
+ if (hashes.length === 0) return [];
1252
+
1253
+ const uniqueHashes: string[] = [];
1254
+ const seen = new Set<string>();
1255
+ for (const h of hashes) {
1256
+ if (seen.has(h)) continue;
1257
+ seen.add(h);
1258
+ uniqueHashes.push(h);
1259
+ if (uniqueHashes.length >= maxPeers) break;
1260
+ }
1261
+
1262
+ const keys = await Promise.all(
1263
+ uniqueHashes.map((hash) => this._resolvePublicKeyFromHash(hash)),
1264
+ );
1265
+ const uniqueKeys: PublicSignKey[] = [];
1266
+ for (const key of keys) {
1267
+ if (!key) continue;
1268
+ const hash = key.hashcode();
1269
+ if (hash === selfHash) continue;
1270
+ uniqueKeys.push(key);
1271
+ }
1272
+ return uniqueKeys;
1273
+ }
1274
+
600
1275
  // @deprecated
601
1276
  private async getRole() {
602
1277
  const segments = await this.getMyReplicationSegments();
@@ -1004,8 +1679,9 @@ export class SharedLog<
1004
1679
  })
1005
1680
  .all();
1006
1681
 
1007
- this.uniqueReplicators.delete(keyHash);
1008
- await this.replicationIndex.del({ query: { hash: keyHash } });
1682
+ this.uniqueReplicators.delete(keyHash);
1683
+ this._replicatorJoinEmitted.delete(keyHash);
1684
+ await this.replicationIndex.del({ query: { hash: keyHash } });
1009
1685
 
1010
1686
  await this.updateOldestTimestampFromIndex();
1011
1687
 
@@ -1030,14 +1706,14 @@ export class SharedLog<
1030
1706
  }
1031
1707
  }
1032
1708
 
1033
- const timestamp = BigInt(+new Date());
1034
- for (const x of deleted) {
1035
- this.replicationChangeDebounceFn.add({
1036
- range: x.value,
1037
- type: "removed",
1038
- timestamp,
1039
- });
1040
- }
1709
+ const timestamp = BigInt(+new Date());
1710
+ for (const x of deleted) {
1711
+ this.replicationChangeDebounceFn.add({
1712
+ range: x.value,
1713
+ type: "removed",
1714
+ timestamp,
1715
+ });
1716
+ }
1041
1717
 
1042
1718
  const pendingMaturity = this.pendingMaturity.get(keyHash);
1043
1719
  if (pendingMaturity) {
@@ -1118,9 +1794,10 @@ export class SharedLog<
1118
1794
  { query: { hash: from.hashcode() } },
1119
1795
  { shape: { id: true } },
1120
1796
  );
1121
- if ((await otherSegmentsIterator.next(1)).length === 0) {
1122
- this.uniqueReplicators.delete(from.hashcode());
1123
- }
1797
+ if ((await otherSegmentsIterator.next(1)).length === 0) {
1798
+ this.uniqueReplicators.delete(from.hashcode());
1799
+ this._replicatorJoinEmitted.delete(from.hashcode());
1800
+ }
1124
1801
  await otherSegmentsIterator.close();
1125
1802
 
1126
1803
  await this.updateOldestTimestampFromIndex();
@@ -1160,6 +1837,7 @@ export class SharedLog<
1160
1837
 
1161
1838
  let diffs: ReplicationChanges<ReplicationRangeIndexable<R>>;
1162
1839
  let deleted: ReplicationRangeIndexable<R>[] | undefined = undefined;
1840
+ let isStoppedReplicating = false;
1163
1841
  if (reset) {
1164
1842
  deleted = (
1165
1843
  await this.replicationIndex
@@ -1198,6 +1876,7 @@ export class SharedLog<
1198
1876
  }
1199
1877
 
1200
1878
  isNewReplicator = prevCount === 0 && ranges.length > 0;
1879
+ isStoppedReplicating = prevCount > 0 && ranges.length === 0;
1201
1880
  } else {
1202
1881
  let batchSize = 100;
1203
1882
  let existing: ReplicationRangeIndexable<R>[] = [];
@@ -1281,7 +1960,16 @@ export class SharedLog<
1281
1960
  diffs = changes;
1282
1961
  }
1283
1962
 
1284
- this.uniqueReplicators.add(from.hashcode());
1963
+ const fromHash = from.hashcode();
1964
+ // Track replicator membership transitions synchronously so join/leave events are
1965
+ // idempotent even if we process concurrent reset messages/unsubscribes.
1966
+ const stoppedTransition =
1967
+ ranges.length === 0 ? this.uniqueReplicators.delete(fromHash) : false;
1968
+ if (ranges.length === 0) {
1969
+ this._replicatorJoinEmitted.delete(fromHash);
1970
+ } else {
1971
+ this.uniqueReplicators.add(fromHash);
1972
+ }
1285
1973
 
1286
1974
  let now = +new Date();
1287
1975
  let minRoleAge = await this.getDefaultMinRoleAge();
@@ -1327,13 +2015,13 @@ export class SharedLog<
1327
2015
  }),
1328
2016
  );
1329
2017
 
1330
- if (rebalance && diff.range.mode !== ReplicationIntent.Strict) {
1331
- // TODO this statement (might) cause issues with triggering pruning if the segment is strict and maturity timings will affect the outcome of rebalancing
1332
- this.replicationChangeDebounceFn.add({
1333
- ...diff,
1334
- matured: true,
1335
- }); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
1336
- }
2018
+ if (rebalance && diff.range.mode !== ReplicationIntent.Strict) {
2019
+ // TODO this statement (might) cause issues with triggering pruning if the segment is strict and maturity timings will affect the outcome of rebalancing
2020
+ this.replicationChangeDebounceFn.add({
2021
+ ...diff,
2022
+ matured: true,
2023
+ }); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
2024
+ }
1337
2025
  pendingRanges.delete(diff.range.idString);
1338
2026
  if (pendingRanges.size === 0) {
1339
2027
  this.pendingMaturity.delete(diff.range.hash);
@@ -1379,27 +2067,38 @@ export class SharedLog<
1379
2067
  }),
1380
2068
  );
1381
2069
 
1382
- if (isNewReplicator) {
1383
- this.events.dispatchEvent(
1384
- new CustomEvent<ReplicatorJoinEvent>("replicator:join", {
1385
- detail: { publicKey: from },
1386
- }),
1387
- );
2070
+ if (isNewReplicator) {
2071
+ if (!this._replicatorJoinEmitted.has(fromHash)) {
2072
+ this._replicatorJoinEmitted.add(fromHash);
2073
+ this.events.dispatchEvent(
2074
+ new CustomEvent<ReplicatorJoinEvent>("replicator:join", {
2075
+ detail: { publicKey: from },
2076
+ }),
2077
+ );
2078
+ }
1388
2079
 
1389
- if (isAllMature) {
1390
- this.events.dispatchEvent(
1391
- new CustomEvent<ReplicatorMatureEvent>("replicator:mature", {
1392
- detail: { publicKey: from },
2080
+ if (isAllMature) {
2081
+ this.events.dispatchEvent(
2082
+ new CustomEvent<ReplicatorMatureEvent>("replicator:mature", {
2083
+ detail: { publicKey: from },
1393
2084
  }),
1394
2085
  );
1395
2086
  }
1396
2087
  }
1397
2088
 
1398
- if (rebalance) {
1399
- for (const diff of diffs) {
1400
- this.replicationChangeDebounceFn.add(diff);
1401
- }
1402
- }
2089
+ if (isStoppedReplicating && stoppedTransition) {
2090
+ this.events.dispatchEvent(
2091
+ new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
2092
+ detail: { publicKey: from },
2093
+ }),
2094
+ );
2095
+ }
2096
+
2097
+ if (rebalance) {
2098
+ for (const diff of diffs) {
2099
+ this.replicationChangeDebounceFn.add(diff);
2100
+ }
2101
+ }
1403
2102
 
1404
2103
  if (!from.equals(this.node.identity.publicKey)) {
1405
2104
  this.rebalanceParticipationDebounced?.call();
@@ -1432,6 +2131,20 @@ export class SharedLog<
1432
2131
  if (change) {
1433
2132
  let addedOrReplaced = change.filter((x) => x.type !== "removed");
1434
2133
  if (addedOrReplaced.length > 0) {
2134
+ // Provider discovery keep-alive (best-effort). This enables bounded targeted fetches
2135
+ // without relying on any global subscriber list.
2136
+ try {
2137
+ const fanoutService = (this.node.services as any).fanout;
2138
+ if (fanoutService?.provide && !this._providerHandle) {
2139
+ this._providerHandle = fanoutService.provide(`shared-log|${this.topic}`, {
2140
+ ttlMs: 120_000,
2141
+ announceIntervalMs: 60_000,
2142
+ });
2143
+ }
2144
+ } catch {
2145
+ // Best-effort only.
2146
+ }
2147
+
1435
2148
  let message:
1436
2149
  | AllReplicatingSegmentsMessage
1437
2150
  | AddedReplicationSegmentMessage
@@ -1506,6 +2219,82 @@ export class SharedLog<
1506
2219
  }
1507
2220
  }
1508
2221
 
2222
+ private clearCheckedPruneRetry(hash: string) {
2223
+ const state = this._checkedPruneRetries.get(hash);
2224
+ if (state?.timer) {
2225
+ clearTimeout(state.timer);
2226
+ }
2227
+ this._checkedPruneRetries.delete(hash);
2228
+ }
2229
+
2230
+ private scheduleCheckedPruneRetry(args: {
2231
+ entry: EntryReplicated<R> | ShallowOrFullEntry<any>;
2232
+ leaders: Map<string, unknown> | Set<string>;
2233
+ }) {
2234
+ if (this.closed) return;
2235
+ if (this._pendingDeletes.has(args.entry.hash)) return;
2236
+
2237
+ const hash = args.entry.hash;
2238
+ const state =
2239
+ this._checkedPruneRetries.get(hash) ?? { attempts: 0 };
2240
+
2241
+ if (state.timer) return;
2242
+ if (state.attempts >= CHECKED_PRUNE_RETRY_MAX_ATTEMPTS) {
2243
+ // Avoid unbounded background retries; a new replication-change event can
2244
+ // always re-enqueue pruning with fresh leader info.
2245
+ return;
2246
+ }
2247
+
2248
+ const attempt = state.attempts + 1;
2249
+ const jitterMs = Math.floor(Math.random() * 250);
2250
+ const delayMs = Math.min(
2251
+ CHECKED_PRUNE_RETRY_MAX_DELAY_MS,
2252
+ 1_000 * 2 ** (attempt - 1) + jitterMs,
2253
+ );
2254
+
2255
+ state.attempts = attempt;
2256
+ state.timer = setTimeout(async () => {
2257
+ const st = this._checkedPruneRetries.get(hash);
2258
+ if (st) st.timer = undefined;
2259
+ if (this.closed) return;
2260
+ if (this._pendingDeletes.has(hash)) return;
2261
+
2262
+ let leadersMap: Map<string, any> | undefined;
2263
+ try {
2264
+ const replicas = decodeReplicas(args.entry).getValue(this);
2265
+ leadersMap = await this.findLeadersFromEntry(args.entry, replicas, {
2266
+ roleAge: 0,
2267
+ });
2268
+ } catch {
2269
+ // Best-effort only.
2270
+ }
2271
+
2272
+ if (!leadersMap || leadersMap.size === 0) {
2273
+ if (args.leaders instanceof Map) {
2274
+ leadersMap = args.leaders as any;
2275
+ } else {
2276
+ leadersMap = new Map<string, any>();
2277
+ for (const k of args.leaders) {
2278
+ leadersMap.set(k, { intersecting: true });
2279
+ }
2280
+ }
2281
+ }
2282
+
2283
+ try {
2284
+ const leadersForRetry = leadersMap ?? new Map<string, any>();
2285
+ await this.pruneDebouncedFnAddIfNotKeeping({
2286
+ key: hash,
2287
+ // TODO types
2288
+ value: { entry: args.entry as any, leaders: leadersForRetry },
2289
+ });
2290
+ } catch {
2291
+ // Best-effort only; pruning will be re-attempted on future changes.
2292
+ }
2293
+ }, delayMs);
2294
+ state.timer.unref?.();
2295
+ this._checkedPruneRetries.set(hash, state);
2296
+ }
2297
+
1509
2298
  async append(
1510
2299
  data: T,
1511
2300
  options?: SharedAppendOptions<T> | undefined,
@@ -1571,286 +2360,30 @@ export class SharedLog<
1571
2360
  if (options?.target !== "none") {
1572
2361
  const target = options?.target;
1573
2362
  const deliveryArg = options?.delivery;
1574
- const delivery: DeliveryOptions | undefined =
1575
- deliveryArg === undefined || deliveryArg === false
1576
- ? undefined
1577
- : deliveryArg === true
1578
- ? {}
1579
- : deliveryArg;
1580
-
1581
- let requireRecipients = false;
1582
- let settleMin: number | undefined;
1583
- let guardDelivery:
1584
- | ((promise: Promise<void>) => Promise<void>)
1585
- | undefined = undefined;
1586
-
1587
- let firstDeliveryPromise: Promise<void> | undefined;
1588
- let deliveryPromises: Promise<void>[] | undefined;
1589
- let addDeliveryPromise: ((promise: Promise<void>) => void) | undefined;
1590
-
1591
- const leadersForDelivery =
1592
- delivery && (target === "replicators" || !target)
1593
- ? new Set(leaders.keys())
1594
- : undefined;
1595
-
1596
- if (delivery) {
1597
- const deliverySettle = delivery.settle ?? true;
1598
- const deliveryTimeout = delivery.timeout;
1599
- const deliverySignal = delivery.signal;
1600
- requireRecipients = delivery.requireRecipients === true;
1601
- settleMin =
1602
- typeof deliverySettle === "object" &&
1603
- Number.isFinite(deliverySettle.min)
1604
- ? Math.max(0, Math.floor(deliverySettle.min))
1605
- : undefined;
1606
-
1607
- guardDelivery =
1608
- deliveryTimeout == null && deliverySignal == null
1609
- ? undefined
1610
- : (promise: Promise<void>) =>
1611
- new Promise<void>((resolve, reject) => {
1612
- let settled = false;
1613
- let timer: ReturnType<typeof setTimeout> | undefined =
1614
- undefined;
1615
- const onAbort = () => {
1616
- if (settled) {
1617
- return;
1618
- }
1619
- settled = true;
1620
- promise.catch(() => {});
1621
- cleanup();
1622
- reject(new AbortError());
1623
- };
1624
-
1625
- const cleanup = () => {
1626
- if (timer != null) {
1627
- clearTimeout(timer);
1628
- timer = undefined;
1629
- }
1630
- deliverySignal?.removeEventListener("abort", onAbort);
1631
- };
2363
+ const hasDelivery = !(deliveryArg === undefined || deliveryArg === false);
1632
2364
 
1633
- if (deliverySignal) {
1634
- if (deliverySignal.aborted) {
1635
- onAbort();
1636
- return;
1637
- }
1638
- deliverySignal.addEventListener("abort", onAbort);
1639
- }
1640
-
1641
- if (deliveryTimeout != null) {
1642
- timer = setTimeout(() => {
1643
- if (settled) {
1644
- return;
1645
- }
1646
- settled = true;
1647
- promise.catch(() => {});
1648
- cleanup();
1649
- reject(new TimeoutError(`Timeout waiting for delivery`));
1650
- }, deliveryTimeout);
1651
- }
1652
-
1653
- promise
1654
- .then(() => {
1655
- if (settled) {
1656
- return;
1657
- }
1658
- settled = true;
1659
- cleanup();
1660
- resolve();
1661
- })
1662
- .catch((e) => {
1663
- if (settled) {
1664
- return;
1665
- }
1666
- settled = true;
1667
- cleanup();
1668
- reject(e);
1669
- });
1670
- });
1671
-
1672
- addDeliveryPromise = (promise: Promise<void>) => {
1673
- if (!firstDeliveryPromise) {
1674
- firstDeliveryPromise = promise;
1675
- return;
1676
- }
1677
- if (!deliveryPromises) {
1678
- deliveryPromises = [firstDeliveryPromise, promise];
1679
- firstDeliveryPromise = undefined;
1680
- return;
1681
- }
1682
- deliveryPromises.push(promise);
1683
- };
2365
+ if (target === "all" && hasDelivery) {
2366
+ throw new Error(
2367
+ `delivery options are not supported with target="all"; fanout broadcast is fire-and-forward`,
2368
+ );
1684
2369
  }
1685
-
1686
- for await (const message of createExchangeHeadsMessages(this.log, [
1687
- result.entry,
1688
- ])) {
1689
- if (target === "replicators" || !target) {
1690
- if (message.heads[0].gidRefrences.length > 0) {
1691
- for (const ref of message.heads[0].gidRefrences) {
1692
- const entryFromGid = this.log.entryIndex.getHeads(ref, false);
1693
- for (const entry of await entryFromGid.all()) {
1694
- let coordinates = await this.getCoordinates(entry);
1695
- if (coordinates == null) {
1696
- coordinates = await this.createCoordinates(
1697
- entry,
1698
- minReplicasValue,
1699
- );
1700
- // TODO are we every to come here?
1701
- }
1702
-
1703
- const result = await this._findLeaders(coordinates);
1704
- for (const [k, v] of result) {
1705
- leaders.set(k, v);
1706
- }
1707
- }
1708
- }
1709
- }
1710
-
1711
- const set = this.addPeersToGidPeerHistory(
1712
- result.entry.meta.gid,
1713
- leaders.keys(),
1714
- );
1715
- let hasRemotePeers = set.has(selfHash) ? set.size > 1 : set.size > 0;
1716
- if (!hasRemotePeers) {
1717
- if (requireRecipients) {
1718
- throw new NoPeersError(this.rpc.topic);
1719
- }
1720
- continue;
1721
- }
1722
-
1723
- if (!delivery) {
1724
- this.rpc
1725
- .send(message, {
1726
- mode: isLeader
1727
- ? new SilentDelivery({ redundancy: 1, to: set })
1728
- : new AcknowledgeDelivery({ redundancy: 1, to: set }),
1729
- })
1730
- .catch((e) => logger.error(e));
1731
- continue;
1732
- }
1733
-
1734
- let expectedRemoteRecipientsCount = 0;
1735
- const ackTo: string[] = [];
1736
- let silentTo: string[] | undefined;
1737
- const ackLimit =
1738
- settleMin == null ? Number.POSITIVE_INFINITY : settleMin;
1739
-
1740
- // Always settle towards the current expected replicators for this entry,
1741
- // not the entire gid peer history.
1742
- for (const peer of leadersForDelivery!) {
1743
- if (peer === selfHash) {
1744
- continue;
1745
- }
1746
- expectedRemoteRecipientsCount++;
1747
- if (ackTo.length < ackLimit) {
1748
- ackTo.push(peer);
1749
- } else {
1750
- silentTo ||= [];
1751
- silentTo.push(peer);
1752
- }
1753
- }
1754
-
1755
- // Still deliver to known peers for the gid (best-effort), but don't let them
1756
- // satisfy the settle requirement.
1757
- for (const peer of set) {
1758
- if (peer === selfHash) {
1759
- continue;
1760
- }
1761
- if (leadersForDelivery!.has(peer)) {
1762
- continue;
1763
- }
1764
- silentTo ||= [];
1765
- silentTo.push(peer);
1766
- }
1767
-
1768
- if (requireRecipients && expectedRemoteRecipientsCount === 0) {
1769
- throw new NoPeersError(this.rpc.topic);
1770
- }
1771
-
1772
- if (
1773
- requireRecipients &&
1774
- ackTo.length + (silentTo?.length || 0) === 0
1775
- ) {
1776
- throw new NoPeersError(this.rpc.topic);
1777
- }
1778
-
1779
- if (ackTo.length > 0) {
1780
- const promise = this.rpc.send(message, {
1781
- mode: new AcknowledgeDelivery({
1782
- redundancy: 1,
1783
- to: ackTo,
1784
- }),
1785
- });
1786
- addDeliveryPromise!(
1787
- guardDelivery ? guardDelivery(promise) : promise,
1788
- );
1789
- }
1790
-
1791
- if (silentTo?.length) {
1792
- this.rpc
1793
- .send(message, {
1794
- mode: new SilentDelivery({ redundancy: 1, to: silentTo }),
1795
- })
1796
- .catch((e) => logger.error(e));
1797
- }
1798
- } else {
1799
- if (!delivery) {
1800
- this.rpc.send(message).catch((e) => logger.error(e));
1801
- continue;
1802
- }
1803
-
1804
- const subscribers = await this.node.services.pubsub.getSubscribers(
1805
- this.rpc.topic,
1806
- );
1807
-
1808
- const ackTo: PublicSignKey[] = [];
1809
- let silentTo: PublicSignKey[] | undefined;
1810
- const ackLimit =
1811
- settleMin == null ? Number.POSITIVE_INFINITY : settleMin;
1812
- for (const subscriber of subscribers || []) {
1813
- if (subscriber.hashcode() === selfHash) {
1814
- continue;
1815
- }
1816
- if (ackTo.length < ackLimit) {
1817
- ackTo.push(subscriber);
1818
- } else {
1819
- silentTo ||= [];
1820
- silentTo.push(subscriber);
1821
- }
1822
- }
1823
-
1824
- if (
1825
- requireRecipients &&
1826
- ackTo.length + (silentTo?.length || 0) === 0
1827
- ) {
1828
- throw new NoPeersError(this.rpc.topic);
1829
- }
1830
-
1831
- if (ackTo.length > 0) {
1832
- const promise = this.rpc.send(message, {
1833
- mode: new AcknowledgeDelivery({ redundancy: 1, to: ackTo }),
1834
- });
1835
- addDeliveryPromise!(
1836
- guardDelivery ? guardDelivery(promise) : promise,
1837
- );
1838
- }
1839
-
1840
- if (silentTo?.length) {
1841
- this.rpc
1842
- .send(message, {
1843
- mode: new SilentDelivery({ redundancy: 1, to: silentTo }),
1844
- })
1845
- .catch((e) => logger.error(e));
1846
- }
1847
- }
2370
+ if (target === "all" && !this._fanoutChannel) {
2371
+ throw new Error(
2372
+ `No fanout channel configured for shared-log topic ${this.topic}`,
2373
+ );
1848
2374
  }
1849
2375
 
1850
- if (deliveryPromises) {
1851
- await Promise.all(deliveryPromises);
1852
- } else if (firstDeliveryPromise) {
1853
- await firstDeliveryPromise;
2376
+ if (target === "all") {
2377
+ await this._appendDeliverToAllFanout(result.entry);
2378
+ } else {
2379
+ await this._appendDeliverToReplicators(
2380
+ result.entry,
2381
+ minReplicasValue,
2382
+ leaders,
2383
+ selfHash,
2384
+ isLeader,
2385
+ deliveryArg,
2386
+ );
1854
2387
  }
1855
2388
  }
1856
2389
 
@@ -1891,14 +2424,18 @@ export class SharedLog<
1891
2424
  this.domain.resolution,
1892
2425
  );
1893
2426
  this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 2e4;
1894
- this._pendingDeletes = new Map();
1895
- this._pendingIHave = new Map();
1896
- this.latestReplicationInfoMessage = new Map();
1897
- this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
1898
- this.recentlyRebalanced = new Cache<string>({ max: 1e4, ttl: 1e5 });
1899
-
1900
- this.uniqueReplicators = new Set();
1901
- this._replicatorsReconciled = false;
2427
+ this._pendingDeletes = new Map();
2428
+ this._pendingIHave = new Map();
2429
+ this.latestReplicationInfoMessage = new Map();
2430
+ this._replicationInfoBlockedPeers = new Set();
2431
+ this._replicationInfoRequestByPeer = new Map();
2432
+ this._replicationInfoApplyQueueByPeer = new Map();
2433
+ this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
2434
+ this.recentlyRebalanced = new Cache<string>({ max: 1e4, ttl: 1e5 });
2435
+
2436
+ this.uniqueReplicators = new Set();
2437
+ this._replicatorJoinEmitted = new Set();
2438
+ this._replicatorsReconciled = false;
1902
2439
 
1903
2440
  this.openTime = +new Date();
1904
2441
  this.oldestOpenTime = this.openTime;
@@ -1935,6 +2472,13 @@ export class SharedLog<
1935
2472
  }
1936
2473
 
1937
2474
  this._closeController = new AbortController();
2475
+ this._closeController.signal.addEventListener("abort", () => {
2476
+ for (const [_peer, state] of this._replicationInfoRequestByPeer) {
2477
+ if (state.timer) clearTimeout(state.timer);
2478
+ }
2479
+ this._replicationInfoRequestByPeer.clear();
2480
+ });
2481
+
1938
2482
  this._isTrustedReplicator = options?.canReplicate;
1939
2483
  this.keep = options?.keep;
1940
2484
  this.pendingMaturity = new Map();
@@ -1942,19 +2486,56 @@ export class SharedLog<
1942
2486
  const id = sha256Base64Sync(this.log.id);
1943
2487
  const storage = await this.node.storage.sublevel(id);
1944
2488
 
1945
- const localBlocks = await new AnyBlockStore(
1946
- await storage.sublevel("blocks"),
1947
- );
2489
+ const localBlocks = await new AnyBlockStore(await storage.sublevel("blocks"));
2490
+ const fanoutService = (this.node.services as any).fanout as FanoutTree | undefined;
2491
+ const blockProviderNamespace = (cid: string) => `cid:${cid}`;
1948
2492
  this.remoteBlocks = new RemoteBlocks({
1949
2493
  local: localBlocks,
1950
- publish: (message, options) =>
1951
- this.rpc.send(
1952
- new BlocksMessage(message),
1953
- (options as WithMode).mode instanceof AnyWhere ? undefined : options,
1954
- ),
2494
+ publish: (message, options) => this.rpc.send(new BlocksMessage(message), options),
1955
2495
  waitFor: this.rpc.waitFor.bind(this.rpc),
1956
2496
  publicKey: this.node.identity.publicKey,
1957
2497
  eagerBlocks: options?.eagerBlocks ?? true,
2498
+ resolveProviders: async (cid, opts) => {
2499
+ // 1) tracker-backed provider directory (best-effort, bounded)
2500
+ try {
2501
+ const providers = await fanoutService?.queryProviders(
2502
+ blockProviderNamespace(cid),
2503
+ {
2504
+ want: 8,
2505
+ timeoutMs: 2_000,
2506
+ queryTimeoutMs: 500,
2507
+ bootstrapMaxPeers: 2,
2508
+ signal: opts?.signal,
2509
+ },
2510
+ );
2511
+ if (providers && providers.length > 0) return providers;
2512
+ } catch {
2513
+ // ignore discovery failures
2514
+ }
2515
+
2516
+ // 2) fallback to currently connected RPC peers
2517
+ const self = this.node.identity.publicKey.hashcode();
2518
+ const out: string[] = [];
2519
+ const peers = (this.rpc as any)?.peers;
2520
+ for (const h of peers?.keys?.() ?? []) {
2521
+ if (h === self) continue;
2522
+ if (out.includes(h)) continue;
2523
+ out.push(h);
2524
+ if (out.length >= 32) break;
2525
+ }
2526
+ return out;
2527
+ },
2528
+ onPut: async (cid) => {
2529
+ // Best-effort directory announce for "get without remote.from" workflows.
2530
+ try {
2531
+ await fanoutService?.announceProvider(blockProviderNamespace(cid), {
2532
+ ttlMs: 120_000,
2533
+ bootstrapMaxPeers: 2,
2534
+ });
2535
+ } catch {
2536
+ // ignore announce failures
2537
+ }
2538
+ },
1958
2539
  });
1959
2540
 
1960
2541
  await this.remoteBlocks.start();
@@ -1981,9 +2562,10 @@ export class SharedLog<
1981
2562
  ],
1982
2563
  })) > 0;
1983
2564
 
1984
- this._gidPeersHistory = new Map();
1985
- this._requestIPruneSent = new Map();
1986
- this._requestIPruneResponseReplicatorSet = new Map();
2565
+ this._gidPeersHistory = new Map();
2566
+ this._requestIPruneSent = new Map();
2567
+ this._requestIPruneResponseReplicatorSet = new Map();
2568
+ this._checkedPruneRetries = new Map();
1987
2569
 
1988
2570
  this.replicationChangeDebounceFn = debounceAggregationChanges<
1989
2571
  ReplicationRangeIndexable<R>
@@ -2068,6 +2650,87 @@ export class SharedLog<
2068
2650
 
2069
2651
  await this.log.open(this.remoteBlocks, this.node.identity, {
2070
2652
  keychain: this.node.services.keychain,
2653
+ resolveRemotePeers: async (hash, options) => {
2654
+ if (options?.signal?.aborted) return undefined;
2655
+
2656
+ const maxPeers = 8;
2657
+ const self = this.node.identity.publicKey.hashcode();
2658
+ const seed = hashToSeed32(hash);
2659
+
2660
+ // Best hint: peers that have recently confirmed having this entry hash.
2661
+ const hinted = this._requestIPruneResponseReplicatorSet.get(hash);
2662
+ if (hinted && hinted.size > 0) {
2663
+ const peers = [...hinted].filter((p) => p !== self);
2664
+ return peers.length > 0
2665
+ ? pickDeterministicSubset(peers, seed, maxPeers)
2666
+ : undefined;
2667
+ }
2668
+
2669
+ // Next: peers we already contacted about this hash (may still have it).
2670
+ const contacted = this._requestIPruneSent.get(hash);
2671
+ if (contacted && contacted.size > 0) {
2672
+ const peers = [...contacted].filter((p) => p !== self);
2673
+ return peers.length > 0
2674
+ ? pickDeterministicSubset(peers, seed, maxPeers)
2675
+ : undefined;
2676
+ }
2677
+
2678
+ let candidates: string[] | undefined;
2679
+
2680
+ // Prefer the replicator cache; fall back to subscribers if we have no other signal.
2681
+ const replicatorCandidates = [...this.uniqueReplicators].filter(
2682
+ (p) => p !== self,
2683
+ );
2684
+ if (replicatorCandidates.length > 0) {
2685
+ candidates = replicatorCandidates;
2686
+ } else {
2687
+ try {
2688
+ const subscribers = await this._getTopicSubscribers(this.topic);
2689
+ const subscriberCandidates =
2690
+ subscribers?.map((k) => k.hashcode()).filter((p) => p !== self) ??
2691
+ [];
2692
+ candidates =
2693
+ subscriberCandidates.length > 0 ? subscriberCandidates : undefined;
2694
+ } catch {
2695
+ // Best-effort only.
2696
+ }
2697
+
2698
+ if (!candidates || candidates.length === 0) {
2699
+ // Last resort: peers we are already directly connected to. This avoids
2700
+ // depending on global membership knowledge in early-join scenarios.
2701
+ const peerMap = (this.node.services.pubsub as any)?.peers;
2702
+ if (peerMap?.keys) {
2703
+ candidates = [...peerMap.keys()];
2704
+ }
2705
+ }
2706
+
2707
+ if (!candidates || candidates.length === 0) {
2708
+ // Even if the pubsub stream has no established peer streams yet, we may
2709
+ // still have a libp2p connection to one or more peers (e.g. bootstrap).
2710
+ const connectionManager = (this.node.services.pubsub as any)?.components
2711
+ ?.connectionManager;
2712
+ const connections = connectionManager?.getConnections?.() ?? [];
2713
+ const connectionHashes: string[] = [];
2714
+ for (const conn of connections) {
2715
+ const peerId = conn?.remotePeer;
2716
+ if (!peerId) continue;
2717
+ try {
2718
+ connectionHashes.push(getPublicKeyFromPeerId(peerId).hashcode());
2719
+ } catch {
2720
+ // Best-effort only.
2721
+ }
2722
+ }
2723
+ if (connectionHashes.length > 0) {
2724
+ candidates = connectionHashes;
2725
+ }
2726
+ }
2727
+ }
2728
+
2729
+ if (!candidates || candidates.length === 0) return undefined;
2730
+ const peers = candidates.filter((p) => p !== self);
2731
+ if (peers.length === 0) return undefined;
2732
+ return pickDeterministicSubset(peers, seed, maxPeers);
2733
+ },
2071
2734
  ...this._logProperties,
2072
2735
  onChange: async (change) => {
2073
2736
  await this.onChange(change);
@@ -2148,6 +2811,7 @@ export class SharedLog<
2148
2811
  );
2149
2812
 
2150
2813
  await this.rpc.subscribe();
2814
+ await this._openFanoutChannel(options?.fanout);
2151
2815
 
2152
2816
  // mark all our replicaiton ranges as "new", this would allow other peers to understand that we recently reopend our database and might need some sync and warmup
2153
2817
  await this.updateTimestampOfOwnedReplicationRanges(); // TODO do we need to do this before subscribing?
@@ -2234,17 +2898,15 @@ export class SharedLog<
2234
2898
  await this.rebalanceParticipation();
2235
2899
 
2236
2900
  // Take into account existing subscription
2237
- (await this.node.services.pubsub.getSubscribers(this.topic))?.forEach(
2238
- (v, k) => {
2239
- if (v.equals(this.node.identity.publicKey)) {
2240
- return;
2241
- }
2242
- if (this.closed) {
2243
- return;
2244
- }
2245
- this.handleSubscriptionChange(v, [this.topic], true);
2246
- },
2247
- );
2901
+ (await this._getTopicSubscribers(this.topic))?.forEach((v) => {
2902
+ if (v.equals(this.node.identity.publicKey)) {
2903
+ return;
2904
+ }
2905
+ if (this.closed) {
2906
+ return;
2907
+ }
2908
+ this.handleSubscriptionChange(v, [this.topic], true);
2909
+ });
2248
2910
  }
2249
2911
 
2250
2912
  async reset() {
@@ -2278,7 +2940,7 @@ export class SharedLog<
2278
2940
  })
2279
2941
  .then(async () => {
2280
2942
  // is reachable, announce change events
2281
- const key = await this.node.services.pubsub.getPublicKey(
2943
+ const key = await this._resolvePublicKeyFromHash(
2282
2944
  segment.value.hash,
2283
2945
  );
2284
2946
  if (!key) {
@@ -2288,22 +2950,26 @@ export class SharedLog<
2288
2950
  );
2289
2951
  }
2290
2952
 
2291
- this.uniqueReplicators.add(key.hashcode());
2953
+ const keyHash = key.hashcode();
2954
+ this.uniqueReplicators.add(keyHash);
2292
2955
 
2293
- this.events.dispatchEvent(
2294
- new CustomEvent<ReplicatorJoinEvent>("replicator:join", {
2295
- detail: { publicKey: key },
2296
- }),
2297
- );
2298
- this.events.dispatchEvent(
2299
- new CustomEvent<ReplicationChangeEvent>(
2300
- "replication:change",
2301
- {
2302
- detail: { publicKey: key },
2303
- },
2304
- ),
2305
- );
2306
- })
2956
+ if (!this._replicatorJoinEmitted.has(keyHash)) {
2957
+ this._replicatorJoinEmitted.add(keyHash);
2958
+ this.events.dispatchEvent(
2959
+ new CustomEvent<ReplicatorJoinEvent>("replicator:join", {
2960
+ detail: { publicKey: key },
2961
+ }),
2962
+ );
2963
+ this.events.dispatchEvent(
2964
+ new CustomEvent<ReplicationChangeEvent>(
2965
+ "replication:change",
2966
+ {
2967
+ detail: { publicKey: key },
2968
+ },
2969
+ ),
2970
+ );
2971
+ }
2972
+ })
2307
2973
  .catch(async (e) => {
2308
2974
  if (isNotStartedError(e)) {
2309
2975
  return; // TODO test this path
@@ -2435,48 +3101,59 @@ export class SharedLog<
2435
3101
  numbers: this.indexableDomain.numbers,
2436
3102
  });
2437
3103
 
2438
- // Check abort signal before building result
2439
- if (options?.signal?.aborted) {
2440
- return [];
2441
- }
3104
+ // Check abort signal before building result
3105
+ if (options?.signal?.aborted) {
3106
+ return [];
3107
+ }
2442
3108
 
2443
- // add all in flight
2444
- for (const [key, _] of this.syncronizer.syncInFlight) {
2445
- set.add(key);
2446
- }
3109
+ // add all in flight
3110
+ for (const [key, _] of this.syncronizer.syncInFlight) {
3111
+ set.add(key);
3112
+ }
2447
3113
 
2448
- if (options?.reachableOnly) {
2449
- // Prefer the live pubsub subscriber set when filtering reachability.
2450
- // `uniqueReplicators` is primarily driven by replication messages and can lag during
2451
- // joins/restarts; using subscribers prevents excluding peers that are reachable but
2452
- // whose replication ranges were loaded from disk or haven't been processed yet.
2453
- const subscribers =
2454
- (await this.node.services.pubsub.getSubscribers(this.topic)) ??
2455
- undefined;
2456
- const subscriberHashcodes = subscribers
2457
- ? new Set(subscribers.map((key) => key.hashcode()))
3114
+ const selfHash = this.node.identity.publicKey.hashcode();
3115
+
3116
+ if (options?.reachableOnly) {
3117
+ const directPeers: Map<string, unknown> | undefined = (this.node.services
3118
+ .pubsub as any)?.peers;
3119
+
3120
+ // Prefer the live pubsub subscriber set when filtering reachability. In some
3121
+ // flows peers can be reachable/active even before (or without) subscriber
3122
+ // state converging, so also consider direct pubsub peers.
3123
+ const subscribers =
3124
+ (await this._getTopicSubscribers(this.topic)) ?? undefined;
3125
+ const subscriberHashcodes = subscribers
3126
+ ? new Set(subscribers.map((key) => key.hashcode()))
2458
3127
  : undefined;
2459
3128
 
3129
+ // If reachability is requested but we have no basis for filtering yet
3130
+ // (subscriber snapshot hasn't converged), return the full cover set.
3131
+ // Otherwise, only keep peers we can currently reach.
3132
+ const canFilter =
3133
+ directPeers != null ||
3134
+ (subscriberHashcodes && subscriberHashcodes.size > 0);
3135
+ if (!canFilter) {
3136
+ return [...set];
3137
+ }
3138
+
2460
3139
  const reachable: string[] = [];
2461
- const selfHash = this.node.identity.publicKey.hashcode();
2462
3140
  for (const peer of set) {
2463
3141
  if (peer === selfHash) {
2464
3142
  reachable.push(peer);
2465
3143
  continue;
2466
3144
  }
2467
3145
  if (
2468
- subscriberHashcodes
2469
- ? subscriberHashcodes.has(peer)
2470
- : this.uniqueReplicators.has(peer)
3146
+ (subscriberHashcodes && subscriberHashcodes.has(peer)) ||
3147
+ (directPeers && directPeers.has(peer))
2471
3148
  ) {
2472
3149
  reachable.push(peer);
2473
3150
  }
2474
3151
  }
2475
3152
  return reachable;
2476
- }
3153
+ }
2477
3154
 
2478
- return [...set];
2479
- } catch (error) {
3155
+ return [...set];
3156
+ } catch (error) {
2480
3157
  // Handle race conditions where the index gets closed during the operation
2481
3158
  if (isNotStartedError(error as Error)) {
2482
3159
  return [];
@@ -2497,6 +3174,13 @@ export class SharedLog<
2497
3174
  this.pendingMaturity.clear();
2498
3175
 
2499
3176
  this.distributeQueue?.clear();
3177
+ this._closeFanoutChannel();
3178
+ try {
3179
+ this._providerHandle?.close();
3180
+ } catch {
3181
+ // ignore
3182
+ }
3183
+ this._providerHandle = undefined;
2500
3184
  this.coordinateToHash.clear();
2501
3185
  this.recentlyRebalanced.clear();
2502
3186
  this.uniqueReplicators.clear();
@@ -2518,28 +3202,87 @@ export class SharedLog<
2518
3202
  v.clear();
2519
3203
  v.promise.resolve(); // TODO or reject?
2520
3204
  }
2521
- for (const [_k, v] of this._pendingIHave) {
2522
- v.clear();
2523
- }
3205
+ for (const [_k, v] of this._pendingIHave) {
3206
+ v.clear();
3207
+ }
3208
+ for (const [_k, v] of this._checkedPruneRetries) {
3209
+ if (v.timer) clearTimeout(v.timer);
3210
+ }
2524
3211
 
2525
3212
  await this.remoteBlocks.stop();
2526
- this._pendingDeletes.clear();
2527
- this._pendingIHave.clear();
2528
- this.latestReplicationInfoMessage.clear();
2529
- this._gidPeersHistory.clear();
2530
- this._requestIPruneSent.clear();
2531
- this._requestIPruneResponseReplicatorSet.clear();
2532
- this.pruneDebouncedFn = undefined as any;
2533
- this.rebalanceParticipationDebounced = undefined;
2534
- this._replicationRangeIndex.stop();
2535
- this._entryCoordinatesIndex.stop();
3213
+ this._pendingDeletes.clear();
3214
+ this._pendingIHave.clear();
3215
+ this._checkedPruneRetries.clear();
3216
+ this.latestReplicationInfoMessage.clear();
3217
+ this._gidPeersHistory.clear();
3218
+ this._requestIPruneSent.clear();
3219
+ this._requestIPruneResponseReplicatorSet.clear();
3220
+ // Cancel any pending debounced timers so they can't fire after we've torn down
3221
+ // indexes/RPC state.
3222
+ this.rebalanceParticipationDebounced?.close();
3223
+ this.replicationChangeDebounceFn?.close?.();
3224
+ this.pruneDebouncedFn?.close?.();
3225
+ this.responseToPruneDebouncedFn?.close?.();
3226
+ this.pruneDebouncedFn = undefined as any;
3227
+ this.rebalanceParticipationDebounced = undefined;
3228
+ this._replicationRangeIndex.stop();
3229
+ this._entryCoordinatesIndex.stop();
2536
3230
  this._replicationRangeIndex = undefined as any;
2537
3231
  this._entryCoordinatesIndex = undefined as any;
2538
3232
 
2539
3233
  this.cpuUsage?.stop?.();
2540
3234
  /* this._totalParticipation = 0; */
2541
3235
  }
2542
- async close(from?: Program): Promise<boolean> {
3236
+ async close(from?: Program): Promise<boolean> {
3237
+ // Best-effort: announce that we are going offline before tearing down
3238
+ // RPC/subscription state.
3239
+ //
3240
+ // Important: do not delete our local replication ranges here. Keeping them
3241
+ // allows `replicate: { type: "resume" }` to restore the previous role on
3242
+ // restart. Explicit `unreplicate()` still clears local state.
3243
+ try {
3244
+ if (!this.closed) {
3245
+ // Prevent any late debounced timers (rebalance/prune) from publishing
3246
+ // replication info after we announce "segments: []". These races can leave
3247
+ // stale segments on remotes after rapid open/close cycles.
3248
+ this._isReplicating = false;
3249
+ this._isAdaptiveReplicating = false;
3250
+ this.rebalanceParticipationDebounced?.close();
3251
+ this.replicationChangeDebounceFn?.close?.();
3252
+ this.pruneDebouncedFn?.close?.();
3253
+ this.responseToPruneDebouncedFn?.close?.();
3254
+
3255
+ // Ensure the "I'm leaving" replication reset is actually published before
3256
+ // the RPC child program closes and unsubscribes from its topic. If we fire
3257
+ // and forget here, the publish can race with `super.close()` and get dropped,
3258
+ // leaving stale replication segments on remotes (flaky join/leave tests).
3259
+ // Also ensure close is bounded even when shard overlays are mid-reconcile.
3260
+ const abort = new AbortController();
3261
+ const abortTimer = setTimeout(() => {
3262
+ try {
3263
+ abort.abort(
3264
+ new TimeoutError(
3265
+ "shared-log close replication reset timed out",
3266
+ ),
3267
+ );
3268
+ } catch {
3269
+ abort.abort();
3270
+ }
3271
+ }, 2_000);
3272
+ try {
3273
+ await this.rpc
3274
+ .send(new AllReplicatingSegmentsMessage({ segments: [] }), {
3275
+ priority: 1,
3276
+ signal: abort.signal,
3277
+ })
3278
+ .catch(() => {});
3279
+ } finally {
3280
+ clearTimeout(abortTimer);
3281
+ }
3282
+ }
3283
+ } catch {
3284
+ // ignore: close should be resilient even if we were never fully started
3285
+ }
2543
3286
  const superClosed = await super.close(from);
2544
3287
  if (!superClosed) {
2545
3288
  return superClosed;
@@ -2549,12 +3292,50 @@ export class SharedLog<
2549
3292
  return true;
2550
3293
  }
2551
3294
 
2552
- async drop(from?: Program): Promise<boolean> {
2553
- const superDropped = await super.drop(from);
2554
- if (!superDropped) {
2555
- return superDropped;
2556
- }
2557
- await this._entryCoordinatesIndex.drop();
3295
+ async drop(from?: Program): Promise<boolean> {
3296
+ // Best-effort: announce that we are going offline before tearing down
3297
+ // RPC/subscription state (same reasoning as in `close()`).
3298
+ try {
3299
+ if (!this.closed) {
3300
+ this._isReplicating = false;
3301
+ this._isAdaptiveReplicating = false;
3302
+ this.rebalanceParticipationDebounced?.close();
3303
+ this.replicationChangeDebounceFn?.close?.();
3304
+ this.pruneDebouncedFn?.close?.();
3305
+ this.responseToPruneDebouncedFn?.close?.();
3306
+
3307
+ const abort = new AbortController();
3308
+ const abortTimer = setTimeout(() => {
3309
+ try {
3310
+ abort.abort(
3311
+ new TimeoutError(
3312
+ "shared-log drop replication reset timed out",
3313
+ ),
3314
+ );
3315
+ } catch {
3316
+ abort.abort();
3317
+ }
3318
+ }, 2_000);
3319
+ try {
3320
+ await this.rpc
3321
+ .send(new AllReplicatingSegmentsMessage({ segments: [] }), {
3322
+ priority: 1,
3323
+ signal: abort.signal,
3324
+ })
3325
+ .catch(() => {});
3326
+ } finally {
3327
+ clearTimeout(abortTimer);
3328
+ }
3329
+ }
3330
+ } catch {
3331
+ // ignore: drop should be resilient even if we were never fully started
3332
+ }
3333
+
3334
+ const superDropped = await super.drop(from);
3335
+ if (!superDropped) {
3336
+ return superDropped;
3337
+ }
3338
+ await this._entryCoordinatesIndex.drop();
2558
3339
  await this._replicationRangeIndex.drop();
2559
3340
  await this.log.drop();
2560
3341
  await this._close();
@@ -2921,20 +3702,20 @@ export class SharedLog<
2921
3702
  return;
2922
3703
  }
2923
3704
 
2924
- const segments = (await this.getMyReplicationSegments()).map((x) =>
2925
- x.toReplicationRange(),
2926
- );
3705
+ const segments = (await this.getMyReplicationSegments()).map((x) =>
3706
+ x.toReplicationRange(),
3707
+ );
2927
3708
 
2928
- this.rpc
2929
- .send(new AllReplicatingSegmentsMessage({ segments }), {
2930
- mode: new SeekDelivery({ to: [context.from], redundancy: 1 }),
2931
- })
2932
- .catch((e) => logger.error(e.toString()));
3709
+ this.rpc
3710
+ .send(new AllReplicatingSegmentsMessage({ segments }), {
3711
+ mode: new AcknowledgeDelivery({ to: [context.from], redundancy: 1 }),
3712
+ })
3713
+ .catch((e) => logger.error(e.toString()));
2933
3714
 
2934
- // for backwards compatibility (v8) remove this when we are sure that all nodes are v9+
2935
- if (this.v8Behaviour) {
2936
- const role = this.getRole();
2937
- if (role instanceof Replicator) {
3715
+ // for backwards compatibility (v8) remove this when we are sure that all nodes are v9+
3716
+ if (this.v8Behaviour) {
3717
+ const role = this.getRole();
3718
+ if (role instanceof Replicator) {
2938
3719
  const fixedSettings = !this._isAdaptiveReplicating;
2939
3720
  if (fixedSettings) {
2940
3721
  await this.rpc.send(
@@ -2959,71 +3740,91 @@ export class SharedLog<
2959
3740
  return;
2960
3741
  }
2961
3742
 
2962
- const replicationInfoMessage = msg as
2963
- | AllReplicatingSegmentsMessage
2964
- | AddedReplicationSegmentMessage;
2965
-
2966
- // Process replication updates even if the sender isn't yet considered "ready" by
2967
- // `Program.waitFor()`. Dropping these messages can lead to missing replicator info
2968
- // (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
2969
- const from = context.from!;
2970
- const messageTimestamp = context.message.header.timestamp;
2971
- (async () => {
2972
- const prev = this.latestReplicationInfoMessage.get(from.hashcode());
2973
- if (prev && prev > messageTimestamp) {
3743
+ const replicationInfoMessage = msg as
3744
+ | AllReplicatingSegmentsMessage
3745
+ | AddedReplicationSegmentMessage;
3746
+
3747
+ // Process replication updates even if the sender isn't yet considered "ready" by
3748
+ // `Program.waitFor()`. Dropping these messages can lead to missing replicator info
3749
+ // (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
3750
+ const from = context.from!;
3751
+ const fromHash = from.hashcode();
3752
+ if (this._replicationInfoBlockedPeers.has(fromHash)) {
2974
3753
  return;
2975
3754
  }
3755
+ const messageTimestamp = context.message.header.timestamp;
3756
+ await this.withReplicationInfoApplyQueue(fromHash, async () => {
3757
+ try {
3758
+ // The peer may have unsubscribed after this message was queued.
3759
+ if (this._replicationInfoBlockedPeers.has(fromHash)) {
3760
+ return;
3761
+ }
2976
3762
 
2977
- this.latestReplicationInfoMessage.set(
2978
- from.hashcode(),
2979
- messageTimestamp,
2980
- );
3763
+ // Process in-order to avoid races where repeated reset messages arrive
3764
+ // concurrently and trigger spurious "added" diffs / rebalancing.
3765
+ const prev = this.latestReplicationInfoMessage.get(fromHash);
3766
+ if (prev && prev > messageTimestamp) {
3767
+ return;
3768
+ }
2981
3769
 
2982
- if (this.closed) {
2983
- return;
2984
- }
3770
+ this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
2985
3771
 
2986
- const reset = msg instanceof AllReplicatingSegmentsMessage;
2987
- await this.addReplicationRange(
2988
- replicationInfoMessage.segments.map((x) =>
2989
- x.toReplicationRangeIndexable(from),
2990
- ),
2991
- from,
2992
- {
2993
- reset,
2994
- checkDuplicates: true,
2995
- timestamp: Number(messageTimestamp),
2996
- },
2997
- );
2998
- })().catch((e) => {
2999
- if (isNotStartedError(e)) {
3000
- return;
3001
- }
3002
- logger.error(
3003
- `Failed to apply replication settings from '${from.hashcode()}': ${
3004
- e?.message ?? e
3005
- }`,
3006
- );
3007
- });
3008
- } else if (msg instanceof StoppedReplicating) {
3009
- if (context.from.equals(this.node.identity.publicKey)) {
3010
- return;
3011
- }
3772
+ if (this.closed) {
3773
+ return;
3774
+ }
3012
3775
 
3013
- const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(
3014
- msg.segmentIds,
3015
- context.from,
3016
- );
3776
+ const reset = msg instanceof AllReplicatingSegmentsMessage;
3777
+ await this.addReplicationRange(
3778
+ replicationInfoMessage.segments.map((x) =>
3779
+ x.toReplicationRangeIndexable(from),
3780
+ ),
3781
+ from,
3782
+ {
3783
+ reset,
3784
+ checkDuplicates: true,
3785
+ timestamp: Number(messageTimestamp),
3786
+ },
3787
+ );
3017
3788
 
3018
- await this.removeReplicationRanges(rangesToRemove, context.from);
3019
- const timestamp = BigInt(+new Date());
3020
- for (const range of rangesToRemove) {
3021
- this.replicationChangeDebounceFn.add({
3022
- range,
3023
- type: "removed",
3024
- timestamp,
3789
+ // If the peer reports any replication segments, stop re-requesting.
3790
+ // (Empty reports can be transient during startup.)
3791
+ if (replicationInfoMessage.segments.length > 0) {
3792
+ this.cancelReplicationInfoRequests(fromHash);
3793
+ }
3794
+ } catch (e) {
3795
+ if (isNotStartedError(e as Error)) {
3796
+ return;
3797
+ }
3798
+ logger.error(
3799
+ `Failed to apply replication settings from '${fromHash}': ${
3800
+ (e as any)?.message ?? e
3801
+ }`,
3802
+ );
3803
+ }
3025
3804
  });
3026
- }
3805
+ } else if (msg instanceof StoppedReplicating) {
3806
+ if (context.from.equals(this.node.identity.publicKey)) {
3807
+ return;
3808
+ }
3809
+ const fromHash = context.from.hashcode();
3810
+ if (this._replicationInfoBlockedPeers.has(fromHash)) {
3811
+ return;
3812
+ }
3813
+
3814
+ const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(
3815
+ msg.segmentIds,
3816
+ context.from,
3817
+ );
3818
+
3819
+ await this.removeReplicationRanges(rangesToRemove, context.from);
3820
+ const timestamp = BigInt(+new Date());
3821
+ for (const range of rangesToRemove) {
3822
+ this.replicationChangeDebounceFn.add({
3823
+ range,
3824
+ type: "removed",
3825
+ timestamp,
3826
+ });
3827
+ }
3027
3828
  } else {
3028
3829
  throw new Error("Unexpected message");
3029
3830
  }
@@ -3325,10 +4126,10 @@ export class SharedLog<
3325
4126
  }
3326
4127
  }
3327
4128
 
3328
- async waitForReplicator(
3329
- key: PublicSignKey,
3330
- options?: {
3331
- signal?: AbortSignal;
4129
+ async waitForReplicator(
4130
+ key: PublicSignKey,
4131
+ options?: {
4132
+ signal?: AbortSignal;
3332
4133
  eager?: boolean;
3333
4134
  roleAge?: number;
3334
4135
  timeout?: number;
@@ -3340,9 +4141,9 @@ export class SharedLog<
3340
4141
  ? undefined
3341
4142
  : (options?.roleAge ?? (await this.getDefaultMinRoleAge()));
3342
4143
 
3343
- let settled = false;
3344
- let timer: ReturnType<typeof setTimeout> | undefined;
3345
- let requestTimer: ReturnType<typeof setTimeout> | undefined;
4144
+ let settled = false;
4145
+ let timer: ReturnType<typeof setTimeout> | undefined;
4146
+ let requestTimer: ReturnType<typeof setTimeout> | undefined;
3346
4147
 
3347
4148
  const clear = () => {
3348
4149
  this.events.removeEventListener("replicator:mature", check);
@@ -3358,14 +4159,19 @@ export class SharedLog<
3358
4159
  }
3359
4160
  };
3360
4161
 
3361
- const resolve = () => {
3362
- if (settled) {
3363
- return;
3364
- }
3365
- settled = true;
3366
- clear();
3367
- deferred.resolve();
3368
- };
4162
+ const resolve = async () => {
4163
+ if (settled) {
4164
+ return;
4165
+ }
4166
+ settled = true;
4167
+ clear();
4168
+ // `waitForReplicator()` is typically used as a precondition before join/replicate
4169
+ // flows. A replicator can become mature and enqueue a debounced rebalance
4170
+ // (`replicationChangeDebounceFn`) slightly later. Flush here so callers don't
4171
+ // observe a "late" rebalance after the wait resolves.
4172
+ await this.replicationChangeDebounceFn?.flush?.();
4173
+ deferred.resolve();
4174
+ };
3369
4175
 
3370
4176
  const reject = (error: Error) => {
3371
4177
  if (settled) {
@@ -3409,13 +4215,14 @@ export class SharedLog<
3409
4215
 
3410
4216
  this.rpc
3411
4217
  .send(new RequestReplicationInfoMessage(), {
3412
- mode: new SeekDelivery({ redundancy: 1, to: [key] }),
4218
+ mode: new AcknowledgeDelivery({ redundancy: 1, to: [key] }),
3413
4219
  })
3414
4220
  .catch((e) => {
3415
4221
  // Best-effort: missing peers / unopened RPC should not fail the wait logic.
3416
4222
  if (isNotStartedError(e as Error)) {
3417
4223
  return;
3418
4224
  }
4225
+ logger.error(e?.toString?.() ?? String(e));
3419
4226
  });
3420
4227
 
3421
4228
  if (requestAttempts < maxRequestAttempts) {
@@ -3423,29 +4230,29 @@ export class SharedLog<
3423
4230
  }
3424
4231
  };
3425
4232
 
3426
- const check = async () => {
3427
- const iterator = this.replicationIndex?.iterate(
3428
- { query: new StringMatch({ key: "hash", value: key.hashcode() }) },
3429
- { reference: true },
3430
- );
3431
- try {
3432
- const rects = await iterator?.next(1);
3433
- const rect = rects?.[0]?.value;
3434
- if (!rect) {
3435
- return;
3436
- }
3437
- if (!options?.eager && resolvedRoleAge != null) {
3438
- if (!isMatured(rect, +new Date(), resolvedRoleAge)) {
4233
+ const check = async () => {
4234
+ const iterator = this.replicationIndex?.iterate(
4235
+ { query: new StringMatch({ key: "hash", value: key.hashcode() }) },
4236
+ { reference: true },
4237
+ );
4238
+ try {
4239
+ const rects = await iterator?.next(1);
4240
+ const rect = rects?.[0]?.value;
4241
+ if (!rect) {
3439
4242
  return;
3440
4243
  }
4244
+ if (!options?.eager && resolvedRoleAge != null) {
4245
+ if (!isMatured(rect, +new Date(), resolvedRoleAge)) {
4246
+ return;
4247
+ }
4248
+ }
4249
+ await resolve();
4250
+ } catch (error) {
4251
+ reject(error instanceof Error ? error : new Error(String(error)));
4252
+ } finally {
4253
+ await iterator?.close();
3441
4254
  }
3442
- resolve();
3443
- } catch (error) {
3444
- reject(error instanceof Error ? error : new Error(String(error)));
3445
- } finally {
3446
- await iterator?.close();
3447
- }
3448
- };
4255
+ };
3449
4256
 
3450
4257
  requestReplicationInfo();
3451
4258
  check();
@@ -3462,59 +4269,77 @@ export class SharedLog<
3462
4269
  coverageThreshold?: number;
3463
4270
  waitForNewPeers?: boolean;
3464
4271
  }) {
3465
- // if no remotes, just return
3466
- const subscribers = await this.node.services.pubsub.getSubscribers(
3467
- this.rpc.topic,
3468
- );
3469
- let waitForNewPeers = options?.waitForNewPeers;
3470
- if (!waitForNewPeers && (subscribers?.length ?? 0) === 0) {
3471
- throw new NoPeersError(this.rpc.topic);
3472
- }
3473
-
3474
4272
  let coverageThreshold = options?.coverageThreshold ?? 1;
3475
4273
  let deferred = pDefer<void>();
4274
+ let settled = false;
3476
4275
 
3477
4276
  const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge());
3478
4277
  const providedCustomRoleAge = options?.roleAge != null;
3479
4278
 
3480
- let checkCoverage = async () => {
4279
+ const resolve = () => {
4280
+ if (settled) return;
4281
+ settled = true;
4282
+ deferred.resolve();
4283
+ };
4284
+
4285
+ const reject = (error: unknown) => {
4286
+ if (settled) return;
4287
+ settled = true;
4288
+ deferred.reject(error);
4289
+ };
4290
+
4291
+ let checkInFlight: Promise<void> | undefined;
4292
+ const checkCoverage = async () => {
3481
4293
  const coverage = await this.calculateCoverage({
3482
4294
  roleAge,
3483
4295
  });
3484
4296
 
3485
4297
  if (coverage >= coverageThreshold) {
3486
- deferred.resolve();
4298
+ resolve();
3487
4299
  return true;
3488
4300
  }
3489
4301
  return false;
3490
4302
  };
4303
+
4304
+ const scheduleCheckCoverage = () => {
4305
+ if (settled || checkInFlight) {
4306
+ return;
4307
+ }
4308
+
4309
+ checkInFlight = checkCoverage()
4310
+ .then(() => {})
4311
+ .catch(reject)
4312
+ .finally(() => {
4313
+ checkInFlight = undefined;
4314
+ });
4315
+ };
3491
4316
  const onReplicatorMature = () => {
3492
- checkCoverage();
4317
+ scheduleCheckCoverage();
3493
4318
  };
3494
4319
  const onReplicationChange = () => {
3495
- checkCoverage();
4320
+ scheduleCheckCoverage();
3496
4321
  };
3497
4322
  this.events.addEventListener("replicator:mature", onReplicatorMature);
3498
4323
  this.events.addEventListener("replication:change", onReplicationChange);
3499
- await checkCoverage();
4324
+ await checkCoverage().catch(reject);
3500
4325
 
3501
- let interval = providedCustomRoleAge
3502
- ? setInterval(() => {
3503
- checkCoverage();
3504
- }, 100)
3505
- : undefined;
4326
+ let intervalMs = providedCustomRoleAge ? 100 : 250;
4327
+ let interval =
4328
+ roleAge > 0
4329
+ ? setInterval(() => {
4330
+ scheduleCheckCoverage();
4331
+ }, intervalMs)
4332
+ : undefined;
3506
4333
 
3507
4334
  let timeout = options?.timeout ?? this.waitForReplicatorTimeout;
3508
4335
  const timer = setTimeout(() => {
3509
4336
  clear();
3510
- deferred.reject(
3511
- new TimeoutError(`Timeout waiting for mature replicators`),
3512
- );
4337
+ reject(new TimeoutError(`Timeout waiting for mature replicators`));
3513
4338
  }, timeout);
3514
4339
 
3515
4340
  const abortListener = () => {
3516
4341
  clear();
3517
- deferred.reject(new AbortError());
4342
+ reject(new AbortError());
3518
4343
  };
3519
4344
 
3520
4345
  if (options?.signal) {
@@ -3708,9 +4533,7 @@ export class SharedLog<
3708
4533
  let subscribers = 1;
3709
4534
  if (!this.rpc.closed) {
3710
4535
  try {
3711
- subscribers =
3712
- (await this.node.services.pubsub.getSubscribers(this.rpc.topic))
3713
- ?.length ?? 1;
4536
+ subscribers = (await this._getTopicSubscribers(this.rpc.topic))?.length ?? 1;
3714
4537
  } catch {
3715
4538
  // Best-effort only; fall back to 1.
3716
4539
  }
@@ -3825,22 +4648,29 @@ export class SharedLog<
3825
4648
  const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge()); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
3826
4649
  const selfHash = this.node.identity.publicKey.hashcode();
3827
4650
 
3828
- // Use `uniqueReplicators` (replicator cache) once we've reconciled it against the
3829
- // persisted replication index. Until then, fall back to live pubsub subscribers
3830
- // and avoid relying on `uniqueReplicators` being complete.
4651
+ // Prefer `uniqueReplicators` (replicator cache) as soon as it has any data.
4652
+ // Falling back to live pubsub subscribers can include non-replicators and can
4653
+ // break delivery/availability when writers are not directly connected.
3831
4654
  let peerFilter: Set<string> | undefined = undefined;
3832
- if (this._replicatorsReconciled && this.uniqueReplicators.size > 0) {
3833
- peerFilter = this.uniqueReplicators.has(selfHash)
3834
- ? this.uniqueReplicators
3835
- : new Set([...this.uniqueReplicators, selfHash]);
4655
+ const selfReplicating = await this.isReplicating();
4656
+ if (this.uniqueReplicators.size > 0) {
4657
+ peerFilter = new Set(this.uniqueReplicators);
4658
+ if (selfReplicating) {
4659
+ peerFilter.add(selfHash);
4660
+ } else {
4661
+ peerFilter.delete(selfHash);
4662
+ }
3836
4663
  } else {
3837
4664
  try {
3838
4665
  const subscribers =
3839
- (await this.node.services.pubsub.getSubscribers(this.topic)) ??
3840
- undefined;
4666
+ (await this._getTopicSubscribers(this.topic)) ?? undefined;
3841
4667
  if (subscribers && subscribers.length > 0) {
3842
4668
  peerFilter = new Set(subscribers.map((key) => key.hashcode()));
3843
- peerFilter.add(selfHash);
4669
+ if (selfReplicating) {
4670
+ peerFilter.add(selfHash);
4671
+ } else {
4672
+ peerFilter.delete(selfHash);
4673
+ }
3844
4674
  }
3845
4675
  } catch {
3846
4676
  // Best-effort only; if pubsub isn't ready, do a full scan.
@@ -3886,76 +4716,160 @@ export class SharedLog<
3886
4716
  );
3887
4717
  }
3888
4718
 
3889
- async handleSubscriptionChange(
3890
- publicKey: PublicSignKey,
3891
- topics: string[],
3892
- subscribed: boolean,
3893
- ) {
3894
- if (!topics.includes(this.topic)) {
4719
+ private withReplicationInfoApplyQueue(
4720
+ peerHash: string,
4721
+ fn: () => Promise<void>,
4722
+ ): Promise<void> {
4723
+ const prev = this._replicationInfoApplyQueueByPeer.get(peerHash);
4724
+ const next = (prev ?? Promise.resolve())
4725
+ .catch(() => {
4726
+ // Avoid stuck queues if a previous apply failed.
4727
+ })
4728
+ .then(fn);
4729
+ this._replicationInfoApplyQueueByPeer.set(peerHash, next);
4730
+ return next.finally(() => {
4731
+ if (this._replicationInfoApplyQueueByPeer.get(peerHash) === next) {
4732
+ this._replicationInfoApplyQueueByPeer.delete(peerHash);
4733
+ }
4734
+ });
4735
+ }
4736
+
4737
+ private cancelReplicationInfoRequests(peerHash: string) {
4738
+ const state = this._replicationInfoRequestByPeer.get(peerHash);
4739
+ if (!state) return;
4740
+ if (state.timer) {
4741
+ clearTimeout(state.timer);
4742
+ }
4743
+ this._replicationInfoRequestByPeer.delete(peerHash);
4744
+ }
4745
+
4746
+ private scheduleReplicationInfoRequests(peer: PublicSignKey) {
4747
+ const peerHash = peer.hashcode();
4748
+ if (this._replicationInfoRequestByPeer.has(peerHash)) {
3895
4749
  return;
3896
4750
  }
3897
4751
 
3898
- if (!subscribed) {
3899
- this.removePeerFromGidPeerHistory(publicKey.hashcode());
4752
+ const state: { attempts: number; timer?: ReturnType<typeof setTimeout> } = {
4753
+ attempts: 0,
4754
+ };
4755
+ this._replicationInfoRequestByPeer.set(peerHash, state);
3900
4756
 
3901
- for (const [k, v] of this._requestIPruneSent) {
3902
- v.delete(publicKey.hashcode());
3903
- if (v.size === 0) {
3904
- this._requestIPruneSent.delete(k);
3905
- }
4757
+ const intervalMs = Math.max(50, this.waitForReplicatorRequestIntervalMs);
4758
+ const maxAttempts = Math.min(
4759
+ 5,
4760
+ this.waitForReplicatorRequestMaxAttempts ??
4761
+ WAIT_FOR_REPLICATOR_REQUEST_MIN_ATTEMPTS,
4762
+ );
4763
+
4764
+ const tick = () => {
4765
+ if (this.closed || this._closeController.signal.aborted) {
4766
+ this.cancelReplicationInfoRequests(peerHash);
4767
+ return;
3906
4768
  }
3907
4769
 
3908
- for (const [k, v] of this._requestIPruneResponseReplicatorSet) {
3909
- v.delete(publicKey.hashcode());
3910
- if (v.size === 0) {
3911
- this._requestIPruneResponseReplicatorSet.delete(k);
3912
- }
4770
+ state.attempts++;
4771
+
4772
+ this.rpc
4773
+ .send(new RequestReplicationInfoMessage(), {
4774
+ mode: new AcknowledgeDelivery({ redundancy: 1, to: [peer] }),
4775
+ })
4776
+ .catch((e) => {
4777
+ // Best-effort: missing peers / unopened RPC should not fail join flows.
4778
+ if (isNotStartedError(e as Error)) {
4779
+ return;
4780
+ }
4781
+ logger.error(e?.toString?.() ?? String(e));
4782
+ });
4783
+
4784
+ if (state.attempts >= maxAttempts) {
4785
+ this.cancelReplicationInfoRequests(peerHash);
4786
+ return;
3913
4787
  }
3914
4788
 
3915
- this.syncronizer.onPeerDisconnected(publicKey);
4789
+ state.timer = setTimeout(tick, intervalMs);
4790
+ state.timer.unref?.();
4791
+ };
3916
4792
 
3917
- (await this.replicationIndex.count({
3918
- query: { hash: publicKey.hashcode() },
3919
- })) > 0 &&
3920
- this.events.dispatchEvent(
3921
- new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
3922
- detail: { publicKey },
3923
- }),
3924
- );
3925
- }
4793
+ tick();
4794
+ }
3926
4795
 
3927
- if (subscribed) {
3928
- const replicationSegments = await this.getMyReplicationSegments();
3929
- if (replicationSegments.length > 0) {
3930
- this.rpc
3931
- .send(
3932
- new AllReplicatingSegmentsMessage({
3933
- segments: replicationSegments.map((x) => x.toReplicationRange()),
4796
+ async handleSubscriptionChange(
4797
+ publicKey: PublicSignKey,
4798
+ topics: string[],
4799
+ subscribed: boolean,
4800
+ ) {
4801
+ if (!topics.includes(this.topic)) {
4802
+ return;
4803
+ }
4804
+
4805
+ const peerHash = publicKey.hashcode();
4806
+ if (subscribed) {
4807
+ this._replicationInfoBlockedPeers.delete(peerHash);
4808
+ } else {
4809
+ this._replicationInfoBlockedPeers.add(peerHash);
4810
+ }
4811
+
4812
+ if (!subscribed) {
4813
+ // Emit replicator:leave at most once per (join -> leave) transition, even if we
4814
+ // concurrently process unsubscribe + replication reset messages for the same peer.
4815
+ const stoppedTransition = this.uniqueReplicators.delete(peerHash);
4816
+ this._replicatorJoinEmitted.delete(peerHash);
4817
+
4818
+ this.cancelReplicationInfoRequests(peerHash);
4819
+ this.removePeerFromGidPeerHistory(peerHash);
4820
+
4821
+ for (const [k, v] of this._requestIPruneSent) {
4822
+ v.delete(peerHash);
4823
+ if (v.size === 0) {
4824
+ this._requestIPruneSent.delete(k);
4825
+ }
4826
+ }
4827
+
4828
+ for (const [k, v] of this._requestIPruneResponseReplicatorSet) {
4829
+ v.delete(peerHash);
4830
+ if (v.size === 0) {
4831
+ this._requestIPruneResponseReplicatorSet.delete(k);
4832
+ }
4833
+ }
4834
+
4835
+ this.syncronizer.onPeerDisconnected(publicKey);
4836
+
4837
+ stoppedTransition &&
4838
+ this.events.dispatchEvent(
4839
+ new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
4840
+ detail: { publicKey },
3934
4841
  }),
3935
- {
3936
- mode: new SeekDelivery({ redundancy: 1, to: [publicKey] }),
3937
- },
3938
- )
3939
- .catch((e) => logger.error(e.toString()));
4842
+ );
4843
+ }
3940
4844
 
3941
- if (this.v8Behaviour) {
3942
- // for backwards compatibility
4845
+ if (subscribed) {
4846
+ const replicationSegments = await this.getMyReplicationSegments();
4847
+ if (replicationSegments.length > 0) {
3943
4848
  this.rpc
3944
- .send(new ResponseRoleMessage({ role: await this.getRole() }), {
3945
- mode: new SeekDelivery({ redundancy: 1, to: [publicKey] }),
3946
- })
4849
+ .send(
4850
+ new AllReplicatingSegmentsMessage({
4851
+ segments: replicationSegments.map((x) => x.toReplicationRange()),
4852
+ }),
4853
+ {
4854
+ mode: new AcknowledgeDelivery({ redundancy: 1, to: [publicKey] }),
4855
+ },
4856
+ )
3947
4857
  .catch((e) => logger.error(e.toString()));
4858
+
4859
+ if (this.v8Behaviour) {
4860
+ // for backwards compatibility
4861
+ this.rpc
4862
+ .send(new ResponseRoleMessage({ role: await this.getRole() }), {
4863
+ mode: new AcknowledgeDelivery({ redundancy: 1, to: [publicKey] }),
4864
+ })
4865
+ .catch((e) => logger.error(e.toString()));
4866
+ }
3948
4867
  }
3949
- }
3950
4868
 
3951
- // Request the remote peer's replication info. This makes joins resilient to
3952
- // timing-sensitive delivery/order issues where we may miss their initial
4869
+ // Request the remote peer's replication info. This makes joins resilient to
4870
+ // timing-sensitive delivery/order issues where we may miss their initial
3953
4871
  // replication announcement.
3954
- this.rpc
3955
- .send(new RequestReplicationInfoMessage(), {
3956
- mode: new SeekDelivery({ redundancy: 1, to: [publicKey] }),
3957
- })
3958
- .catch((e) => logger.error(e.toString()));
4872
+ this.scheduleReplicationInfoRequests(publicKey);
3959
4873
  } else {
3960
4874
  await this.removeReplicator(publicKey);
3961
4875
  }
@@ -3998,8 +4912,8 @@ export class SharedLog<
3998
4912
  leaders: Map<string, unknown> | Set<string>;
3999
4913
  }
4000
4914
  >,
4001
- options?: { timeout?: number; unchecked?: boolean },
4002
- ): Promise<any>[] {
4915
+ options?: { timeout?: number; unchecked?: boolean },
4916
+ ): Promise<any>[] {
4003
4917
  if (options?.unchecked) {
4004
4918
  return [...entries.values()].map((x) => {
4005
4919
  this._gidPeersHistory.delete(x.entry.meta.gid);
@@ -4024,30 +4938,57 @@ export class SharedLog<
4024
4938
  // - An entry is joined, where min replicas is lower than before (for all heads for this particular gid) and therefore we are not replicating anymore for this particular gid
4025
4939
  // - Peers join and leave, which means we might not be a replicator anymore
4026
4940
 
4027
- const promises: Promise<any>[] = [];
4941
+ const promises: Promise<any>[] = [];
4028
4942
 
4029
- let peerToEntries: Map<string, string[]> = new Map();
4030
- let cleanupTimer: ReturnType<typeof setTimeout>[] = [];
4943
+ let peerToEntries: Map<string, string[]> = new Map();
4944
+ let cleanupTimer: ReturnType<typeof setTimeout>[] = [];
4945
+ const explicitTimeout = options?.timeout != null;
4031
4946
 
4032
- for (const { entry, leaders } of entries.values()) {
4033
- for (const leader of leaders.keys()) {
4034
- let set = peerToEntries.get(leader);
4035
- if (!set) {
4036
- set = [];
4037
- peerToEntries.set(leader, set);
4038
- }
4947
+ for (const { entry, leaders } of entries.values()) {
4948
+ for (const leader of leaders.keys()) {
4949
+ let set = peerToEntries.get(leader);
4950
+ if (!set) {
4951
+ set = [];
4952
+ peerToEntries.set(leader, set);
4953
+ }
4039
4954
 
4040
- set.push(entry.hash);
4041
- }
4955
+ set.push(entry.hash);
4956
+ }
4042
4957
 
4043
- const pendingPrev = this._pendingDeletes.get(entry.hash);
4044
- if (pendingPrev) {
4045
- promises.push(pendingPrev.promise.promise);
4046
- continue;
4047
- }
4958
+ const pendingPrev = this._pendingDeletes.get(entry.hash);
4959
+ if (pendingPrev) {
4960
+ // If a background prune is already in-flight, an explicit prune request should
4961
+ // still respect the caller's timeout. Otherwise, tests (and user calls) can
4962
+ // block on the longer "checked prune" timeout derived from
4963
+ // `_respondToIHaveTimeout + waitForReplicatorTimeout`, which is intentionally
4964
+ // large for resiliency.
4965
+ if (explicitTimeout) {
4966
+ const timeoutMs = Math.max(0, Math.floor(options?.timeout ?? 0));
4967
+ promises.push(
4968
+ new Promise((resolve, reject) => {
4969
+ // Mirror the checked-prune error prefix so existing callers/tests can
4970
+ // match on the message substring.
4971
+ const timer = setTimeout(() => {
4972
+ reject(
4973
+ new Error(
4974
+ `Timeout for checked pruning after ${timeoutMs}ms (pending=true closed=${this.closed})`,
4975
+ ),
4976
+ );
4977
+ }, timeoutMs);
4978
+ timer.unref?.();
4979
+ pendingPrev.promise.promise
4980
+ .then(resolve, reject)
4981
+ .finally(() => clearTimeout(timer));
4982
+ }),
4983
+ );
4984
+ } else {
4985
+ promises.push(pendingPrev.promise.promise);
4986
+ }
4987
+ continue;
4988
+ }
4048
4989
 
4049
- const minReplicas = decodeReplicas(entry);
4050
- const deferredPromise: DeferredPromise<void> = pDefer();
4990
+ const minReplicas = decodeReplicas(entry);
4991
+ const deferredPromise: DeferredPromise<void> = pDefer();
4051
4992
 
4052
4993
  const clear = () => {
4053
4994
  const pending = this._pendingDeletes.get(entry.hash);
@@ -4057,12 +4998,13 @@ export class SharedLog<
4057
4998
  clearTimeout(timeout);
4058
4999
  };
4059
5000
 
4060
- const resolve = () => {
4061
- clear();
4062
- cleanupTimer.push(
4063
- setTimeout(async () => {
4064
- this._gidPeersHistory.delete(entry.meta.gid);
4065
- this.removePruneRequestSent(entry.hash);
5001
+ const resolve = () => {
5002
+ clear();
5003
+ this.clearCheckedPruneRetry(entry.hash);
5004
+ cleanupTimer.push(
5005
+ setTimeout(async () => {
5006
+ this._gidPeersHistory.delete(entry.meta.gid);
5007
+ this.removePruneRequestSent(entry.hash);
4066
5008
  this._requestIPruneResponseReplicatorSet.delete(entry.hash);
4067
5009
 
4068
5010
  if (
@@ -4106,12 +5048,19 @@ export class SharedLog<
4106
5048
  );
4107
5049
  };
4108
5050
 
4109
- const reject = (e: any) => {
4110
- clear();
4111
- this.removePruneRequestSent(entry.hash);
4112
- this._requestIPruneResponseReplicatorSet.delete(entry.hash);
4113
- deferredPromise.reject(e);
4114
- };
5051
+ const reject = (e: any) => {
5052
+ clear();
5053
+ const isCheckedPruneTimeout =
5054
+ e instanceof Error &&
5055
+ typeof e.message === "string" &&
5056
+ e.message.startsWith("Timeout for checked pruning");
5057
+ if (explicitTimeout || !isCheckedPruneTimeout) {
5058
+ this.clearCheckedPruneRetry(entry.hash);
5059
+ }
5060
+ this.removePruneRequestSent(entry.hash);
5061
+ this._requestIPruneResponseReplicatorSet.delete(entry.hash);
5062
+ deferredPromise.reject(e);
5063
+ };
4115
5064
 
4116
5065
  let cursor: NumberFromType<R>[] | undefined = undefined;
4117
5066
 
@@ -4129,14 +5078,20 @@ export class SharedLog<
4129
5078
  PRUNE_DEBOUNCE_INTERVAL * 2,
4130
5079
  );
4131
5080
 
4132
- const timeout = setTimeout(() => {
4133
- reject(
4134
- new Error(
4135
- `Timeout for checked pruning after ${checkedPruneTimeoutMs}ms (closed=${this.closed})`,
4136
- ),
4137
- );
4138
- }, checkedPruneTimeoutMs);
4139
- timeout.unref?.();
5081
+ const timeout = setTimeout(() => {
5082
+ // For internal/background prune flows (no explicit timeout), retry a few times
5083
+ // to avoid "permanently prunable" entries when `_pendingIHave` expires under
5084
+ // heavy load.
5085
+ if (!explicitTimeout) {
5086
+ this.scheduleCheckedPruneRetry({ entry, leaders });
5087
+ }
5088
+ reject(
5089
+ new Error(
5090
+ `Timeout for checked pruning after ${checkedPruneTimeoutMs}ms (closed=${this.closed})`,
5091
+ ),
5092
+ );
5093
+ }, checkedPruneTimeoutMs);
5094
+ timeout.unref?.();
4140
5095
 
4141
5096
  this._pendingDeletes.set(entry.hash, {
4142
5097
  promise: deferredPromise,
@@ -4173,20 +5128,22 @@ export class SharedLog<
4173
5128
  let existCounter = this._requestIPruneResponseReplicatorSet.get(
4174
5129
  entry.hash,
4175
5130
  );
4176
- if (!existCounter) {
4177
- existCounter = new Set();
4178
- this._requestIPruneResponseReplicatorSet.set(
4179
- entry.hash,
4180
- existCounter,
4181
- );
4182
- }
4183
- existCounter.add(publicKeyHash);
5131
+ if (!existCounter) {
5132
+ existCounter = new Set();
5133
+ this._requestIPruneResponseReplicatorSet.set(
5134
+ entry.hash,
5135
+ existCounter,
5136
+ );
5137
+ }
5138
+ existCounter.add(publicKeyHash);
5139
+ // Seed provider hints so future remote reads can avoid extra round-trips.
5140
+ this.remoteBlocks.hintProviders(entry.hash, [publicKeyHash]);
4184
5141
 
4185
- if (minReplicasValue <= existCounter.size) {
4186
- resolve();
4187
- }
4188
- },
4189
- });
5142
+ if (minReplicasValue <= existCounter.size) {
5143
+ resolve();
5144
+ }
5145
+ },
5146
+ });
4190
5147
 
4191
5148
  promises.push(deferredPromise.promise);
4192
5149
  }
@@ -4222,16 +5179,58 @@ export class SharedLog<
4222
5179
  }
4223
5180
  };
4224
5181
 
4225
- for (const [k, v] of peerToEntries) {
4226
- emitMessages(v, k);
4227
- }
5182
+ for (const [k, v] of peerToEntries) {
5183
+ emitMessages(v, k);
5184
+ }
4228
5185
 
4229
- let cleanup = () => {
4230
- for (const timer of cleanupTimer) {
4231
- clearTimeout(timer);
5186
+ // Keep remote `_pendingIHave` alive in the common "leader doesn't have entry yet"
5187
+ // case. This is intentionally disabled when an explicit timeout is provided to
5188
+ // preserve unit tests that assert remote `_pendingIHave` clears promptly.
5189
+ if (!explicitTimeout && peerToEntries.size > 0) {
5190
+ const respondToIHaveTimeout = Number(this._respondToIHaveTimeout ?? 0);
5191
+ const resendIntervalMs = Math.min(
5192
+ CHECKED_PRUNE_RESEND_INTERVAL_MAX_MS,
5193
+ Math.max(
5194
+ CHECKED_PRUNE_RESEND_INTERVAL_MIN_MS,
5195
+ Math.floor(respondToIHaveTimeout / 2) || 1_000,
5196
+ ),
5197
+ );
5198
+ let inFlight = false;
5199
+ const timer = setInterval(() => {
5200
+ if (inFlight) return;
5201
+ if (this.closed) return;
5202
+
5203
+ const pendingByPeer: [string, string[]][] = [];
5204
+ for (const [peer, hashes] of peerToEntries) {
5205
+ const pending = hashes.filter((h) => this._pendingDeletes.has(h));
5206
+ if (pending.length > 0) {
5207
+ pendingByPeer.push([peer, pending]);
5208
+ }
5209
+ }
5210
+ if (pendingByPeer.length === 0) {
5211
+ clearInterval(timer);
5212
+ return;
5213
+ }
5214
+
5215
+ inFlight = true;
5216
+ Promise.allSettled(
5217
+ pendingByPeer.map(([peer, hashes]) =>
5218
+ emitMessages(hashes, peer).catch(() => {}),
5219
+ ),
5220
+ ).finally(() => {
5221
+ inFlight = false;
5222
+ });
5223
+ }, resendIntervalMs);
5224
+ timer.unref?.();
5225
+ cleanupTimer.push(timer as any);
4232
5226
  }
4233
- this._closeController.signal.removeEventListener("abort", cleanup);
4234
- };
5227
+
5228
+ let cleanup = () => {
5229
+ for (const timer of cleanupTimer) {
5230
+ clearTimeout(timer);
5231
+ }
5232
+ this._closeController.signal.removeEventListener("abort", cleanup);
5233
+ };
4235
5234
 
4236
5235
  Promise.allSettled(promises).finally(cleanup);
4237
5236
  this._closeController.signal.addEventListener("abort", cleanup);
@@ -4303,12 +5302,21 @@ export class SharedLog<
4303
5302
  * that we potentially need to share with other peers
4304
5303
  */
4305
5304
 
4306
- if (this.closed) {
4307
- return;
4308
- }
5305
+ if (this.closed) {
5306
+ return;
5307
+ }
4309
5308
 
4310
5309
  await this.log.trim();
4311
5310
 
5311
+ const batchedChanges = Array.isArray(changeOrChanges[0])
5312
+ ? (changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>[])
5313
+ : [changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>];
5314
+ const changes = batchedChanges.flat();
5315
+ // On removed ranges (peer leaves / shrink), gid-level history can hide
5316
+ // per-entry gaps. Force a fresh delivery pass for reassigned entries.
5317
+ const forceFreshDelivery = changes.some((change) => change.type === "removed");
5318
+ const gidPeersHistorySnapshot = new Map<string, Set<string> | undefined>();
5319
+
4312
5320
  const changed = false;
4313
5321
 
4314
5322
  try {
@@ -4318,7 +5326,7 @@ export class SharedLog<
4318
5326
  > = new Map();
4319
5327
 
4320
5328
  for await (const entryReplicated of toRebalance<R>(
4321
- changeOrChanges,
5329
+ changes,
4322
5330
  this.entryCoordinatesIndex,
4323
5331
  this.recentlyRebalanced,
4324
5332
  )) {
@@ -4326,7 +5334,16 @@ export class SharedLog<
4326
5334
  break;
4327
5335
  }
4328
5336
 
4329
- let oldPeersSet = this._gidPeersHistory.get(entryReplicated.gid);
5337
+ let oldPeersSet: Set<string> | undefined;
5338
+ if (!forceFreshDelivery) {
5339
+ const gid = entryReplicated.gid;
5340
+ oldPeersSet = gidPeersHistorySnapshot.get(gid);
5341
+ if (!gidPeersHistorySnapshot.has(gid)) {
5342
+ const existing = this._gidPeersHistory.get(gid);
5343
+ oldPeersSet = existing ? new Set(existing) : undefined;
5344
+ gidPeersHistorySnapshot.set(gid, oldPeersSet);
5345
+ }
5346
+ }
4330
5347
  let isLeader = false;
4331
5348
 
4332
5349
  let currentPeers = await this.findLeaders(
@@ -4405,32 +5422,51 @@ export class SharedLog<
4405
5422
  }
4406
5423
  }
4407
5424
 
4408
- async _onUnsubscription(evt: CustomEvent<UnsubcriptionEvent>) {
4409
- logger.trace(
4410
- `Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(
4411
- evt.detail.topics.map((x) => x),
4412
- )} '`,
4413
- );
4414
- this.latestReplicationInfoMessage.delete(evt.detail.from.hashcode());
5425
+ async _onUnsubscription(evt: CustomEvent<UnsubcriptionEvent>) {
5426
+ logger.trace(
5427
+ `Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(
5428
+ evt.detail.topics.map((x) => x),
5429
+ )} '`,
5430
+ );
5431
+ if (!evt.detail.topics.includes(this.topic)) {
5432
+ return;
5433
+ }
4415
5434
 
4416
- return this.handleSubscriptionChange(
4417
- evt.detail.from,
4418
- evt.detail.topics,
4419
- false,
4420
- );
4421
- }
5435
+ const fromHash = evt.detail.from.hashcode();
5436
+ this._replicationInfoBlockedPeers.add(fromHash);
4422
5437
 
4423
- async _onSubscription(evt: CustomEvent<SubscriptionEvent>) {
4424
- logger.trace(
4425
- `New peer '${evt.detail.from.hashcode()}' connected to '${JSON.stringify(
4426
- evt.detail.topics.map((x) => x),
4427
- )}'`,
4428
- );
4429
- this.remoteBlocks.onReachable(evt.detail.from);
5438
+ // Keep a per-peer timestamp watermark when we observe an unsubscribe. This
5439
+ // prevents late/out-of-order replication-info messages from re-introducing
5440
+ // stale segments for a peer that has already left the topic.
5441
+ const now = BigInt(+new Date());
5442
+ const prev = this.latestReplicationInfoMessage.get(fromHash);
5443
+ if (!prev || prev < now) {
5444
+ this.latestReplicationInfoMessage.set(fromHash, now);
5445
+ }
5446
+
5447
+ return this.handleSubscriptionChange(
5448
+ evt.detail.from,
5449
+ evt.detail.topics,
5450
+ false,
5451
+ );
5452
+ }
5453
+
5454
+ async _onSubscription(evt: CustomEvent<SubscriptionEvent>) {
5455
+ logger.trace(
5456
+ `New peer '${evt.detail.from.hashcode()}' connected to '${JSON.stringify(
5457
+ evt.detail.topics.map((x) => x),
5458
+ )}'`,
5459
+ );
5460
+ if (!evt.detail.topics.includes(this.topic)) {
5461
+ return;
5462
+ }
5463
+
5464
+ this.remoteBlocks.onReachable(evt.detail.from);
5465
+ this._replicationInfoBlockedPeers.delete(evt.detail.from.hashcode());
4430
5466
 
4431
- return this.handleSubscriptionChange(
4432
- evt.detail.from,
4433
- evt.detail.topics,
5467
+ return this.handleSubscriptionChange(
5468
+ evt.detail.from,
5469
+ evt.detail.topics,
4434
5470
  true,
4435
5471
  );
4436
5472
  }