@peerbit/shared-log 12.3.4-ad0f88c → 12.3.5-000e3f1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -1,10 +1,11 @@
1
- import { BorshError, field, variant } from "@dao-xyz/borsh";
1
+ import { BorshError, deserialize, field, serialize, variant } from "@dao-xyz/borsh";
2
2
  import { AnyBlockStore, RemoteBlocks } from "@peerbit/blocks";
3
3
  import { cidifyString } from "@peerbit/blocks-interface";
4
4
  import { Cache } from "@peerbit/cache";
5
5
  import {
6
6
  AccessError,
7
7
  PublicSignKey,
8
+ getPublicKeyFromPeerId,
8
9
  sha256Base64Sync,
9
10
  sha256Sync,
10
11
  } from "@peerbit/crypto";
@@ -31,7 +32,16 @@ import {
31
32
  } from "@peerbit/log";
32
33
  import { logger as loggerFn } from "@peerbit/logger";
33
34
  import { ClosedError, Program, type ProgramEvents } from "@peerbit/program";
34
- import { waitForSubscribers } from "@peerbit/pubsub";
35
+ import {
36
+ FanoutChannel,
37
+ type FanoutProviderHandle,
38
+ type FanoutTree,
39
+ type FanoutTreeChannelOptions,
40
+ type FanoutTreeDataEvent,
41
+ type FanoutTreeUnicastEvent,
42
+ type FanoutTreeJoinOptions,
43
+ waitForSubscribers,
44
+ } from "@peerbit/pubsub";
35
45
  import {
36
46
  SubscriptionEvent,
37
47
  UnsubcriptionEvent,
@@ -40,10 +50,10 @@ import { RPC, type RequestContext } from "@peerbit/rpc";
40
50
  import {
41
51
  AcknowledgeDelivery,
42
52
  AnyWhere,
53
+ DataMessage,
54
+ MessageHeader,
43
55
  NotStartedError,
44
- SeekDelivery,
45
56
  SilentDelivery,
46
- type WithMode,
47
57
  } from "@peerbit/stream-interface";
48
58
  import {
49
59
  AbortError,
@@ -69,6 +79,7 @@ import {
69
79
  ResponseIPrune,
70
80
  createExchangeHeadsMessages,
71
81
  } from "./exchange-heads.js";
82
+ import { FanoutEnvelope } from "./fanout-envelope.js";
72
83
  import {
73
84
  MAX_U32,
74
85
  MAX_U64,
@@ -189,6 +200,36 @@ const getLatestEntry = (
189
200
  return latest;
190
201
  };
191
202
 
203
+ const hashToSeed32 = (str: string) => {
204
+ // FNV-1a 32-bit, fast and deterministic.
205
+ let hash = 0x811c9dc5;
206
+ for (let i = 0; i < str.length; i++) {
207
+ hash ^= str.charCodeAt(i);
208
+ hash = Math.imul(hash, 0x01000193);
209
+ }
210
+ return hash >>> 0;
211
+ };
212
+
213
+ const pickDeterministicSubset = (peers: string[], seed: number, max: number) => {
214
+ if (peers.length <= max) return peers;
215
+
216
+ const subset: string[] = [];
217
+ const used = new Set<string>();
218
+ let x = seed || 1;
219
+ while (subset.length < max) {
220
+ // xorshift32
221
+ x ^= x << 13;
222
+ x ^= x >>> 17;
223
+ x ^= x << 5;
224
+ const peer = peers[(x >>> 0) % peers.length];
225
+ if (!used.has(peer)) {
226
+ used.add(peer);
227
+ subset.push(peer);
228
+ }
229
+ }
230
+ return subset;
231
+ };
232
+
192
233
  export type ReplicationLimitsOptions =
193
234
  | Partial<ReplicationLimits>
194
235
  | { min?: number; max?: number };
@@ -373,6 +414,7 @@ export type SharedLogOptions<
373
414
  compatibility?: number;
374
415
  domain?: ReplicationDomainConstructor<D>;
375
416
  eagerBlocks?: boolean | { cacheSize?: number };
417
+ fanout?: SharedLogFanoutOptions;
376
418
  };
377
419
 
378
420
  export const DEFAULT_MIN_REPLICAS = 2;
@@ -385,6 +427,10 @@ export const WAIT_FOR_REPLICATOR_REQUEST_MIN_ATTEMPTS = 3;
385
427
  // Prefer making pruning robust without timing-based heuristics.
386
428
  export const WAIT_FOR_PRUNE_DELAY = 0;
387
429
  const PRUNE_DEBOUNCE_INTERVAL = 500;
430
+ const CHECKED_PRUNE_RESEND_INTERVAL_MIN_MS = 250;
431
+ const CHECKED_PRUNE_RESEND_INTERVAL_MAX_MS = 5_000;
432
+ const CHECKED_PRUNE_RETRY_MAX_ATTEMPTS = 3;
433
+ const CHECKED_PRUNE_RETRY_MAX_DELAY_MS = 30_000;
388
434
 
389
435
  // DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
390
436
  const RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL = 1000;
@@ -395,6 +441,17 @@ const RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR = 1e-3;
395
441
 
396
442
  const DEFAULT_DISTRIBUTION_DEBOUNCE_TIME = 500;
397
443
 
444
+ const DEFAULT_SHARED_LOG_FANOUT_CHANNEL_OPTIONS: Omit<
445
+ FanoutTreeChannelOptions,
446
+ "role"
447
+ > = {
448
+ msgRate: 30,
449
+ msgSize: 1024,
450
+ uploadLimitBps: 5_000_000,
451
+ maxChildren: 24,
452
+ repair: true,
453
+ };
454
+
398
455
  const getIdForDynamicRange = (publicKey: PublicSignKey) => {
399
456
  return sha256Sync(
400
457
  concat([publicKey.bytes, new TextEncoder().encode("dynamic")]),
@@ -424,13 +481,29 @@ export type DeliveryOptions = {
424
481
  signal?: AbortSignal;
425
482
  };
426
483
 
427
- export type SharedAppendOptions<T> = AppendOptions<T> & {
484
+ export type SharedLogFanoutOptions = {
485
+ root?: string;
486
+ channel?: Partial<Omit<FanoutTreeChannelOptions, "role">>;
487
+ join?: FanoutTreeJoinOptions;
488
+ };
489
+
490
+ type SharedAppendBaseOptions<T> = AppendOptions<T> & {
428
491
  replicas?: AbsoluteReplicas | number;
429
492
  replicate?: boolean;
430
- target?: "all" | "replicators" | "none";
431
- delivery?: false | true | DeliveryOptions;
432
493
  };
433
494
 
495
+ export type SharedAppendOptions<T> =
496
+ | (SharedAppendBaseOptions<T> & {
497
+ target?: "replicators" | "none";
498
+ delivery?: false | true | DeliveryOptions;
499
+ })
500
+ | (SharedAppendBaseOptions<T> & {
501
+ // target=all uses the fanout data plane and intentionally does not expose
502
+ // per-recipient settle semantics from RPC delivery options.
503
+ target: "all";
504
+ delivery?: false | undefined;
505
+ });
506
+
434
507
  export type ReplicatorJoinEvent = { publicKey: PublicSignKey };
435
508
  export type ReplicatorLeaveEvent = { publicKey: PublicSignKey };
436
509
  export type ReplicationChangeEvent = { publicKey: PublicSignKey };
@@ -463,11 +536,12 @@ export class SharedLog<
463
536
 
464
537
  private _replicationRangeIndex!: Index<ReplicationRangeIndexable<R>>;
465
538
  private _entryCoordinatesIndex!: Index<EntryReplicated<R>>;
466
- private coordinateToHash!: Cache<string>;
467
- private recentlyRebalanced!: Cache<string>;
539
+ private coordinateToHash!: Cache<string>;
540
+ private recentlyRebalanced!: Cache<string>;
468
541
 
469
- uniqueReplicators!: Set<string>;
470
- private _replicatorsReconciled!: boolean;
542
+ uniqueReplicators!: Set<string>;
543
+ private _replicatorJoinEmitted!: Set<string>;
544
+ private _replicatorsReconciled!: boolean;
471
545
 
472
546
  /* private _totalParticipation!: number; */
473
547
 
@@ -476,6 +550,10 @@ export class SharedLog<
476
550
 
477
551
  private _onSubscriptionFn!: (arg: any) => any;
478
552
  private _onUnsubscriptionFn!: (arg: any) => any;
553
+ private _onFanoutDataFn?: (arg: any) => void;
554
+ private _onFanoutUnicastFn?: (arg: any) => void;
555
+ private _fanoutChannel?: FanoutChannel;
556
+ private _providerHandle?: FanoutProviderHandle;
479
557
 
480
558
  private _isTrustedReplicator?: (
481
559
  publicKey: PublicSignKey,
@@ -519,6 +597,15 @@ export class SharedLog<
519
597
  >; // map of peerId to timeout
520
598
 
521
599
  private latestReplicationInfoMessage!: Map<string, bigint>;
600
+ // Peers that have unsubscribed from this log's topic. We ignore replication-info
601
+ // messages from them until we see a new subscription, to avoid re-introducing
602
+ // stale membership state during close/unsubscribe races.
603
+ private _replicationInfoBlockedPeers!: Set<string>;
604
+ private _replicationInfoRequestByPeer!: Map<
605
+ string,
606
+ { attempts: number; timer?: ReturnType<typeof setTimeout> }
607
+ >;
608
+ private _replicationInfoApplyQueueByPeer!: Map<string, Promise<void>>;
522
609
 
523
610
  private remoteBlocks!: RemoteBlocks;
524
611
 
@@ -552,6 +639,10 @@ export class SharedLog<
552
639
 
553
640
  private _requestIPruneSent!: Map<string, Set<string>>; // tracks entry hash to peer hash for requesting I prune messages
554
641
  private _requestIPruneResponseReplicatorSet!: Map<string, Set<string>>; // tracks entry hash to peer hash
642
+ private _checkedPruneRetries!: Map<
643
+ string,
644
+ { attempts: number; timer?: ReturnType<typeof setTimeout> }
645
+ >;
555
646
 
556
647
  private replicationChangeDebounceFn!: ReturnType<
557
648
  typeof debounceAggregationChanges<ReplicationRangeIndexable<R>>
@@ -597,6 +688,590 @@ export class SharedLog<
597
688
  return (this.compatibility ?? Number.MAX_VALUE) < 9;
598
689
  }
599
690
 
691
+ private getFanoutChannelOptions(
692
+ options?: SharedLogFanoutOptions,
693
+ ): Omit<FanoutTreeChannelOptions, "role"> {
694
+ return {
695
+ ...DEFAULT_SHARED_LOG_FANOUT_CHANNEL_OPTIONS,
696
+ ...(options?.channel ?? {}),
697
+ };
698
+ }
699
+
700
+ private async _openFanoutChannel(options?: SharedLogFanoutOptions) {
701
+ this._closeFanoutChannel();
702
+ if (!options) {
703
+ return;
704
+ }
705
+
706
+ const fanoutService = (this.node.services as any).fanout;
707
+ if (!fanoutService) {
708
+ throw new Error(
709
+ `Fanout is configured for shared-log topic ${this.topic}, but no fanout service is available on this client`,
710
+ );
711
+ }
712
+
713
+ const resolvedRoot =
714
+ options.root ??
715
+ (await (fanoutService as any)?.topicRootControlPlane?.resolveTopicRoot?.(
716
+ this.topic,
717
+ ));
718
+ if (!resolvedRoot) {
719
+ throw new Error(
720
+ `Fanout is configured for shared-log topic ${this.topic}, but no fanout root was provided and none could be resolved`,
721
+ );
722
+ }
723
+
724
+ const channel = new FanoutChannel(fanoutService, {
725
+ topic: this.topic,
726
+ root: resolvedRoot,
727
+ });
728
+ this._fanoutChannel = channel;
729
+
730
+ this._onFanoutDataFn =
731
+ this._onFanoutDataFn ||
732
+ ((evt: any) => {
733
+ const detail = (evt as CustomEvent<FanoutTreeDataEvent>)?.detail;
734
+ if (!detail) {
735
+ return;
736
+ }
737
+ void this._onFanoutData(detail).catch((error) => logger.error(error));
738
+ });
739
+ channel.addEventListener("data", this._onFanoutDataFn);
740
+
741
+ this._onFanoutUnicastFn =
742
+ this._onFanoutUnicastFn ||
743
+ ((evt: any) => {
744
+ const detail = (evt as CustomEvent<FanoutTreeUnicastEvent>)?.detail;
745
+ if (!detail) {
746
+ return;
747
+ }
748
+ void this._onFanoutUnicast(detail).catch((error) => logger.error(error));
749
+ });
750
+ channel.addEventListener("unicast", this._onFanoutUnicastFn);
751
+
752
+ try {
753
+ const channelOptions = this.getFanoutChannelOptions(options);
754
+ if (resolvedRoot === fanoutService.publicKeyHash) {
755
+ await channel.openAsRoot(channelOptions);
756
+ return;
757
+ }
758
+ await channel.join(channelOptions, options.join);
759
+ } catch (error) {
760
+ this._closeFanoutChannel();
761
+ throw error;
762
+ }
763
+ }
764
+
765
+ private _closeFanoutChannel() {
766
+ if (this._fanoutChannel) {
767
+ if (this._onFanoutDataFn) {
768
+ this._fanoutChannel.removeEventListener("data", this._onFanoutDataFn);
769
+ }
770
+ if (this._onFanoutUnicastFn) {
771
+ this._fanoutChannel.removeEventListener(
772
+ "unicast",
773
+ this._onFanoutUnicastFn,
774
+ );
775
+ }
776
+ this._fanoutChannel.close();
777
+ }
778
+ this._fanoutChannel = undefined;
779
+ }
780
+
781
+ private async _onFanoutData(detail: FanoutTreeDataEvent) {
782
+ let envelope: FanoutEnvelope;
783
+ try {
784
+ envelope = deserialize(detail.payload, FanoutEnvelope);
785
+ } catch (error) {
786
+ if (error instanceof BorshError) {
787
+ return;
788
+ }
789
+ throw error;
790
+ }
791
+
792
+ let message: TransportMessage;
793
+ try {
794
+ message = deserialize(envelope.payload, TransportMessage);
795
+ } catch (error) {
796
+ if (error instanceof BorshError) {
797
+ return;
798
+ }
799
+ throw error;
800
+ }
801
+
802
+ if (!(message instanceof ExchangeHeadsMessage)) {
803
+ return;
804
+ }
805
+
806
+ const from =
807
+ (await this._resolvePublicKeyFromHash(envelope.from)) ??
808
+ ({ hashcode: () => envelope.from } as PublicSignKey);
809
+
810
+ const contextMessage = new DataMessage({
811
+ header: new MessageHeader({
812
+ session: 0,
813
+ mode: new AnyWhere(),
814
+ priority: 0,
815
+ }),
816
+ });
817
+ contextMessage.header.timestamp = envelope.timestamp;
818
+
819
+ await this.onMessage(message, {
820
+ from,
821
+ message: contextMessage,
822
+ });
823
+ }
824
+
825
+ private async _onFanoutUnicast(detail: FanoutTreeUnicastEvent) {
826
+ let message: TransportMessage;
827
+ try {
828
+ message = deserialize(detail.payload, TransportMessage);
829
+ } catch (error) {
830
+ if (error instanceof BorshError) {
831
+ return;
832
+ }
833
+ throw error;
834
+ }
835
+
836
+ const fromHash = detail.origin || detail.from;
837
+ const from =
838
+ (await this._resolvePublicKeyFromHash(fromHash)) ??
839
+ ({ hashcode: () => fromHash } as PublicSignKey);
840
+
841
+ const contextMessage = new DataMessage({
842
+ header: new MessageHeader({
843
+ session: 0,
844
+ mode: new AnyWhere(),
845
+ priority: 0,
846
+ }),
847
+ });
848
+ contextMessage.header.timestamp = detail.timestamp;
849
+
850
+ await this.onMessage(message, {
851
+ from,
852
+ message: contextMessage,
853
+ });
854
+ }
855
+
856
+ private async _publishExchangeHeadsViaFanout(
857
+ message: ExchangeHeadsMessage<any>,
858
+ ): Promise<void> {
859
+ if (!this._fanoutChannel) {
860
+ throw new Error(
861
+ `No fanout channel configured for shared-log topic ${this.topic}`,
862
+ );
863
+ }
864
+ const envelope = new FanoutEnvelope({
865
+ from: this.node.identity.publicKey.hashcode(),
866
+ timestamp: BigInt(Date.now()),
867
+ payload: serialize(message),
868
+ });
869
+ await this._fanoutChannel.publish(serialize(envelope));
870
+ }
871
+
872
+ private _parseDeliveryOptions(
873
+ deliveryArg: false | true | DeliveryOptions | undefined,
874
+ ): {
875
+ delivery?: DeliveryOptions;
876
+ requireRecipients: boolean;
877
+ settleMin?: number;
878
+ wrap?: (promise: Promise<void>) => Promise<void>;
879
+ } {
880
+ const delivery: DeliveryOptions | undefined =
881
+ deliveryArg === undefined || deliveryArg === false
882
+ ? undefined
883
+ : deliveryArg === true
884
+ ? {}
885
+ : deliveryArg;
886
+ if (!delivery) {
887
+ return {
888
+ delivery: undefined,
889
+ requireRecipients: false,
890
+ settleMin: undefined,
891
+ wrap: undefined,
892
+ };
893
+ }
894
+
895
+ const deliverySettle = delivery.settle ?? true;
896
+ const deliveryTimeout = delivery.timeout;
897
+ const deliverySignal = delivery.signal;
898
+ const requireRecipients = delivery.requireRecipients === true;
899
+ const settleMin =
900
+ typeof deliverySettle === "object" && Number.isFinite(deliverySettle.min)
901
+ ? Math.max(0, Math.floor(deliverySettle.min))
902
+ : undefined;
903
+
904
+ const wrap =
905
+ deliveryTimeout == null && deliverySignal == null
906
+ ? undefined
907
+ : (promise: Promise<void>) =>
908
+ new Promise<void>((resolve, reject) => {
909
+ let settled = false;
910
+ let timer: ReturnType<typeof setTimeout> | undefined = undefined;
911
+ const onAbort = () => {
912
+ if (settled) {
913
+ return;
914
+ }
915
+ settled = true;
916
+ promise.catch(() => {});
917
+ cleanup();
918
+ reject(new AbortError());
919
+ };
920
+
921
+ const cleanup = () => {
922
+ if (timer != null) {
923
+ clearTimeout(timer);
924
+ timer = undefined;
925
+ }
926
+ deliverySignal?.removeEventListener("abort", onAbort);
927
+ };
928
+
929
+ if (deliverySignal) {
930
+ if (deliverySignal.aborted) {
931
+ onAbort();
932
+ return;
933
+ }
934
+ deliverySignal.addEventListener("abort", onAbort);
935
+ }
936
+
937
+ if (deliveryTimeout != null) {
938
+ timer = setTimeout(() => {
939
+ if (settled) {
940
+ return;
941
+ }
942
+ settled = true;
943
+ promise.catch(() => {});
944
+ cleanup();
945
+ reject(new TimeoutError(`Timeout waiting for delivery`));
946
+ }, deliveryTimeout);
947
+ }
948
+
949
+ promise
950
+ .then(() => {
951
+ if (settled) {
952
+ return;
953
+ }
954
+ settled = true;
955
+ cleanup();
956
+ resolve();
957
+ })
958
+ .catch((error) => {
959
+ if (settled) {
960
+ return;
961
+ }
962
+ settled = true;
963
+ cleanup();
964
+ reject(error);
965
+ });
966
+ });
967
+
968
+ return {
969
+ delivery,
970
+ requireRecipients,
971
+ settleMin,
972
+ wrap,
973
+ };
974
+ }
975
+
976
+ private async _appendDeliverToReplicators(
977
+ entry: Entry<T>,
978
+ minReplicasValue: number,
979
+ leaders: Map<string, any>,
980
+ selfHash: string,
981
+ isLeader: boolean,
982
+ deliveryArg: false | true | DeliveryOptions | undefined,
983
+ ) {
984
+ const { delivery, requireRecipients, settleMin, wrap } =
985
+ this._parseDeliveryOptions(deliveryArg);
986
+ const pending: Promise<void>[] = [];
987
+ const track = (promise: Promise<void>) => {
988
+ pending.push(wrap ? wrap(promise) : promise);
989
+ };
990
+ const fanoutUnicastOptions =
991
+ delivery?.timeout != null || delivery?.signal != null
992
+ ? { timeoutMs: delivery.timeout, signal: delivery.signal }
993
+ : undefined;
994
+
995
+ for await (const message of createExchangeHeadsMessages(this.log, [entry])) {
996
+ await this._mergeLeadersFromGidReferences(message, minReplicasValue, leaders);
997
+ const leadersForDelivery = delivery ? new Set(leaders.keys()) : undefined;
998
+
999
+ const set = this.addPeersToGidPeerHistory(entry.meta.gid, leaders.keys());
1000
+ const hasRemotePeers = set.has(selfHash) ? set.size > 1 : set.size > 0;
1001
+ if (!hasRemotePeers) {
1002
+ if (requireRecipients) {
1003
+ throw new NoPeersError(this.rpc.topic);
1004
+ }
1005
+ continue;
1006
+ }
1007
+
1008
+ if (!delivery) {
1009
+ this.rpc
1010
+ .send(message, {
1011
+ mode: isLeader
1012
+ ? new SilentDelivery({ redundancy: 1, to: set })
1013
+ : new AcknowledgeDelivery({ redundancy: 1, to: set }),
1014
+ })
1015
+ .catch((error) => logger.error(error));
1016
+ continue;
1017
+ }
1018
+
1019
+ const orderedRemoteRecipients: string[] = [];
1020
+ for (const peer of leadersForDelivery!) {
1021
+ if (peer === selfHash) {
1022
+ continue;
1023
+ }
1024
+ orderedRemoteRecipients.push(peer);
1025
+ }
1026
+ for (const peer of set) {
1027
+ if (peer === selfHash) {
1028
+ continue;
1029
+ }
1030
+ if (leadersForDelivery!.has(peer)) {
1031
+ continue;
1032
+ }
1033
+ orderedRemoteRecipients.push(peer);
1034
+ }
1035
+
1036
+ const ackTo: string[] = [];
1037
+ let silentTo: string[] | undefined;
1038
+ // Default delivery semantics: require enough remote ACKs to reach the requested
1039
+ // replication degree (local append counts as 1).
1040
+ const ackLimit =
1041
+ settleMin == null ? Math.max(0, minReplicasValue - 1) : settleMin;
1042
+
1043
+ for (const peer of orderedRemoteRecipients) {
1044
+ if (ackTo.length < ackLimit) {
1045
+ ackTo.push(peer);
1046
+ } else {
1047
+ silentTo ||= [];
1048
+ silentTo.push(peer);
1049
+ }
1050
+ }
1051
+
1052
+ if (requireRecipients && orderedRemoteRecipients.length === 0) {
1053
+ throw new NoPeersError(this.rpc.topic);
1054
+ }
1055
+ if (requireRecipients && ackTo.length + (silentTo?.length || 0) === 0) {
1056
+ throw new NoPeersError(this.rpc.topic);
1057
+ }
1058
+
1059
+ if (ackTo.length > 0) {
1060
+ const payload = serialize(message);
1061
+ for (const peer of ackTo) {
1062
+ track(
1063
+ (async () => {
1064
+ // Unified decision point:
1065
+ // - If we can prove a cheap direct path (connected or routed), use it.
1066
+ // - Otherwise, fall back to the fanout unicast ACK path (bounded overlay routing).
1067
+ // - If that fails, fall back to pubsub/RPC routing which may flood to discover routes.
1068
+ const pubsub: any = this.node.services.pubsub as any;
1069
+ const canDirectFast =
1070
+ Boolean(pubsub?.peers?.get?.(peer)?.isWritable) ||
1071
+ Boolean(
1072
+ pubsub?.routes?.isReachable?.(
1073
+ pubsub?.publicKeyHash,
1074
+ peer,
1075
+ 0,
1076
+ ),
1077
+ );
1078
+
1079
+ if (canDirectFast) {
1080
+ await this.rpc.send(message, {
1081
+ mode: new AcknowledgeDelivery({
1082
+ redundancy: 1,
1083
+ to: [peer],
1084
+ }),
1085
+ });
1086
+ return;
1087
+ }
1088
+
1089
+ if (this._fanoutChannel) {
1090
+ try {
1091
+ await this._fanoutChannel.unicastToAck(
1092
+ peer,
1093
+ payload,
1094
+ fanoutUnicastOptions,
1095
+ );
1096
+ return;
1097
+ } catch {
1098
+ // fall back below
1099
+ }
1100
+ }
1101
+ await this.rpc.send(message, {
1102
+ mode: new AcknowledgeDelivery({
1103
+ redundancy: 1,
1104
+ to: [peer],
1105
+ }),
1106
+ });
1107
+ })(),
1108
+ );
1109
+ }
1110
+ }
1111
+
1112
+ if (silentTo?.length) {
1113
+ this.rpc
1114
+ .send(message, {
1115
+ mode: new SilentDelivery({ redundancy: 1, to: silentTo }),
1116
+ })
1117
+ .catch((error) => logger.error(error));
1118
+ }
1119
+ }
1120
+
1121
+ if (pending.length > 0) {
1122
+ await Promise.all(pending);
1123
+ }
1124
+ }
1125
+
1126
+ private async _mergeLeadersFromGidReferences(
1127
+ message: ExchangeHeadsMessage<any>,
1128
+ minReplicasValue: number,
1129
+ leaders: Map<string, any>,
1130
+ ) {
1131
+ const gidReferences = message.heads[0]?.gidRefrences;
1132
+ if (!gidReferences || gidReferences.length === 0) {
1133
+ return;
1134
+ }
1135
+
1136
+ for (const gidReference of gidReferences) {
1137
+ const entryFromGid = this.log.entryIndex.getHeads(gidReference, false);
1138
+ for (const gidEntry of await entryFromGid.all()) {
1139
+ let coordinates = await this.getCoordinates(gidEntry);
1140
+ if (coordinates == null) {
1141
+ coordinates = await this.createCoordinates(gidEntry, minReplicasValue);
1142
+ }
1143
+
1144
+ const found = await this._findLeaders(coordinates);
1145
+ for (const [key, value] of found) {
1146
+ leaders.set(key, value);
1147
+ }
1148
+ }
1149
+ }
1150
+ }
1151
+
1152
+ private async _appendDeliverToAllFanout(entry: Entry<T>) {
1153
+ for await (const message of createExchangeHeadsMessages(this.log, [entry])) {
1154
+ await this._publishExchangeHeadsViaFanout(message);
1155
+ }
1156
+ }
1157
+
1158
+ private async _resolvePublicKeyFromHash(
1159
+ hash: string,
1160
+ ): Promise<PublicSignKey | undefined> {
1161
+ const fanoutService = (this.node.services as any).fanout;
1162
+ return (
1163
+ fanoutService?.getPublicKey?.(hash) ??
1164
+ this.node.services.pubsub.getPublicKey(hash)
1165
+ );
1166
+ }
1167
+
1168
+ private async _getTopicSubscribers(
1169
+ topic: string,
1170
+ ): Promise<PublicSignKey[] | undefined> {
1171
+ const maxPeers = 64;
1172
+
1173
+ // Prefer the bounded peer set we already know from the fanout overlay.
1174
+ if (this._fanoutChannel && (topic === this.topic || topic === this.rpc.topic)) {
1175
+ const hashes = this._fanoutChannel
1176
+ .getPeerHashes({ includeSelf: false })
1177
+ .slice(0, maxPeers);
1178
+ if (hashes.length === 0) return [];
1179
+
1180
+ const keys = await Promise.all(
1181
+ hashes.map((hash) => this._resolvePublicKeyFromHash(hash)),
1182
+ );
1183
+ const uniqueKeys: PublicSignKey[] = [];
1184
+ const seen = new Set<string>();
1185
+ const selfHash = this.node.identity.publicKey.hashcode();
1186
+ for (const key of keys) {
1187
+ if (!key) continue;
1188
+ const hash = key.hashcode();
1189
+ if (hash === selfHash) continue;
1190
+ if (seen.has(hash)) continue;
1191
+ seen.add(hash);
1192
+ uniqueKeys.push(key);
1193
+ }
1194
+ return uniqueKeys;
1195
+ }
1196
+
1197
+ const selfHash = this.node.identity.publicKey.hashcode();
1198
+ const hashes: string[] = [];
1199
+
1200
+ // Best-effort provider discovery (bounded). This requires bootstrap trackers.
1201
+ try {
1202
+ const fanoutService = (this.node.services as any).fanout;
1203
+ if (fanoutService?.queryProviders) {
1204
+ const ns = `shared-log|${this.topic}`;
1205
+ const seed = hashToSeed32(topic);
1206
+ const providers: string[] = await fanoutService.queryProviders(ns, {
1207
+ want: maxPeers,
1208
+ seed,
1209
+ });
1210
+ for (const h of providers ?? []) {
1211
+ if (!h || h === selfHash) continue;
1212
+ hashes.push(h);
1213
+ if (hashes.length >= maxPeers) break;
1214
+ }
1215
+ }
1216
+ } catch {
1217
+ // Best-effort only.
1218
+ }
1219
+
1220
+ // Next, use already-connected peer streams (bounded and cheap).
1221
+ const peerMap: Map<string, unknown> | undefined = (this.node.services.pubsub as any)
1222
+ ?.peers;
1223
+ if (peerMap?.keys) {
1224
+ for (const h of peerMap.keys()) {
1225
+ if (!h || h === selfHash) continue;
1226
+ hashes.push(h);
1227
+ if (hashes.length >= maxPeers) break;
1228
+ }
1229
+ }
1230
+
1231
+ // Finally, fall back to libp2p connections (e.g. bootstrap peers) without requiring
1232
+ // any global topic membership view.
1233
+ if (hashes.length < maxPeers) {
1234
+ const connectionManager = (this.node.services.pubsub as any)?.components
1235
+ ?.connectionManager;
1236
+ const connections = connectionManager?.getConnections?.() ?? [];
1237
+ for (const conn of connections) {
1238
+ const peerId = conn?.remotePeer;
1239
+ if (!peerId) continue;
1240
+ try {
1241
+ const h = getPublicKeyFromPeerId(peerId).hashcode();
1242
+ if (!h || h === selfHash) continue;
1243
+ hashes.push(h);
1244
+ if (hashes.length >= maxPeers) break;
1245
+ } catch {
1246
+ // Best-effort only.
1247
+ }
1248
+ }
1249
+ }
1250
+
1251
+ if (hashes.length === 0) return [];
1252
+
1253
+ const uniqueHashes: string[] = [];
1254
+ const seen = new Set<string>();
1255
+ for (const h of hashes) {
1256
+ if (seen.has(h)) continue;
1257
+ seen.add(h);
1258
+ uniqueHashes.push(h);
1259
+ if (uniqueHashes.length >= maxPeers) break;
1260
+ }
1261
+
1262
+ const keys = await Promise.all(
1263
+ uniqueHashes.map((hash) => this._resolvePublicKeyFromHash(hash)),
1264
+ );
1265
+ const uniqueKeys: PublicSignKey[] = [];
1266
+ for (const key of keys) {
1267
+ if (!key) continue;
1268
+ const hash = key.hashcode();
1269
+ if (hash === selfHash) continue;
1270
+ uniqueKeys.push(key);
1271
+ }
1272
+ return uniqueKeys;
1273
+ }
1274
+
600
1275
  // @deprecated
601
1276
  private async getRole() {
602
1277
  const segments = await this.getMyReplicationSegments();
@@ -1004,8 +1679,9 @@ export class SharedLog<
1004
1679
  })
1005
1680
  .all();
1006
1681
 
1007
- this.uniqueReplicators.delete(keyHash);
1008
- await this.replicationIndex.del({ query: { hash: keyHash } });
1682
+ this.uniqueReplicators.delete(keyHash);
1683
+ this._replicatorJoinEmitted.delete(keyHash);
1684
+ await this.replicationIndex.del({ query: { hash: keyHash } });
1009
1685
 
1010
1686
  await this.updateOldestTimestampFromIndex();
1011
1687
 
@@ -1030,14 +1706,14 @@ export class SharedLog<
1030
1706
  }
1031
1707
  }
1032
1708
 
1033
- const timestamp = BigInt(+new Date());
1034
- for (const x of deleted) {
1035
- this.replicationChangeDebounceFn.add({
1036
- range: x.value,
1037
- type: "removed",
1038
- timestamp,
1039
- });
1040
- }
1709
+ const timestamp = BigInt(+new Date());
1710
+ for (const x of deleted) {
1711
+ this.replicationChangeDebounceFn.add({
1712
+ range: x.value,
1713
+ type: "removed",
1714
+ timestamp,
1715
+ });
1716
+ }
1041
1717
 
1042
1718
  const pendingMaturity = this.pendingMaturity.get(keyHash);
1043
1719
  if (pendingMaturity) {
@@ -1118,9 +1794,10 @@ export class SharedLog<
1118
1794
  { query: { hash: from.hashcode() } },
1119
1795
  { shape: { id: true } },
1120
1796
  );
1121
- if ((await otherSegmentsIterator.next(1)).length === 0) {
1122
- this.uniqueReplicators.delete(from.hashcode());
1123
- }
1797
+ if ((await otherSegmentsIterator.next(1)).length === 0) {
1798
+ this.uniqueReplicators.delete(from.hashcode());
1799
+ this._replicatorJoinEmitted.delete(from.hashcode());
1800
+ }
1124
1801
  await otherSegmentsIterator.close();
1125
1802
 
1126
1803
  await this.updateOldestTimestampFromIndex();
@@ -1160,6 +1837,7 @@ export class SharedLog<
1160
1837
 
1161
1838
  let diffs: ReplicationChanges<ReplicationRangeIndexable<R>>;
1162
1839
  let deleted: ReplicationRangeIndexable<R>[] | undefined = undefined;
1840
+ let isStoppedReplicating = false;
1163
1841
  if (reset) {
1164
1842
  deleted = (
1165
1843
  await this.replicationIndex
@@ -1198,6 +1876,7 @@ export class SharedLog<
1198
1876
  }
1199
1877
 
1200
1878
  isNewReplicator = prevCount === 0 && ranges.length > 0;
1879
+ isStoppedReplicating = prevCount > 0 && ranges.length === 0;
1201
1880
  } else {
1202
1881
  let batchSize = 100;
1203
1882
  let existing: ReplicationRangeIndexable<R>[] = [];
@@ -1281,7 +1960,16 @@ export class SharedLog<
1281
1960
  diffs = changes;
1282
1961
  }
1283
1962
 
1284
- this.uniqueReplicators.add(from.hashcode());
1963
+ const fromHash = from.hashcode();
1964
+ // Track replicator membership transitions synchronously so join/leave events are
1965
+ // idempotent even if we process concurrent reset messages/unsubscribes.
1966
+ const stoppedTransition =
1967
+ ranges.length === 0 ? this.uniqueReplicators.delete(fromHash) : false;
1968
+ if (ranges.length === 0) {
1969
+ this._replicatorJoinEmitted.delete(fromHash);
1970
+ } else {
1971
+ this.uniqueReplicators.add(fromHash);
1972
+ }
1285
1973
 
1286
1974
  let now = +new Date();
1287
1975
  let minRoleAge = await this.getDefaultMinRoleAge();
@@ -1327,13 +2015,13 @@ export class SharedLog<
1327
2015
  }),
1328
2016
  );
1329
2017
 
1330
- if (rebalance && diff.range.mode !== ReplicationIntent.Strict) {
1331
- // TODO this statement (might) cause issues with triggering pruning if the segment is strict and maturity timings will affect the outcome of rebalancing
1332
- this.replicationChangeDebounceFn.add({
1333
- ...diff,
1334
- matured: true,
1335
- }); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
1336
- }
2018
+ if (rebalance && diff.range.mode !== ReplicationIntent.Strict) {
2019
+ // TODO this statement (might) cause issues with triggering pruning if the segment is strict and maturity timings will affect the outcome of rebalancing
2020
+ this.replicationChangeDebounceFn.add({
2021
+ ...diff,
2022
+ matured: true,
2023
+ }); // we need to call this here because the outcom of findLeaders will be different when some ranges become mature, i.e. some of data we own might be prunable!
2024
+ }
1337
2025
  pendingRanges.delete(diff.range.idString);
1338
2026
  if (pendingRanges.size === 0) {
1339
2027
  this.pendingMaturity.delete(diff.range.hash);
@@ -1379,28 +2067,39 @@ export class SharedLog<
1379
2067
  }),
1380
2068
  );
1381
2069
 
1382
- if (isNewReplicator) {
1383
- this.events.dispatchEvent(
1384
- new CustomEvent<ReplicatorJoinEvent>("replicator:join", {
1385
- detail: { publicKey: from },
1386
- }),
1387
- );
2070
+ if (isNewReplicator) {
2071
+ if (!this._replicatorJoinEmitted.has(fromHash)) {
2072
+ this._replicatorJoinEmitted.add(fromHash);
2073
+ this.events.dispatchEvent(
2074
+ new CustomEvent<ReplicatorJoinEvent>("replicator:join", {
2075
+ detail: { publicKey: from },
2076
+ }),
2077
+ );
2078
+ }
1388
2079
 
1389
- if (isAllMature) {
1390
- this.events.dispatchEvent(
1391
- new CustomEvent<ReplicatorMatureEvent>("replicator:mature", {
1392
- detail: { publicKey: from },
2080
+ if (isAllMature) {
2081
+ this.events.dispatchEvent(
2082
+ new CustomEvent<ReplicatorMatureEvent>("replicator:mature", {
2083
+ detail: { publicKey: from },
1393
2084
  }),
1394
2085
  );
1395
2086
  }
1396
2087
  }
1397
2088
 
1398
- if (rebalance) {
1399
- for (const diff of diffs) {
1400
- this.replicationChangeDebounceFn.add(diff);
1401
- }
2089
+ if (isStoppedReplicating && stoppedTransition) {
2090
+ this.events.dispatchEvent(
2091
+ new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
2092
+ detail: { publicKey: from },
2093
+ }),
2094
+ );
1402
2095
  }
1403
2096
 
2097
+ if (rebalance) {
2098
+ for (const diff of diffs) {
2099
+ this.replicationChangeDebounceFn.add(diff);
2100
+ }
2101
+ }
2102
+
1404
2103
  if (!from.equals(this.node.identity.publicKey)) {
1405
2104
  this.rebalanceParticipationDebounced?.call();
1406
2105
  }
@@ -1432,6 +2131,20 @@ export class SharedLog<
1432
2131
  if (change) {
1433
2132
  let addedOrReplaced = change.filter((x) => x.type !== "removed");
1434
2133
  if (addedOrReplaced.length > 0) {
2134
+ // Provider discovery keep-alive (best-effort). This enables bounded targeted fetches
2135
+ // without relying on any global subscriber list.
2136
+ try {
2137
+ const fanoutService = (this.node.services as any).fanout;
2138
+ if (fanoutService?.provide && !this._providerHandle) {
2139
+ this._providerHandle = fanoutService.provide(`shared-log|${this.topic}`, {
2140
+ ttlMs: 120_000,
2141
+ announceIntervalMs: 60_000,
2142
+ });
2143
+ }
2144
+ } catch {
2145
+ // Best-effort only.
2146
+ }
2147
+
1435
2148
  let message:
1436
2149
  | AllReplicatingSegmentsMessage
1437
2150
  | AddedReplicationSegmentMessage
@@ -1506,6 +2219,82 @@ export class SharedLog<
1506
2219
  }
1507
2220
  }
1508
2221
 
2222
+ private clearCheckedPruneRetry(hash: string) {
2223
+ const state = this._checkedPruneRetries.get(hash);
2224
+ if (state?.timer) {
2225
+ clearTimeout(state.timer);
2226
+ }
2227
+ this._checkedPruneRetries.delete(hash);
2228
+ }
2229
+
2230
+ private scheduleCheckedPruneRetry(args: {
2231
+ entry: EntryReplicated<R> | ShallowOrFullEntry<any>;
2232
+ leaders: Map<string, unknown> | Set<string>;
2233
+ }) {
2234
+ if (this.closed) return;
2235
+ if (this._pendingDeletes.has(args.entry.hash)) return;
2236
+
2237
+ const hash = args.entry.hash;
2238
+ const state =
2239
+ this._checkedPruneRetries.get(hash) ?? { attempts: 0 };
2240
+
2241
+ if (state.timer) return;
2242
+ if (state.attempts >= CHECKED_PRUNE_RETRY_MAX_ATTEMPTS) {
2243
+ // Avoid unbounded background retries; a new replication-change event can
2244
+ // always re-enqueue pruning with fresh leader info.
2245
+ return;
2246
+ }
2247
+
2248
+ const attempt = state.attempts + 1;
2249
+ const jitterMs = Math.floor(Math.random() * 250);
2250
+ const delayMs = Math.min(
2251
+ CHECKED_PRUNE_RETRY_MAX_DELAY_MS,
2252
+ 1_000 * 2 ** (attempt - 1) + jitterMs,
2253
+ );
2254
+
2255
+ state.attempts = attempt;
2256
+ state.timer = setTimeout(async () => {
2257
+ const st = this._checkedPruneRetries.get(hash);
2258
+ if (st) st.timer = undefined;
2259
+ if (this.closed) return;
2260
+ if (this._pendingDeletes.has(hash)) return;
2261
+
2262
+ let leadersMap: Map<string, any> | undefined;
2263
+ try {
2264
+ const replicas = decodeReplicas(args.entry).getValue(this);
2265
+ leadersMap = await this.findLeadersFromEntry(args.entry, replicas, {
2266
+ roleAge: 0,
2267
+ });
2268
+ } catch {
2269
+ // Best-effort only.
2270
+ }
2271
+
2272
+ if (!leadersMap || leadersMap.size === 0) {
2273
+ if (args.leaders instanceof Map) {
2274
+ leadersMap = args.leaders as any;
2275
+ } else {
2276
+ leadersMap = new Map<string, any>();
2277
+ for (const k of args.leaders) {
2278
+ leadersMap.set(k, { intersecting: true });
2279
+ }
2280
+ }
2281
+ }
2282
+
2283
+ try {
2284
+ const leadersForRetry = leadersMap ?? new Map<string, any>();
2285
+ await this.pruneDebouncedFnAddIfNotKeeping({
2286
+ key: hash,
2287
+ // TODO types
2288
+ value: { entry: args.entry as any, leaders: leadersForRetry },
2289
+ });
2290
+ } catch {
2291
+ // Best-effort only; pruning will be re-attempted on future changes.
2292
+ }
2293
+ }, delayMs);
2294
+ state.timer.unref?.();
2295
+ this._checkedPruneRetries.set(hash, state);
2296
+ }
2297
+
1509
2298
  async append(
1510
2299
  data: T,
1511
2300
  options?: SharedAppendOptions<T> | undefined,
@@ -1571,286 +2360,30 @@ export class SharedLog<
1571
2360
  if (options?.target !== "none") {
1572
2361
  const target = options?.target;
1573
2362
  const deliveryArg = options?.delivery;
1574
- const delivery: DeliveryOptions | undefined =
1575
- deliveryArg === undefined || deliveryArg === false
1576
- ? undefined
1577
- : deliveryArg === true
1578
- ? {}
1579
- : deliveryArg;
1580
-
1581
- let requireRecipients = false;
1582
- let settleMin: number | undefined;
1583
- let guardDelivery:
1584
- | ((promise: Promise<void>) => Promise<void>)
1585
- | undefined = undefined;
1586
-
1587
- let firstDeliveryPromise: Promise<void> | undefined;
1588
- let deliveryPromises: Promise<void>[] | undefined;
1589
- let addDeliveryPromise: ((promise: Promise<void>) => void) | undefined;
1590
-
1591
- const leadersForDelivery =
1592
- delivery && (target === "replicators" || !target)
1593
- ? new Set(leaders.keys())
1594
- : undefined;
2363
+ const hasDelivery = !(deliveryArg === undefined || deliveryArg === false);
1595
2364
 
1596
- if (delivery) {
1597
- const deliverySettle = delivery.settle ?? true;
1598
- const deliveryTimeout = delivery.timeout;
1599
- const deliverySignal = delivery.signal;
1600
- requireRecipients = delivery.requireRecipients === true;
1601
- settleMin =
1602
- typeof deliverySettle === "object" &&
1603
- Number.isFinite(deliverySettle.min)
1604
- ? Math.max(0, Math.floor(deliverySettle.min))
1605
- : undefined;
1606
-
1607
- guardDelivery =
1608
- deliveryTimeout == null && deliverySignal == null
1609
- ? undefined
1610
- : (promise: Promise<void>) =>
1611
- new Promise<void>((resolve, reject) => {
1612
- let settled = false;
1613
- let timer: ReturnType<typeof setTimeout> | undefined =
1614
- undefined;
1615
- const onAbort = () => {
1616
- if (settled) {
1617
- return;
1618
- }
1619
- settled = true;
1620
- promise.catch(() => {});
1621
- cleanup();
1622
- reject(new AbortError());
1623
- };
1624
-
1625
- const cleanup = () => {
1626
- if (timer != null) {
1627
- clearTimeout(timer);
1628
- timer = undefined;
1629
- }
1630
- deliverySignal?.removeEventListener("abort", onAbort);
1631
- };
1632
-
1633
- if (deliverySignal) {
1634
- if (deliverySignal.aborted) {
1635
- onAbort();
1636
- return;
1637
- }
1638
- deliverySignal.addEventListener("abort", onAbort);
1639
- }
1640
-
1641
- if (deliveryTimeout != null) {
1642
- timer = setTimeout(() => {
1643
- if (settled) {
1644
- return;
1645
- }
1646
- settled = true;
1647
- promise.catch(() => {});
1648
- cleanup();
1649
- reject(new TimeoutError(`Timeout waiting for delivery`));
1650
- }, deliveryTimeout);
1651
- }
1652
-
1653
- promise
1654
- .then(() => {
1655
- if (settled) {
1656
- return;
1657
- }
1658
- settled = true;
1659
- cleanup();
1660
- resolve();
1661
- })
1662
- .catch((e) => {
1663
- if (settled) {
1664
- return;
1665
- }
1666
- settled = true;
1667
- cleanup();
1668
- reject(e);
1669
- });
1670
- });
1671
-
1672
- addDeliveryPromise = (promise: Promise<void>) => {
1673
- if (!firstDeliveryPromise) {
1674
- firstDeliveryPromise = promise;
1675
- return;
1676
- }
1677
- if (!deliveryPromises) {
1678
- deliveryPromises = [firstDeliveryPromise, promise];
1679
- firstDeliveryPromise = undefined;
1680
- return;
1681
- }
1682
- deliveryPromises.push(promise);
1683
- };
2365
+ if (target === "all" && hasDelivery) {
2366
+ throw new Error(
2367
+ `delivery options are not supported with target="all"; fanout broadcast is fire-and-forward`,
2368
+ );
1684
2369
  }
1685
-
1686
- for await (const message of createExchangeHeadsMessages(this.log, [
1687
- result.entry,
1688
- ])) {
1689
- if (target === "replicators" || !target) {
1690
- if (message.heads[0].gidRefrences.length > 0) {
1691
- for (const ref of message.heads[0].gidRefrences) {
1692
- const entryFromGid = this.log.entryIndex.getHeads(ref, false);
1693
- for (const entry of await entryFromGid.all()) {
1694
- let coordinates = await this.getCoordinates(entry);
1695
- if (coordinates == null) {
1696
- coordinates = await this.createCoordinates(
1697
- entry,
1698
- minReplicasValue,
1699
- );
1700
- // TODO are we every to come here?
1701
- }
1702
-
1703
- const result = await this._findLeaders(coordinates);
1704
- for (const [k, v] of result) {
1705
- leaders.set(k, v);
1706
- }
1707
- }
1708
- }
1709
- }
1710
-
1711
- const set = this.addPeersToGidPeerHistory(
1712
- result.entry.meta.gid,
1713
- leaders.keys(),
1714
- );
1715
- let hasRemotePeers = set.has(selfHash) ? set.size > 1 : set.size > 0;
1716
- if (!hasRemotePeers) {
1717
- if (requireRecipients) {
1718
- throw new NoPeersError(this.rpc.topic);
1719
- }
1720
- continue;
1721
- }
1722
-
1723
- if (!delivery) {
1724
- this.rpc
1725
- .send(message, {
1726
- mode: isLeader
1727
- ? new SilentDelivery({ redundancy: 1, to: set })
1728
- : new AcknowledgeDelivery({ redundancy: 1, to: set }),
1729
- })
1730
- .catch((e) => logger.error(e));
1731
- continue;
1732
- }
1733
-
1734
- let expectedRemoteRecipientsCount = 0;
1735
- const ackTo: string[] = [];
1736
- let silentTo: string[] | undefined;
1737
- const ackLimit =
1738
- settleMin == null ? Number.POSITIVE_INFINITY : settleMin;
1739
-
1740
- // Always settle towards the current expected replicators for this entry,
1741
- // not the entire gid peer history.
1742
- for (const peer of leadersForDelivery!) {
1743
- if (peer === selfHash) {
1744
- continue;
1745
- }
1746
- expectedRemoteRecipientsCount++;
1747
- if (ackTo.length < ackLimit) {
1748
- ackTo.push(peer);
1749
- } else {
1750
- silentTo ||= [];
1751
- silentTo.push(peer);
1752
- }
1753
- }
1754
-
1755
- // Still deliver to known peers for the gid (best-effort), but don't let them
1756
- // satisfy the settle requirement.
1757
- for (const peer of set) {
1758
- if (peer === selfHash) {
1759
- continue;
1760
- }
1761
- if (leadersForDelivery!.has(peer)) {
1762
- continue;
1763
- }
1764
- silentTo ||= [];
1765
- silentTo.push(peer);
1766
- }
1767
-
1768
- if (requireRecipients && expectedRemoteRecipientsCount === 0) {
1769
- throw new NoPeersError(this.rpc.topic);
1770
- }
1771
-
1772
- if (
1773
- requireRecipients &&
1774
- ackTo.length + (silentTo?.length || 0) === 0
1775
- ) {
1776
- throw new NoPeersError(this.rpc.topic);
1777
- }
1778
-
1779
- if (ackTo.length > 0) {
1780
- const promise = this.rpc.send(message, {
1781
- mode: new AcknowledgeDelivery({
1782
- redundancy: 1,
1783
- to: ackTo,
1784
- }),
1785
- });
1786
- addDeliveryPromise!(
1787
- guardDelivery ? guardDelivery(promise) : promise,
1788
- );
1789
- }
1790
-
1791
- if (silentTo?.length) {
1792
- this.rpc
1793
- .send(message, {
1794
- mode: new SilentDelivery({ redundancy: 1, to: silentTo }),
1795
- })
1796
- .catch((e) => logger.error(e));
1797
- }
1798
- } else {
1799
- if (!delivery) {
1800
- this.rpc.send(message).catch((e) => logger.error(e));
1801
- continue;
1802
- }
1803
-
1804
- const subscribers = await this.node.services.pubsub.getSubscribers(
1805
- this.rpc.topic,
1806
- );
1807
-
1808
- const ackTo: PublicSignKey[] = [];
1809
- let silentTo: PublicSignKey[] | undefined;
1810
- const ackLimit =
1811
- settleMin == null ? Number.POSITIVE_INFINITY : settleMin;
1812
- for (const subscriber of subscribers || []) {
1813
- if (subscriber.hashcode() === selfHash) {
1814
- continue;
1815
- }
1816
- if (ackTo.length < ackLimit) {
1817
- ackTo.push(subscriber);
1818
- } else {
1819
- silentTo ||= [];
1820
- silentTo.push(subscriber);
1821
- }
1822
- }
1823
-
1824
- if (
1825
- requireRecipients &&
1826
- ackTo.length + (silentTo?.length || 0) === 0
1827
- ) {
1828
- throw new NoPeersError(this.rpc.topic);
1829
- }
1830
-
1831
- if (ackTo.length > 0) {
1832
- const promise = this.rpc.send(message, {
1833
- mode: new AcknowledgeDelivery({ redundancy: 1, to: ackTo }),
1834
- });
1835
- addDeliveryPromise!(
1836
- guardDelivery ? guardDelivery(promise) : promise,
1837
- );
1838
- }
1839
-
1840
- if (silentTo?.length) {
1841
- this.rpc
1842
- .send(message, {
1843
- mode: new SilentDelivery({ redundancy: 1, to: silentTo }),
1844
- })
1845
- .catch((e) => logger.error(e));
1846
- }
1847
- }
2370
+ if (target === "all" && !this._fanoutChannel) {
2371
+ throw new Error(
2372
+ `No fanout channel configured for shared-log topic ${this.topic}`,
2373
+ );
1848
2374
  }
1849
2375
 
1850
- if (deliveryPromises) {
1851
- await Promise.all(deliveryPromises);
1852
- } else if (firstDeliveryPromise) {
1853
- await firstDeliveryPromise;
2376
+ if (target === "all") {
2377
+ await this._appendDeliverToAllFanout(result.entry);
2378
+ } else {
2379
+ await this._appendDeliverToReplicators(
2380
+ result.entry,
2381
+ minReplicasValue,
2382
+ leaders,
2383
+ selfHash,
2384
+ isLeader,
2385
+ deliveryArg,
2386
+ );
1854
2387
  }
1855
2388
  }
1856
2389
 
@@ -1891,14 +2424,18 @@ export class SharedLog<
1891
2424
  this.domain.resolution,
1892
2425
  );
1893
2426
  this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 2e4;
1894
- this._pendingDeletes = new Map();
1895
- this._pendingIHave = new Map();
1896
- this.latestReplicationInfoMessage = new Map();
1897
- this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
1898
- this.recentlyRebalanced = new Cache<string>({ max: 1e4, ttl: 1e5 });
1899
-
1900
- this.uniqueReplicators = new Set();
1901
- this._replicatorsReconciled = false;
2427
+ this._pendingDeletes = new Map();
2428
+ this._pendingIHave = new Map();
2429
+ this.latestReplicationInfoMessage = new Map();
2430
+ this._replicationInfoBlockedPeers = new Set();
2431
+ this._replicationInfoRequestByPeer = new Map();
2432
+ this._replicationInfoApplyQueueByPeer = new Map();
2433
+ this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
2434
+ this.recentlyRebalanced = new Cache<string>({ max: 1e4, ttl: 1e5 });
2435
+
2436
+ this.uniqueReplicators = new Set();
2437
+ this._replicatorJoinEmitted = new Set();
2438
+ this._replicatorsReconciled = false;
1902
2439
 
1903
2440
  this.openTime = +new Date();
1904
2441
  this.oldestOpenTime = this.openTime;
@@ -1935,6 +2472,13 @@ export class SharedLog<
1935
2472
  }
1936
2473
 
1937
2474
  this._closeController = new AbortController();
2475
+ this._closeController.signal.addEventListener("abort", () => {
2476
+ for (const [_peer, state] of this._replicationInfoRequestByPeer) {
2477
+ if (state.timer) clearTimeout(state.timer);
2478
+ }
2479
+ this._replicationInfoRequestByPeer.clear();
2480
+ });
2481
+
1938
2482
  this._isTrustedReplicator = options?.canReplicate;
1939
2483
  this.keep = options?.keep;
1940
2484
  this.pendingMaturity = new Map();
@@ -1942,19 +2486,56 @@ export class SharedLog<
1942
2486
  const id = sha256Base64Sync(this.log.id);
1943
2487
  const storage = await this.node.storage.sublevel(id);
1944
2488
 
1945
- const localBlocks = await new AnyBlockStore(
1946
- await storage.sublevel("blocks"),
1947
- );
2489
+ const localBlocks = await new AnyBlockStore(await storage.sublevel("blocks"));
2490
+ const fanoutService = (this.node.services as any).fanout as FanoutTree | undefined;
2491
+ const blockProviderNamespace = (cid: string) => `cid:${cid}`;
1948
2492
  this.remoteBlocks = new RemoteBlocks({
1949
2493
  local: localBlocks,
1950
- publish: (message, options) =>
1951
- this.rpc.send(
1952
- new BlocksMessage(message),
1953
- (options as WithMode).mode instanceof AnyWhere ? undefined : options,
1954
- ),
2494
+ publish: (message, options) => this.rpc.send(new BlocksMessage(message), options),
1955
2495
  waitFor: this.rpc.waitFor.bind(this.rpc),
1956
2496
  publicKey: this.node.identity.publicKey,
1957
2497
  eagerBlocks: options?.eagerBlocks ?? true,
2498
+ resolveProviders: async (cid, opts) => {
2499
+ // 1) tracker-backed provider directory (best-effort, bounded)
2500
+ try {
2501
+ const providers = await fanoutService?.queryProviders(
2502
+ blockProviderNamespace(cid),
2503
+ {
2504
+ want: 8,
2505
+ timeoutMs: 2_000,
2506
+ queryTimeoutMs: 500,
2507
+ bootstrapMaxPeers: 2,
2508
+ signal: opts?.signal,
2509
+ },
2510
+ );
2511
+ if (providers && providers.length > 0) return providers;
2512
+ } catch {
2513
+ // ignore discovery failures
2514
+ }
2515
+
2516
+ // 2) fallback to currently connected RPC peers
2517
+ const self = this.node.identity.publicKey.hashcode();
2518
+ const out: string[] = [];
2519
+ const peers = (this.rpc as any)?.peers;
2520
+ for (const h of peers?.keys?.() ?? []) {
2521
+ if (h === self) continue;
2522
+ if (out.includes(h)) continue;
2523
+ out.push(h);
2524
+ if (out.length >= 32) break;
2525
+ }
2526
+ return out;
2527
+ },
2528
+ onPut: async (cid) => {
2529
+ // Best-effort directory announce for "get without remote.from" workflows.
2530
+ try {
2531
+ await fanoutService?.announceProvider(blockProviderNamespace(cid), {
2532
+ ttlMs: 120_000,
2533
+ bootstrapMaxPeers: 2,
2534
+ });
2535
+ } catch {
2536
+ // ignore announce failures
2537
+ }
2538
+ },
1958
2539
  });
1959
2540
 
1960
2541
  await this.remoteBlocks.start();
@@ -1981,9 +2562,10 @@ export class SharedLog<
1981
2562
  ],
1982
2563
  })) > 0;
1983
2564
 
1984
- this._gidPeersHistory = new Map();
1985
- this._requestIPruneSent = new Map();
1986
- this._requestIPruneResponseReplicatorSet = new Map();
2565
+ this._gidPeersHistory = new Map();
2566
+ this._requestIPruneSent = new Map();
2567
+ this._requestIPruneResponseReplicatorSet = new Map();
2568
+ this._checkedPruneRetries = new Map();
1987
2569
 
1988
2570
  this.replicationChangeDebounceFn = debounceAggregationChanges<
1989
2571
  ReplicationRangeIndexable<R>
@@ -2068,6 +2650,87 @@ export class SharedLog<
2068
2650
 
2069
2651
  await this.log.open(this.remoteBlocks, this.node.identity, {
2070
2652
  keychain: this.node.services.keychain,
2653
+ resolveRemotePeers: async (hash, options) => {
2654
+ if (options?.signal?.aborted) return undefined;
2655
+
2656
+ const maxPeers = 8;
2657
+ const self = this.node.identity.publicKey.hashcode();
2658
+ const seed = hashToSeed32(hash);
2659
+
2660
+ // Best hint: peers that have recently confirmed having this entry hash.
2661
+ const hinted = this._requestIPruneResponseReplicatorSet.get(hash);
2662
+ if (hinted && hinted.size > 0) {
2663
+ const peers = [...hinted].filter((p) => p !== self);
2664
+ return peers.length > 0
2665
+ ? pickDeterministicSubset(peers, seed, maxPeers)
2666
+ : undefined;
2667
+ }
2668
+
2669
+ // Next: peers we already contacted about this hash (may still have it).
2670
+ const contacted = this._requestIPruneSent.get(hash);
2671
+ if (contacted && contacted.size > 0) {
2672
+ const peers = [...contacted].filter((p) => p !== self);
2673
+ return peers.length > 0
2674
+ ? pickDeterministicSubset(peers, seed, maxPeers)
2675
+ : undefined;
2676
+ }
2677
+
2678
+ let candidates: string[] | undefined;
2679
+
2680
+ // Prefer the replicator cache; fall back to subscribers if we have no other signal.
2681
+ const replicatorCandidates = [...this.uniqueReplicators].filter(
2682
+ (p) => p !== self,
2683
+ );
2684
+ if (replicatorCandidates.length > 0) {
2685
+ candidates = replicatorCandidates;
2686
+ } else {
2687
+ try {
2688
+ const subscribers = await this._getTopicSubscribers(this.topic);
2689
+ const subscriberCandidates =
2690
+ subscribers?.map((k) => k.hashcode()).filter((p) => p !== self) ??
2691
+ [];
2692
+ candidates =
2693
+ subscriberCandidates.length > 0 ? subscriberCandidates : undefined;
2694
+ } catch {
2695
+ // Best-effort only.
2696
+ }
2697
+
2698
+ if (!candidates || candidates.length === 0) {
2699
+ // Last resort: peers we are already directly connected to. This avoids
2700
+ // depending on global membership knowledge in early-join scenarios.
2701
+ const peerMap = (this.node.services.pubsub as any)?.peers;
2702
+ if (peerMap?.keys) {
2703
+ candidates = [...peerMap.keys()];
2704
+ }
2705
+ }
2706
+
2707
+ if (!candidates || candidates.length === 0) {
2708
+ // Even if the pubsub stream has no established peer streams yet, we may
2709
+ // still have a libp2p connection to one or more peers (e.g. bootstrap).
2710
+ const connectionManager = (this.node.services.pubsub as any)?.components
2711
+ ?.connectionManager;
2712
+ const connections = connectionManager?.getConnections?.() ?? [];
2713
+ const connectionHashes: string[] = [];
2714
+ for (const conn of connections) {
2715
+ const peerId = conn?.remotePeer;
2716
+ if (!peerId) continue;
2717
+ try {
2718
+ connectionHashes.push(getPublicKeyFromPeerId(peerId).hashcode());
2719
+ } catch {
2720
+ // Best-effort only.
2721
+ }
2722
+ }
2723
+ if (connectionHashes.length > 0) {
2724
+ candidates = connectionHashes;
2725
+ }
2726
+ }
2727
+ }
2728
+
2729
+ if (!candidates || candidates.length === 0) return undefined;
2730
+ const peers = candidates.filter((p) => p !== self);
2731
+ if (peers.length === 0) return undefined;
2732
+ return pickDeterministicSubset(peers, seed, maxPeers);
2733
+ },
2071
2734
  ...this._logProperties,
2072
2735
  onChange: async (change) => {
2073
2736
  await this.onChange(change);
@@ -2148,6 +2811,7 @@ export class SharedLog<
2148
2811
  );
2149
2812
 
2150
2813
  await this.rpc.subscribe();
2814
+ await this._openFanoutChannel(options?.fanout);
2151
2815
 
2152
2816
  // mark all our replicaiton ranges as "new", this would allow other peers to understand that we recently reopend our database and might need some sync and warmup
2153
2817
  await this.updateTimestampOfOwnedReplicationRanges(); // TODO do we need to do this before subscribing?
@@ -2234,17 +2898,15 @@ export class SharedLog<
2234
2898
  await this.rebalanceParticipation();
2235
2899
 
2236
2900
  // Take into account existing subscription
2237
- (await this.node.services.pubsub.getSubscribers(this.topic))?.forEach(
2238
- (v, k) => {
2239
- if (v.equals(this.node.identity.publicKey)) {
2240
- return;
2241
- }
2242
- if (this.closed) {
2243
- return;
2244
- }
2245
- this.handleSubscriptionChange(v, [this.topic], true);
2246
- },
2247
- );
2901
+ (await this._getTopicSubscribers(this.topic))?.forEach((v) => {
2902
+ if (v.equals(this.node.identity.publicKey)) {
2903
+ return;
2904
+ }
2905
+ if (this.closed) {
2906
+ return;
2907
+ }
2908
+ this.handleSubscriptionChange(v, [this.topic], true);
2909
+ });
2248
2910
  }
2249
2911
 
2250
2912
  async reset() {
@@ -2278,7 +2940,7 @@ export class SharedLog<
2278
2940
  })
2279
2941
  .then(async () => {
2280
2942
  // is reachable, announce change events
2281
- const key = await this.node.services.pubsub.getPublicKey(
2943
+ const key = await this._resolvePublicKeyFromHash(
2282
2944
  segment.value.hash,
2283
2945
  );
2284
2946
  if (!key) {
@@ -2288,22 +2950,26 @@ export class SharedLog<
2288
2950
  );
2289
2951
  }
2290
2952
 
2291
- this.uniqueReplicators.add(key.hashcode());
2953
+ const keyHash = key.hashcode();
2954
+ this.uniqueReplicators.add(keyHash);
2292
2955
 
2293
- this.events.dispatchEvent(
2294
- new CustomEvent<ReplicatorJoinEvent>("replicator:join", {
2295
- detail: { publicKey: key },
2296
- }),
2297
- );
2298
- this.events.dispatchEvent(
2299
- new CustomEvent<ReplicationChangeEvent>(
2300
- "replication:change",
2301
- {
2302
- detail: { publicKey: key },
2303
- },
2304
- ),
2305
- );
2306
- })
2956
+ if (!this._replicatorJoinEmitted.has(keyHash)) {
2957
+ this._replicatorJoinEmitted.add(keyHash);
2958
+ this.events.dispatchEvent(
2959
+ new CustomEvent<ReplicatorJoinEvent>("replicator:join", {
2960
+ detail: { publicKey: key },
2961
+ }),
2962
+ );
2963
+ this.events.dispatchEvent(
2964
+ new CustomEvent<ReplicationChangeEvent>(
2965
+ "replication:change",
2966
+ {
2967
+ detail: { publicKey: key },
2968
+ },
2969
+ ),
2970
+ );
2971
+ }
2972
+ })
2307
2973
  .catch(async (e) => {
2308
2974
  if (isNotStartedError(e)) {
2309
2975
  return; // TODO test this path
@@ -2435,48 +3101,59 @@ export class SharedLog<
2435
3101
  numbers: this.indexableDomain.numbers,
2436
3102
  });
2437
3103
 
2438
- // Check abort signal before building result
2439
- if (options?.signal?.aborted) {
2440
- return [];
2441
- }
3104
+ // Check abort signal before building result
3105
+ if (options?.signal?.aborted) {
3106
+ return [];
3107
+ }
2442
3108
 
2443
- // add all in flight
2444
- for (const [key, _] of this.syncronizer.syncInFlight) {
2445
- set.add(key);
2446
- }
3109
+ // add all in flight
3110
+ for (const [key, _] of this.syncronizer.syncInFlight) {
3111
+ set.add(key);
3112
+ }
2447
3113
 
2448
- if (options?.reachableOnly) {
2449
- // Prefer the live pubsub subscriber set when filtering reachability.
2450
- // `uniqueReplicators` is primarily driven by replication messages and can lag during
2451
- // joins/restarts; using subscribers prevents excluding peers that are reachable but
2452
- // whose replication ranges were loaded from disk or haven't been processed yet.
2453
- const subscribers =
2454
- (await this.node.services.pubsub.getSubscribers(this.topic)) ??
2455
- undefined;
2456
- const subscriberHashcodes = subscribers
2457
- ? new Set(subscribers.map((key) => key.hashcode()))
3114
+ const selfHash = this.node.identity.publicKey.hashcode();
3115
+
3116
+ if (options?.reachableOnly) {
3117
+ const directPeers: Map<string, unknown> | undefined = (this.node.services
3118
+ .pubsub as any)?.peers;
3119
+
3120
+ // Prefer the live pubsub subscriber set when filtering reachability. In some
3121
+ // flows peers can be reachable/active even before (or without) subscriber
3122
+ // state converging, so also consider direct pubsub peers.
3123
+ const subscribers =
3124
+ (await this._getTopicSubscribers(this.topic)) ?? undefined;
3125
+ const subscriberHashcodes = subscribers
3126
+ ? new Set(subscribers.map((key) => key.hashcode()))
2458
3127
  : undefined;
2459
3128
 
3129
+ // If reachability is requested but we have no basis for filtering yet
3130
+ // (subscriber snapshot hasn't converged), return the full cover set.
3131
+ // Otherwise, only keep peers we can currently reach.
3132
+ const canFilter =
3133
+ directPeers != null ||
3134
+ (subscriberHashcodes && subscriberHashcodes.size > 0);
3135
+ if (!canFilter) {
3136
+ return [...set];
3137
+ }
3138
+
2460
3139
  const reachable: string[] = [];
2461
- const selfHash = this.node.identity.publicKey.hashcode();
2462
3140
  for (const peer of set) {
2463
3141
  if (peer === selfHash) {
2464
3142
  reachable.push(peer);
2465
3143
  continue;
2466
3144
  }
2467
3145
  if (
2468
- subscriberHashcodes
2469
- ? subscriberHashcodes.has(peer)
2470
- : this.uniqueReplicators.has(peer)
3146
+ (subscriberHashcodes && subscriberHashcodes.has(peer)) ||
3147
+ (directPeers && directPeers.has(peer))
2471
3148
  ) {
2472
3149
  reachable.push(peer);
2473
3150
  }
2474
3151
  }
2475
3152
  return reachable;
2476
- }
3153
+ }
2477
3154
 
2478
- return [...set];
2479
- } catch (error) {
3155
+ return [...set];
3156
+ } catch (error) {
2480
3157
  // Handle race conditions where the index gets closed during the operation
2481
3158
  if (isNotStartedError(error as Error)) {
2482
3159
  return [];
@@ -2497,6 +3174,13 @@ export class SharedLog<
2497
3174
  this.pendingMaturity.clear();
2498
3175
 
2499
3176
  this.distributeQueue?.clear();
3177
+ this._closeFanoutChannel();
3178
+ try {
3179
+ this._providerHandle?.close();
3180
+ } catch {
3181
+ // ignore
3182
+ }
3183
+ this._providerHandle = undefined;
2500
3184
  this.coordinateToHash.clear();
2501
3185
  this.recentlyRebalanced.clear();
2502
3186
  this.uniqueReplicators.clear();
@@ -2518,28 +3202,87 @@ export class SharedLog<
2518
3202
  v.clear();
2519
3203
  v.promise.resolve(); // TODO or reject?
2520
3204
  }
2521
- for (const [_k, v] of this._pendingIHave) {
2522
- v.clear();
2523
- }
3205
+ for (const [_k, v] of this._pendingIHave) {
3206
+ v.clear();
3207
+ }
3208
+ for (const [_k, v] of this._checkedPruneRetries) {
3209
+ if (v.timer) clearTimeout(v.timer);
3210
+ }
2524
3211
 
2525
3212
  await this.remoteBlocks.stop();
2526
- this._pendingDeletes.clear();
2527
- this._pendingIHave.clear();
2528
- this.latestReplicationInfoMessage.clear();
2529
- this._gidPeersHistory.clear();
2530
- this._requestIPruneSent.clear();
2531
- this._requestIPruneResponseReplicatorSet.clear();
2532
- this.pruneDebouncedFn = undefined as any;
2533
- this.rebalanceParticipationDebounced = undefined;
2534
- this._replicationRangeIndex.stop();
2535
- this._entryCoordinatesIndex.stop();
3213
+ this._pendingDeletes.clear();
3214
+ this._pendingIHave.clear();
3215
+ this._checkedPruneRetries.clear();
3216
+ this.latestReplicationInfoMessage.clear();
3217
+ this._gidPeersHistory.clear();
3218
+ this._requestIPruneSent.clear();
3219
+ this._requestIPruneResponseReplicatorSet.clear();
3220
+ // Cancel any pending debounced timers so they can't fire after we've torn down
3221
+ // indexes/RPC state.
3222
+ this.rebalanceParticipationDebounced?.close();
3223
+ this.replicationChangeDebounceFn?.close?.();
3224
+ this.pruneDebouncedFn?.close?.();
3225
+ this.responseToPruneDebouncedFn?.close?.();
3226
+ this.pruneDebouncedFn = undefined as any;
3227
+ this.rebalanceParticipationDebounced = undefined;
3228
+ this._replicationRangeIndex.stop();
3229
+ this._entryCoordinatesIndex.stop();
2536
3230
  this._replicationRangeIndex = undefined as any;
2537
3231
  this._entryCoordinatesIndex = undefined as any;
2538
3232
 
2539
3233
  this.cpuUsage?.stop?.();
2540
3234
  /* this._totalParticipation = 0; */
2541
3235
  }
2542
- async close(from?: Program): Promise<boolean> {
3236
+ async close(from?: Program): Promise<boolean> {
3237
+ // Best-effort: announce that we are going offline before tearing down
3238
+ // RPC/subscription state.
3239
+ //
3240
+ // Important: do not delete our local replication ranges here. Keeping them
3241
+ // allows `replicate: { type: "resume" }` to restore the previous role on
3242
+ // restart. Explicit `unreplicate()` still clears local state.
3243
+ try {
3244
+ if (!this.closed) {
3245
+ // Prevent any late debounced timers (rebalance/prune) from publishing
3246
+ // replication info after we announce "segments: []". These races can leave
3247
+ // stale segments on remotes after rapid open/close cycles.
3248
+ this._isReplicating = false;
3249
+ this._isAdaptiveReplicating = false;
3250
+ this.rebalanceParticipationDebounced?.close();
3251
+ this.replicationChangeDebounceFn?.close?.();
3252
+ this.pruneDebouncedFn?.close?.();
3253
+ this.responseToPruneDebouncedFn?.close?.();
3254
+
3255
+ // Ensure the "I'm leaving" replication reset is actually published before
3256
+ // the RPC child program closes and unsubscribes from its topic. If we fire
3257
+ // and forget here, the publish can race with `super.close()` and get dropped,
3258
+ // leaving stale replication segments on remotes (flaky join/leave tests).
3259
+ // Also ensure close is bounded even when shard overlays are mid-reconcile.
3260
+ const abort = new AbortController();
3261
+ const abortTimer = setTimeout(() => {
3262
+ try {
3263
+ abort.abort(
3264
+ new TimeoutError(
3265
+ "shared-log close replication reset timed out",
3266
+ ),
3267
+ );
3268
+ } catch {
3269
+ abort.abort();
3270
+ }
3271
+ }, 2_000);
3272
+ try {
3273
+ await this.rpc
3274
+ .send(new AllReplicatingSegmentsMessage({ segments: [] }), {
3275
+ priority: 1,
3276
+ signal: abort.signal,
3277
+ })
3278
+ .catch(() => {});
3279
+ } finally {
3280
+ clearTimeout(abortTimer);
3281
+ }
3282
+ }
3283
+ } catch {
3284
+ // ignore: close should be resilient even if we were never fully started
3285
+ }
2543
3286
  const superClosed = await super.close(from);
2544
3287
  if (!superClosed) {
2545
3288
  return superClosed;
@@ -2549,12 +3292,50 @@ export class SharedLog<
2549
3292
  return true;
2550
3293
  }
2551
3294
 
2552
- async drop(from?: Program): Promise<boolean> {
2553
- const superDropped = await super.drop(from);
2554
- if (!superDropped) {
2555
- return superDropped;
2556
- }
2557
- await this._entryCoordinatesIndex.drop();
3295
+ async drop(from?: Program): Promise<boolean> {
3296
+ // Best-effort: announce that we are going offline before tearing down
3297
+ // RPC/subscription state (same reasoning as in `close()`).
3298
+ try {
3299
+ if (!this.closed) {
3300
+ this._isReplicating = false;
3301
+ this._isAdaptiveReplicating = false;
3302
+ this.rebalanceParticipationDebounced?.close();
3303
+ this.replicationChangeDebounceFn?.close?.();
3304
+ this.pruneDebouncedFn?.close?.();
3305
+ this.responseToPruneDebouncedFn?.close?.();
3306
+
3307
+ const abort = new AbortController();
3308
+ const abortTimer = setTimeout(() => {
3309
+ try {
3310
+ abort.abort(
3311
+ new TimeoutError(
3312
+ "shared-log drop replication reset timed out",
3313
+ ),
3314
+ );
3315
+ } catch {
3316
+ abort.abort();
3317
+ }
3318
+ }, 2_000);
3319
+ try {
3320
+ await this.rpc
3321
+ .send(new AllReplicatingSegmentsMessage({ segments: [] }), {
3322
+ priority: 1,
3323
+ signal: abort.signal,
3324
+ })
3325
+ .catch(() => {});
3326
+ } finally {
3327
+ clearTimeout(abortTimer);
3328
+ }
3329
+ }
3330
+ } catch {
3331
+ // ignore: drop should be resilient even if we were never fully started
3332
+ }
3333
+
3334
+ const superDropped = await super.drop(from);
3335
+ if (!superDropped) {
3336
+ return superDropped;
3337
+ }
3338
+ await this._entryCoordinatesIndex.drop();
2558
3339
  await this._replicationRangeIndex.drop();
2559
3340
  await this.log.drop();
2560
3341
  await this._close();
@@ -2921,20 +3702,20 @@ export class SharedLog<
2921
3702
  return;
2922
3703
  }
2923
3704
 
2924
- const segments = (await this.getMyReplicationSegments()).map((x) =>
2925
- x.toReplicationRange(),
2926
- );
3705
+ const segments = (await this.getMyReplicationSegments()).map((x) =>
3706
+ x.toReplicationRange(),
3707
+ );
2927
3708
 
2928
- this.rpc
2929
- .send(new AllReplicatingSegmentsMessage({ segments }), {
2930
- mode: new SeekDelivery({ to: [context.from], redundancy: 1 }),
2931
- })
2932
- .catch((e) => logger.error(e.toString()));
3709
+ this.rpc
3710
+ .send(new AllReplicatingSegmentsMessage({ segments }), {
3711
+ mode: new AcknowledgeDelivery({ to: [context.from], redundancy: 1 }),
3712
+ })
3713
+ .catch((e) => logger.error(e.toString()));
2933
3714
 
2934
- // for backwards compatibility (v8) remove this when we are sure that all nodes are v9+
2935
- if (this.v8Behaviour) {
2936
- const role = this.getRole();
2937
- if (role instanceof Replicator) {
3715
+ // for backwards compatibility (v8) remove this when we are sure that all nodes are v9+
3716
+ if (this.v8Behaviour) {
3717
+ const role = this.getRole();
3718
+ if (role instanceof Replicator) {
2938
3719
  const fixedSettings = !this._isAdaptiveReplicating;
2939
3720
  if (fixedSettings) {
2940
3721
  await this.rpc.send(
@@ -2959,71 +3740,91 @@ export class SharedLog<
2959
3740
  return;
2960
3741
  }
2961
3742
 
2962
- const replicationInfoMessage = msg as
2963
- | AllReplicatingSegmentsMessage
2964
- | AddedReplicationSegmentMessage;
2965
-
2966
- // Process replication updates even if the sender isn't yet considered "ready" by
2967
- // `Program.waitFor()`. Dropping these messages can lead to missing replicator info
2968
- // (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
2969
- const from = context.from!;
2970
- const messageTimestamp = context.message.header.timestamp;
2971
- (async () => {
2972
- const prev = this.latestReplicationInfoMessage.get(from.hashcode());
2973
- if (prev && prev > messageTimestamp) {
3743
+ const replicationInfoMessage = msg as
3744
+ | AllReplicatingSegmentsMessage
3745
+ | AddedReplicationSegmentMessage;
3746
+
3747
+ // Process replication updates even if the sender isn't yet considered "ready" by
3748
+ // `Program.waitFor()`. Dropping these messages can lead to missing replicator info
3749
+ // (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
3750
+ const from = context.from!;
3751
+ const fromHash = from.hashcode();
3752
+ if (this._replicationInfoBlockedPeers.has(fromHash)) {
2974
3753
  return;
2975
3754
  }
3755
+ const messageTimestamp = context.message.header.timestamp;
3756
+ await this.withReplicationInfoApplyQueue(fromHash, async () => {
3757
+ try {
3758
+ // The peer may have unsubscribed after this message was queued.
3759
+ if (this._replicationInfoBlockedPeers.has(fromHash)) {
3760
+ return;
3761
+ }
2976
3762
 
2977
- this.latestReplicationInfoMessage.set(
2978
- from.hashcode(),
2979
- messageTimestamp,
2980
- );
3763
+ // Process in-order to avoid races where repeated reset messages arrive
3764
+ // concurrently and trigger spurious "added" diffs / rebalancing.
3765
+ const prev = this.latestReplicationInfoMessage.get(fromHash);
3766
+ if (prev && prev > messageTimestamp) {
3767
+ return;
3768
+ }
2981
3769
 
2982
- if (this.closed) {
2983
- return;
2984
- }
3770
+ this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
2985
3771
 
2986
- const reset = msg instanceof AllReplicatingSegmentsMessage;
2987
- await this.addReplicationRange(
2988
- replicationInfoMessage.segments.map((x) =>
2989
- x.toReplicationRangeIndexable(from),
2990
- ),
2991
- from,
2992
- {
2993
- reset,
2994
- checkDuplicates: true,
2995
- timestamp: Number(messageTimestamp),
2996
- },
2997
- );
2998
- })().catch((e) => {
2999
- if (isNotStartedError(e)) {
3000
- return;
3001
- }
3002
- logger.error(
3003
- `Failed to apply replication settings from '${from.hashcode()}': ${
3004
- e?.message ?? e
3005
- }`,
3006
- );
3007
- });
3008
- } else if (msg instanceof StoppedReplicating) {
3009
- if (context.from.equals(this.node.identity.publicKey)) {
3010
- return;
3011
- }
3772
+ if (this.closed) {
3773
+ return;
3774
+ }
3012
3775
 
3013
- const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(
3014
- msg.segmentIds,
3015
- context.from,
3016
- );
3776
+ const reset = msg instanceof AllReplicatingSegmentsMessage;
3777
+ await this.addReplicationRange(
3778
+ replicationInfoMessage.segments.map((x) =>
3779
+ x.toReplicationRangeIndexable(from),
3780
+ ),
3781
+ from,
3782
+ {
3783
+ reset,
3784
+ checkDuplicates: true,
3785
+ timestamp: Number(messageTimestamp),
3786
+ },
3787
+ );
3017
3788
 
3018
- await this.removeReplicationRanges(rangesToRemove, context.from);
3019
- const timestamp = BigInt(+new Date());
3020
- for (const range of rangesToRemove) {
3021
- this.replicationChangeDebounceFn.add({
3022
- range,
3023
- type: "removed",
3024
- timestamp,
3789
+ // If the peer reports any replication segments, stop re-requesting.
3790
+ // (Empty reports can be transient during startup.)
3791
+ if (replicationInfoMessage.segments.length > 0) {
3792
+ this.cancelReplicationInfoRequests(fromHash);
3793
+ }
3794
+ } catch (e) {
3795
+ if (isNotStartedError(e as Error)) {
3796
+ return;
3797
+ }
3798
+ logger.error(
3799
+ `Failed to apply replication settings from '${fromHash}': ${
3800
+ (e as any)?.message ?? e
3801
+ }`,
3802
+ );
3803
+ }
3025
3804
  });
3026
- }
3805
+ } else if (msg instanceof StoppedReplicating) {
3806
+ if (context.from.equals(this.node.identity.publicKey)) {
3807
+ return;
3808
+ }
3809
+ const fromHash = context.from.hashcode();
3810
+ if (this._replicationInfoBlockedPeers.has(fromHash)) {
3811
+ return;
3812
+ }
3813
+
3814
+ const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(
3815
+ msg.segmentIds,
3816
+ context.from,
3817
+ );
3818
+
3819
+ await this.removeReplicationRanges(rangesToRemove, context.from);
3820
+ const timestamp = BigInt(+new Date());
3821
+ for (const range of rangesToRemove) {
3822
+ this.replicationChangeDebounceFn.add({
3823
+ range,
3824
+ type: "removed",
3825
+ timestamp,
3826
+ });
3827
+ }
3027
3828
  } else {
3028
3829
  throw new Error("Unexpected message");
3029
3830
  }
@@ -3325,10 +4126,10 @@ export class SharedLog<
3325
4126
  }
3326
4127
  }
3327
4128
 
3328
- async waitForReplicator(
3329
- key: PublicSignKey,
3330
- options?: {
3331
- signal?: AbortSignal;
4129
+ async waitForReplicator(
4130
+ key: PublicSignKey,
4131
+ options?: {
4132
+ signal?: AbortSignal;
3332
4133
  eager?: boolean;
3333
4134
  roleAge?: number;
3334
4135
  timeout?: number;
@@ -3340,9 +4141,9 @@ export class SharedLog<
3340
4141
  ? undefined
3341
4142
  : (options?.roleAge ?? (await this.getDefaultMinRoleAge()));
3342
4143
 
3343
- let settled = false;
3344
- let timer: ReturnType<typeof setTimeout> | undefined;
3345
- let requestTimer: ReturnType<typeof setTimeout> | undefined;
4144
+ let settled = false;
4145
+ let timer: ReturnType<typeof setTimeout> | undefined;
4146
+ let requestTimer: ReturnType<typeof setTimeout> | undefined;
3346
4147
 
3347
4148
  const clear = () => {
3348
4149
  this.events.removeEventListener("replicator:mature", check);
@@ -3358,14 +4159,19 @@ export class SharedLog<
3358
4159
  }
3359
4160
  };
3360
4161
 
3361
- const resolve = () => {
3362
- if (settled) {
3363
- return;
3364
- }
3365
- settled = true;
3366
- clear();
3367
- deferred.resolve();
3368
- };
4162
+ const resolve = async () => {
4163
+ if (settled) {
4164
+ return;
4165
+ }
4166
+ settled = true;
4167
+ clear();
4168
+ // `waitForReplicator()` is typically used as a precondition before join/replicate
4169
+ // flows. A replicator can become mature and enqueue a debounced rebalance
4170
+ // (`replicationChangeDebounceFn`) slightly later. Flush here so callers don't
4171
+ // observe a "late" rebalance after the wait resolves.
4172
+ await this.replicationChangeDebounceFn?.flush?.();
4173
+ deferred.resolve();
4174
+ };
3369
4175
 
3370
4176
  const reject = (error: Error) => {
3371
4177
  if (settled) {
@@ -3409,13 +4215,14 @@ export class SharedLog<
3409
4215
 
3410
4216
  this.rpc
3411
4217
  .send(new RequestReplicationInfoMessage(), {
3412
- mode: new SeekDelivery({ redundancy: 1, to: [key] }),
4218
+ mode: new AcknowledgeDelivery({ redundancy: 1, to: [key] }),
3413
4219
  })
3414
4220
  .catch((e) => {
3415
4221
  // Best-effort: missing peers / unopened RPC should not fail the wait logic.
3416
4222
  if (isNotStartedError(e as Error)) {
3417
4223
  return;
3418
4224
  }
4225
+ logger.error(e?.toString?.() ?? String(e));
3419
4226
  });
3420
4227
 
3421
4228
  if (requestAttempts < maxRequestAttempts) {
@@ -3423,29 +4230,29 @@ export class SharedLog<
3423
4230
  }
3424
4231
  };
3425
4232
 
3426
- const check = async () => {
3427
- const iterator = this.replicationIndex?.iterate(
3428
- { query: new StringMatch({ key: "hash", value: key.hashcode() }) },
3429
- { reference: true },
3430
- );
3431
- try {
3432
- const rects = await iterator?.next(1);
3433
- const rect = rects?.[0]?.value;
3434
- if (!rect) {
3435
- return;
3436
- }
3437
- if (!options?.eager && resolvedRoleAge != null) {
3438
- if (!isMatured(rect, +new Date(), resolvedRoleAge)) {
4233
+ const check = async () => {
4234
+ const iterator = this.replicationIndex?.iterate(
4235
+ { query: new StringMatch({ key: "hash", value: key.hashcode() }) },
4236
+ { reference: true },
4237
+ );
4238
+ try {
4239
+ const rects = await iterator?.next(1);
4240
+ const rect = rects?.[0]?.value;
4241
+ if (!rect) {
3439
4242
  return;
3440
4243
  }
4244
+ if (!options?.eager && resolvedRoleAge != null) {
4245
+ if (!isMatured(rect, +new Date(), resolvedRoleAge)) {
4246
+ return;
4247
+ }
4248
+ }
4249
+ await resolve();
4250
+ } catch (error) {
4251
+ reject(error instanceof Error ? error : new Error(String(error)));
4252
+ } finally {
4253
+ await iterator?.close();
3441
4254
  }
3442
- resolve();
3443
- } catch (error) {
3444
- reject(error instanceof Error ? error : new Error(String(error)));
3445
- } finally {
3446
- await iterator?.close();
3447
- }
3448
- };
4255
+ };
3449
4256
 
3450
4257
  requestReplicationInfo();
3451
4258
  check();
@@ -3462,15 +4269,6 @@ export class SharedLog<
3462
4269
  coverageThreshold?: number;
3463
4270
  waitForNewPeers?: boolean;
3464
4271
  }) {
3465
- // if no remotes, just return
3466
- const subscribers = await this.node.services.pubsub.getSubscribers(
3467
- this.rpc.topic,
3468
- );
3469
- let waitForNewPeers = options?.waitForNewPeers;
3470
- if (!waitForNewPeers && (subscribers?.length ?? 0) === 0) {
3471
- throw new NoPeersError(this.rpc.topic);
3472
- }
3473
-
3474
4272
  let coverageThreshold = options?.coverageThreshold ?? 1;
3475
4273
  let deferred = pDefer<void>();
3476
4274
  let settled = false;
@@ -3735,9 +4533,7 @@ export class SharedLog<
3735
4533
  let subscribers = 1;
3736
4534
  if (!this.rpc.closed) {
3737
4535
  try {
3738
- subscribers =
3739
- (await this.node.services.pubsub.getSubscribers(this.rpc.topic))
3740
- ?.length ?? 1;
4536
+ subscribers = (await this._getTopicSubscribers(this.rpc.topic))?.length ?? 1;
3741
4537
  } catch {
3742
4538
  // Best-effort only; fall back to 1.
3743
4539
  }
@@ -3852,22 +4648,29 @@ export class SharedLog<
3852
4648
  const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge()); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
3853
4649
  const selfHash = this.node.identity.publicKey.hashcode();
3854
4650
 
3855
- // Use `uniqueReplicators` (replicator cache) once we've reconciled it against the
3856
- // persisted replication index. Until then, fall back to live pubsub subscribers
3857
- // and avoid relying on `uniqueReplicators` being complete.
4651
+ // Prefer `uniqueReplicators` (replicator cache) as soon as it has any data.
4652
+ // Falling back to live pubsub subscribers can include non-replicators and can
4653
+ // break delivery/availability when writers are not directly connected.
3858
4654
  let peerFilter: Set<string> | undefined = undefined;
3859
- if (this._replicatorsReconciled && this.uniqueReplicators.size > 0) {
3860
- peerFilter = this.uniqueReplicators.has(selfHash)
3861
- ? this.uniqueReplicators
3862
- : new Set([...this.uniqueReplicators, selfHash]);
4655
+ const selfReplicating = await this.isReplicating();
4656
+ if (this.uniqueReplicators.size > 0) {
4657
+ peerFilter = new Set(this.uniqueReplicators);
4658
+ if (selfReplicating) {
4659
+ peerFilter.add(selfHash);
4660
+ } else {
4661
+ peerFilter.delete(selfHash);
4662
+ }
3863
4663
  } else {
3864
4664
  try {
3865
4665
  const subscribers =
3866
- (await this.node.services.pubsub.getSubscribers(this.topic)) ??
3867
- undefined;
4666
+ (await this._getTopicSubscribers(this.topic)) ?? undefined;
3868
4667
  if (subscribers && subscribers.length > 0) {
3869
4668
  peerFilter = new Set(subscribers.map((key) => key.hashcode()));
3870
- peerFilter.add(selfHash);
4669
+ if (selfReplicating) {
4670
+ peerFilter.add(selfHash);
4671
+ } else {
4672
+ peerFilter.delete(selfHash);
4673
+ }
3871
4674
  }
3872
4675
  } catch {
3873
4676
  // Best-effort only; if pubsub isn't ready, do a full scan.
@@ -3913,76 +4716,160 @@ export class SharedLog<
3913
4716
  );
3914
4717
  }
3915
4718
 
3916
- async handleSubscriptionChange(
3917
- publicKey: PublicSignKey,
3918
- topics: string[],
3919
- subscribed: boolean,
3920
- ) {
3921
- if (!topics.includes(this.topic)) {
4719
+ private withReplicationInfoApplyQueue(
4720
+ peerHash: string,
4721
+ fn: () => Promise<void>,
4722
+ ): Promise<void> {
4723
+ const prev = this._replicationInfoApplyQueueByPeer.get(peerHash);
4724
+ const next = (prev ?? Promise.resolve())
4725
+ .catch(() => {
4726
+ // Avoid stuck queues if a previous apply failed.
4727
+ })
4728
+ .then(fn);
4729
+ this._replicationInfoApplyQueueByPeer.set(peerHash, next);
4730
+ return next.finally(() => {
4731
+ if (this._replicationInfoApplyQueueByPeer.get(peerHash) === next) {
4732
+ this._replicationInfoApplyQueueByPeer.delete(peerHash);
4733
+ }
4734
+ });
4735
+ }
4736
+
4737
+ private cancelReplicationInfoRequests(peerHash: string) {
4738
+ const state = this._replicationInfoRequestByPeer.get(peerHash);
4739
+ if (!state) return;
4740
+ if (state.timer) {
4741
+ clearTimeout(state.timer);
4742
+ }
4743
+ this._replicationInfoRequestByPeer.delete(peerHash);
4744
+ }
4745
+
4746
+ private scheduleReplicationInfoRequests(peer: PublicSignKey) {
4747
+ const peerHash = peer.hashcode();
4748
+ if (this._replicationInfoRequestByPeer.has(peerHash)) {
3922
4749
  return;
3923
4750
  }
3924
4751
 
3925
- if (!subscribed) {
3926
- this.removePeerFromGidPeerHistory(publicKey.hashcode());
4752
+ const state: { attempts: number; timer?: ReturnType<typeof setTimeout> } = {
4753
+ attempts: 0,
4754
+ };
4755
+ this._replicationInfoRequestByPeer.set(peerHash, state);
3927
4756
 
3928
- for (const [k, v] of this._requestIPruneSent) {
3929
- v.delete(publicKey.hashcode());
3930
- if (v.size === 0) {
3931
- this._requestIPruneSent.delete(k);
3932
- }
4757
+ const intervalMs = Math.max(50, this.waitForReplicatorRequestIntervalMs);
4758
+ const maxAttempts = Math.min(
4759
+ 5,
4760
+ this.waitForReplicatorRequestMaxAttempts ??
4761
+ WAIT_FOR_REPLICATOR_REQUEST_MIN_ATTEMPTS,
4762
+ );
4763
+
4764
+ const tick = () => {
4765
+ if (this.closed || this._closeController.signal.aborted) {
4766
+ this.cancelReplicationInfoRequests(peerHash);
4767
+ return;
3933
4768
  }
3934
4769
 
3935
- for (const [k, v] of this._requestIPruneResponseReplicatorSet) {
3936
- v.delete(publicKey.hashcode());
3937
- if (v.size === 0) {
3938
- this._requestIPruneResponseReplicatorSet.delete(k);
3939
- }
4770
+ state.attempts++;
4771
+
4772
+ this.rpc
4773
+ .send(new RequestReplicationInfoMessage(), {
4774
+ mode: new AcknowledgeDelivery({ redundancy: 1, to: [peer] }),
4775
+ })
4776
+ .catch((e) => {
4777
+ // Best-effort: missing peers / unopened RPC should not fail join flows.
4778
+ if (isNotStartedError(e as Error)) {
4779
+ return;
4780
+ }
4781
+ logger.error(e?.toString?.() ?? String(e));
4782
+ });
4783
+
4784
+ if (state.attempts >= maxAttempts) {
4785
+ this.cancelReplicationInfoRequests(peerHash);
4786
+ return;
3940
4787
  }
3941
4788
 
3942
- this.syncronizer.onPeerDisconnected(publicKey);
4789
+ state.timer = setTimeout(tick, intervalMs);
4790
+ state.timer.unref?.();
4791
+ };
3943
4792
 
3944
- (await this.replicationIndex.count({
3945
- query: { hash: publicKey.hashcode() },
3946
- })) > 0 &&
3947
- this.events.dispatchEvent(
3948
- new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
3949
- detail: { publicKey },
3950
- }),
3951
- );
3952
- }
4793
+ tick();
4794
+ }
3953
4795
 
3954
- if (subscribed) {
3955
- const replicationSegments = await this.getMyReplicationSegments();
3956
- if (replicationSegments.length > 0) {
3957
- this.rpc
3958
- .send(
3959
- new AllReplicatingSegmentsMessage({
3960
- segments: replicationSegments.map((x) => x.toReplicationRange()),
4796
+ async handleSubscriptionChange(
4797
+ publicKey: PublicSignKey,
4798
+ topics: string[],
4799
+ subscribed: boolean,
4800
+ ) {
4801
+ if (!topics.includes(this.topic)) {
4802
+ return;
4803
+ }
4804
+
4805
+ const peerHash = publicKey.hashcode();
4806
+ if (subscribed) {
4807
+ this._replicationInfoBlockedPeers.delete(peerHash);
4808
+ } else {
4809
+ this._replicationInfoBlockedPeers.add(peerHash);
4810
+ }
4811
+
4812
+ if (!subscribed) {
4813
+ // Emit replicator:leave at most once per (join -> leave) transition, even if we
4814
+ // concurrently process unsubscribe + replication reset messages for the same peer.
4815
+ const stoppedTransition = this.uniqueReplicators.delete(peerHash);
4816
+ this._replicatorJoinEmitted.delete(peerHash);
4817
+
4818
+ this.cancelReplicationInfoRequests(peerHash);
4819
+ this.removePeerFromGidPeerHistory(peerHash);
4820
+
4821
+ for (const [k, v] of this._requestIPruneSent) {
4822
+ v.delete(peerHash);
4823
+ if (v.size === 0) {
4824
+ this._requestIPruneSent.delete(k);
4825
+ }
4826
+ }
4827
+
4828
+ for (const [k, v] of this._requestIPruneResponseReplicatorSet) {
4829
+ v.delete(peerHash);
4830
+ if (v.size === 0) {
4831
+ this._requestIPruneResponseReplicatorSet.delete(k);
4832
+ }
4833
+ }
4834
+
4835
+ this.syncronizer.onPeerDisconnected(publicKey);
4836
+
4837
+ stoppedTransition &&
4838
+ this.events.dispatchEvent(
4839
+ new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
4840
+ detail: { publicKey },
3961
4841
  }),
3962
- {
3963
- mode: new SeekDelivery({ redundancy: 1, to: [publicKey] }),
3964
- },
3965
- )
3966
- .catch((e) => logger.error(e.toString()));
4842
+ );
4843
+ }
3967
4844
 
3968
- if (this.v8Behaviour) {
3969
- // for backwards compatibility
4845
+ if (subscribed) {
4846
+ const replicationSegments = await this.getMyReplicationSegments();
4847
+ if (replicationSegments.length > 0) {
3970
4848
  this.rpc
3971
- .send(new ResponseRoleMessage({ role: await this.getRole() }), {
3972
- mode: new SeekDelivery({ redundancy: 1, to: [publicKey] }),
3973
- })
4849
+ .send(
4850
+ new AllReplicatingSegmentsMessage({
4851
+ segments: replicationSegments.map((x) => x.toReplicationRange()),
4852
+ }),
4853
+ {
4854
+ mode: new AcknowledgeDelivery({ redundancy: 1, to: [publicKey] }),
4855
+ },
4856
+ )
3974
4857
  .catch((e) => logger.error(e.toString()));
4858
+
4859
+ if (this.v8Behaviour) {
4860
+ // for backwards compatibility
4861
+ this.rpc
4862
+ .send(new ResponseRoleMessage({ role: await this.getRole() }), {
4863
+ mode: new AcknowledgeDelivery({ redundancy: 1, to: [publicKey] }),
4864
+ })
4865
+ .catch((e) => logger.error(e.toString()));
4866
+ }
3975
4867
  }
3976
- }
3977
4868
 
3978
- // Request the remote peer's replication info. This makes joins resilient to
3979
- // timing-sensitive delivery/order issues where we may miss their initial
4869
+ // Request the remote peer's replication info. This makes joins resilient to
4870
+ // timing-sensitive delivery/order issues where we may miss their initial
3980
4871
  // replication announcement.
3981
- this.rpc
3982
- .send(new RequestReplicationInfoMessage(), {
3983
- mode: new SeekDelivery({ redundancy: 1, to: [publicKey] }),
3984
- })
3985
- .catch((e) => logger.error(e.toString()));
4872
+ this.scheduleReplicationInfoRequests(publicKey);
3986
4873
  } else {
3987
4874
  await this.removeReplicator(publicKey);
3988
4875
  }
@@ -4025,8 +4912,8 @@ export class SharedLog<
4025
4912
  leaders: Map<string, unknown> | Set<string>;
4026
4913
  }
4027
4914
  >,
4028
- options?: { timeout?: number; unchecked?: boolean },
4029
- ): Promise<any>[] {
4915
+ options?: { timeout?: number; unchecked?: boolean },
4916
+ ): Promise<any>[] {
4030
4917
  if (options?.unchecked) {
4031
4918
  return [...entries.values()].map((x) => {
4032
4919
  this._gidPeersHistory.delete(x.entry.meta.gid);
@@ -4051,30 +4938,57 @@ export class SharedLog<
4051
4938
  // - An entry is joined, where min replicas is lower than before (for all heads for this particular gid) and therefore we are not replicating anymore for this particular gid
4052
4939
  // - Peers join and leave, which means we might not be a replicator anymore
4053
4940
 
4054
- const promises: Promise<any>[] = [];
4941
+ const promises: Promise<any>[] = [];
4055
4942
 
4056
- let peerToEntries: Map<string, string[]> = new Map();
4057
- let cleanupTimer: ReturnType<typeof setTimeout>[] = [];
4943
+ let peerToEntries: Map<string, string[]> = new Map();
4944
+ let cleanupTimer: ReturnType<typeof setTimeout>[] = [];
4945
+ const explicitTimeout = options?.timeout != null;
4058
4946
 
4059
- for (const { entry, leaders } of entries.values()) {
4060
- for (const leader of leaders.keys()) {
4061
- let set = peerToEntries.get(leader);
4062
- if (!set) {
4063
- set = [];
4064
- peerToEntries.set(leader, set);
4065
- }
4947
+ for (const { entry, leaders } of entries.values()) {
4948
+ for (const leader of leaders.keys()) {
4949
+ let set = peerToEntries.get(leader);
4950
+ if (!set) {
4951
+ set = [];
4952
+ peerToEntries.set(leader, set);
4953
+ }
4066
4954
 
4067
- set.push(entry.hash);
4068
- }
4955
+ set.push(entry.hash);
4956
+ }
4069
4957
 
4070
- const pendingPrev = this._pendingDeletes.get(entry.hash);
4071
- if (pendingPrev) {
4072
- promises.push(pendingPrev.promise.promise);
4073
- continue;
4074
- }
4958
+ const pendingPrev = this._pendingDeletes.get(entry.hash);
4959
+ if (pendingPrev) {
4960
+ // If a background prune is already in-flight, an explicit prune request should
4961
+ // still respect the caller's timeout. Otherwise, tests (and user calls) can
4962
+ // block on the longer "checked prune" timeout derived from
4963
+ // `_respondToIHaveTimeout + waitForReplicatorTimeout`, which is intentionally
4964
+ // large for resiliency.
4965
+ if (explicitTimeout) {
4966
+ const timeoutMs = Math.max(0, Math.floor(options?.timeout ?? 0));
4967
+ promises.push(
4968
+ new Promise((resolve, reject) => {
4969
+ // Mirror the checked-prune error prefix so existing callers/tests can
4970
+ // match on the message substring.
4971
+ const timer = setTimeout(() => {
4972
+ reject(
4973
+ new Error(
4974
+ `Timeout for checked pruning after ${timeoutMs}ms (pending=true closed=${this.closed})`,
4975
+ ),
4976
+ );
4977
+ }, timeoutMs);
4978
+ timer.unref?.();
4979
+ pendingPrev.promise.promise
4980
+ .then(resolve, reject)
4981
+ .finally(() => clearTimeout(timer));
4982
+ }),
4983
+ );
4984
+ } else {
4985
+ promises.push(pendingPrev.promise.promise);
4986
+ }
4987
+ continue;
4988
+ }
4075
4989
 
4076
- const minReplicas = decodeReplicas(entry);
4077
- const deferredPromise: DeferredPromise<void> = pDefer();
4990
+ const minReplicas = decodeReplicas(entry);
4991
+ const deferredPromise: DeferredPromise<void> = pDefer();
4078
4992
 
4079
4993
  const clear = () => {
4080
4994
  const pending = this._pendingDeletes.get(entry.hash);
@@ -4084,12 +4998,13 @@ export class SharedLog<
4084
4998
  clearTimeout(timeout);
4085
4999
  };
4086
5000
 
4087
- const resolve = () => {
4088
- clear();
4089
- cleanupTimer.push(
4090
- setTimeout(async () => {
4091
- this._gidPeersHistory.delete(entry.meta.gid);
4092
- this.removePruneRequestSent(entry.hash);
5001
+ const resolve = () => {
5002
+ clear();
5003
+ this.clearCheckedPruneRetry(entry.hash);
5004
+ cleanupTimer.push(
5005
+ setTimeout(async () => {
5006
+ this._gidPeersHistory.delete(entry.meta.gid);
5007
+ this.removePruneRequestSent(entry.hash);
4093
5008
  this._requestIPruneResponseReplicatorSet.delete(entry.hash);
4094
5009
 
4095
5010
  if (
@@ -4133,12 +5048,19 @@ export class SharedLog<
4133
5048
  );
4134
5049
  };
4135
5050
 
4136
- const reject = (e: any) => {
4137
- clear();
4138
- this.removePruneRequestSent(entry.hash);
4139
- this._requestIPruneResponseReplicatorSet.delete(entry.hash);
4140
- deferredPromise.reject(e);
4141
- };
5051
+ const reject = (e: any) => {
5052
+ clear();
5053
+ const isCheckedPruneTimeout =
5054
+ e instanceof Error &&
5055
+ typeof e.message === "string" &&
5056
+ e.message.startsWith("Timeout for checked pruning");
5057
+ if (explicitTimeout || !isCheckedPruneTimeout) {
5058
+ this.clearCheckedPruneRetry(entry.hash);
5059
+ }
5060
+ this.removePruneRequestSent(entry.hash);
5061
+ this._requestIPruneResponseReplicatorSet.delete(entry.hash);
5062
+ deferredPromise.reject(e);
5063
+ };
4142
5064
 
4143
5065
  let cursor: NumberFromType<R>[] | undefined = undefined;
4144
5066
 
@@ -4156,14 +5078,20 @@ export class SharedLog<
4156
5078
  PRUNE_DEBOUNCE_INTERVAL * 2,
4157
5079
  );
4158
5080
 
4159
- const timeout = setTimeout(() => {
4160
- reject(
4161
- new Error(
4162
- `Timeout for checked pruning after ${checkedPruneTimeoutMs}ms (closed=${this.closed})`,
4163
- ),
4164
- );
4165
- }, checkedPruneTimeoutMs);
4166
- timeout.unref?.();
5081
+ const timeout = setTimeout(() => {
5082
+ // For internal/background prune flows (no explicit timeout), retry a few times
5083
+ // to avoid "permanently prunable" entries when `_pendingIHave` expires under
5084
+ // heavy load.
5085
+ if (!explicitTimeout) {
5086
+ this.scheduleCheckedPruneRetry({ entry, leaders });
5087
+ }
5088
+ reject(
5089
+ new Error(
5090
+ `Timeout for checked pruning after ${checkedPruneTimeoutMs}ms (closed=${this.closed})`,
5091
+ ),
5092
+ );
5093
+ }, checkedPruneTimeoutMs);
5094
+ timeout.unref?.();
4167
5095
 
4168
5096
  this._pendingDeletes.set(entry.hash, {
4169
5097
  promise: deferredPromise,
@@ -4200,20 +5128,22 @@ export class SharedLog<
4200
5128
  let existCounter = this._requestIPruneResponseReplicatorSet.get(
4201
5129
  entry.hash,
4202
5130
  );
4203
- if (!existCounter) {
4204
- existCounter = new Set();
4205
- this._requestIPruneResponseReplicatorSet.set(
4206
- entry.hash,
4207
- existCounter,
4208
- );
4209
- }
4210
- existCounter.add(publicKeyHash);
5131
+ if (!existCounter) {
5132
+ existCounter = new Set();
5133
+ this._requestIPruneResponseReplicatorSet.set(
5134
+ entry.hash,
5135
+ existCounter,
5136
+ );
5137
+ }
5138
+ existCounter.add(publicKeyHash);
5139
+ // Seed provider hints so future remote reads can avoid extra round-trips.
5140
+ this.remoteBlocks.hintProviders(entry.hash, [publicKeyHash]);
4211
5141
 
4212
- if (minReplicasValue <= existCounter.size) {
4213
- resolve();
4214
- }
4215
- },
4216
- });
5142
+ if (minReplicasValue <= existCounter.size) {
5143
+ resolve();
5144
+ }
5145
+ },
5146
+ });
4217
5147
 
4218
5148
  promises.push(deferredPromise.promise);
4219
5149
  }
@@ -4249,16 +5179,58 @@ export class SharedLog<
4249
5179
  }
4250
5180
  };
4251
5181
 
4252
- for (const [k, v] of peerToEntries) {
4253
- emitMessages(v, k);
4254
- }
5182
+ for (const [k, v] of peerToEntries) {
5183
+ emitMessages(v, k);
5184
+ }
4255
5185
 
4256
- let cleanup = () => {
4257
- for (const timer of cleanupTimer) {
4258
- clearTimeout(timer);
5186
+ // Keep remote `_pendingIHave` alive in the common "leader doesn't have entry yet"
5187
+ // case. This is intentionally disabled when an explicit timeout is provided to
5188
+ // preserve unit tests that assert remote `_pendingIHave` clears promptly.
5189
+ if (!explicitTimeout && peerToEntries.size > 0) {
5190
+ const respondToIHaveTimeout = Number(this._respondToIHaveTimeout ?? 0);
5191
+ const resendIntervalMs = Math.min(
5192
+ CHECKED_PRUNE_RESEND_INTERVAL_MAX_MS,
5193
+ Math.max(
5194
+ CHECKED_PRUNE_RESEND_INTERVAL_MIN_MS,
5195
+ Math.floor(respondToIHaveTimeout / 2) || 1_000,
5196
+ ),
5197
+ );
5198
+ let inFlight = false;
5199
+ const timer = setInterval(() => {
5200
+ if (inFlight) return;
5201
+ if (this.closed) return;
5202
+
5203
+ const pendingByPeer: [string, string[]][] = [];
5204
+ for (const [peer, hashes] of peerToEntries) {
5205
+ const pending = hashes.filter((h) => this._pendingDeletes.has(h));
5206
+ if (pending.length > 0) {
5207
+ pendingByPeer.push([peer, pending]);
5208
+ }
5209
+ }
5210
+ if (pendingByPeer.length === 0) {
5211
+ clearInterval(timer);
5212
+ return;
5213
+ }
5214
+
5215
+ inFlight = true;
5216
+ Promise.allSettled(
5217
+ pendingByPeer.map(([peer, hashes]) =>
5218
+ emitMessages(hashes, peer).catch(() => {}),
5219
+ ),
5220
+ ).finally(() => {
5221
+ inFlight = false;
5222
+ });
5223
+ }, resendIntervalMs);
5224
+ timer.unref?.();
5225
+ cleanupTimer.push(timer as any);
4259
5226
  }
4260
- this._closeController.signal.removeEventListener("abort", cleanup);
4261
- };
5227
+
5228
+ let cleanup = () => {
5229
+ for (const timer of cleanupTimer) {
5230
+ clearTimeout(timer);
5231
+ }
5232
+ this._closeController.signal.removeEventListener("abort", cleanup);
5233
+ };
4262
5234
 
4263
5235
  Promise.allSettled(promises).finally(cleanup);
4264
5236
  this._closeController.signal.addEventListener("abort", cleanup);
@@ -4330,12 +5302,21 @@ export class SharedLog<
4330
5302
  * that we potentially need to share with other peers
4331
5303
  */
4332
5304
 
4333
- if (this.closed) {
4334
- return;
4335
- }
5305
+ if (this.closed) {
5306
+ return;
5307
+ }
4336
5308
 
4337
5309
  await this.log.trim();
4338
5310
 
5311
+ const batchedChanges = Array.isArray(changeOrChanges[0])
5312
+ ? (changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>[])
5313
+ : [changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>];
5314
+ const changes = batchedChanges.flat();
5315
+ // On removed ranges (peer leaves / shrink), gid-level history can hide
5316
+ // per-entry gaps. Force a fresh delivery pass for reassigned entries.
5317
+ const forceFreshDelivery = changes.some((change) => change.type === "removed");
5318
+ const gidPeersHistorySnapshot = new Map<string, Set<string> | undefined>();
5319
+
4339
5320
  const changed = false;
4340
5321
 
4341
5322
  try {
@@ -4345,7 +5326,7 @@ export class SharedLog<
4345
5326
  > = new Map();
4346
5327
 
4347
5328
  for await (const entryReplicated of toRebalance<R>(
4348
- changeOrChanges,
5329
+ changes,
4349
5330
  this.entryCoordinatesIndex,
4350
5331
  this.recentlyRebalanced,
4351
5332
  )) {
@@ -4353,7 +5334,16 @@ export class SharedLog<
4353
5334
  break;
4354
5335
  }
4355
5336
 
4356
- let oldPeersSet = this._gidPeersHistory.get(entryReplicated.gid);
5337
+ let oldPeersSet: Set<string> | undefined;
5338
+ if (!forceFreshDelivery) {
5339
+ const gid = entryReplicated.gid;
5340
+ oldPeersSet = gidPeersHistorySnapshot.get(gid);
5341
+ if (!gidPeersHistorySnapshot.has(gid)) {
5342
+ const existing = this._gidPeersHistory.get(gid);
5343
+ oldPeersSet = existing ? new Set(existing) : undefined;
5344
+ gidPeersHistorySnapshot.set(gid, oldPeersSet);
5345
+ }
5346
+ }
4357
5347
  let isLeader = false;
4358
5348
 
4359
5349
  let currentPeers = await this.findLeaders(
@@ -4432,32 +5422,51 @@ export class SharedLog<
4432
5422
  }
4433
5423
  }
4434
5424
 
4435
- async _onUnsubscription(evt: CustomEvent<UnsubcriptionEvent>) {
4436
- logger.trace(
4437
- `Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(
4438
- evt.detail.topics.map((x) => x),
4439
- )} '`,
4440
- );
4441
- this.latestReplicationInfoMessage.delete(evt.detail.from.hashcode());
5425
+ async _onUnsubscription(evt: CustomEvent<UnsubcriptionEvent>) {
5426
+ logger.trace(
5427
+ `Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(
5428
+ evt.detail.topics.map((x) => x),
5429
+ )} '`,
5430
+ );
5431
+ if (!evt.detail.topics.includes(this.topic)) {
5432
+ return;
5433
+ }
4442
5434
 
4443
- return this.handleSubscriptionChange(
4444
- evt.detail.from,
4445
- evt.detail.topics,
4446
- false,
4447
- );
4448
- }
5435
+ const fromHash = evt.detail.from.hashcode();
5436
+ this._replicationInfoBlockedPeers.add(fromHash);
4449
5437
 
4450
- async _onSubscription(evt: CustomEvent<SubscriptionEvent>) {
4451
- logger.trace(
4452
- `New peer '${evt.detail.from.hashcode()}' connected to '${JSON.stringify(
4453
- evt.detail.topics.map((x) => x),
4454
- )}'`,
4455
- );
4456
- this.remoteBlocks.onReachable(evt.detail.from);
5438
+ // Keep a per-peer timestamp watermark when we observe an unsubscribe. This
5439
+ // prevents late/out-of-order replication-info messages from re-introducing
5440
+ // stale segments for a peer that has already left the topic.
5441
+ const now = BigInt(+new Date());
5442
+ const prev = this.latestReplicationInfoMessage.get(fromHash);
5443
+ if (!prev || prev < now) {
5444
+ this.latestReplicationInfoMessage.set(fromHash, now);
5445
+ }
5446
+
5447
+ return this.handleSubscriptionChange(
5448
+ evt.detail.from,
5449
+ evt.detail.topics,
5450
+ false,
5451
+ );
5452
+ }
5453
+
5454
+ async _onSubscription(evt: CustomEvent<SubscriptionEvent>) {
5455
+ logger.trace(
5456
+ `New peer '${evt.detail.from.hashcode()}' connected to '${JSON.stringify(
5457
+ evt.detail.topics.map((x) => x),
5458
+ )}'`,
5459
+ );
5460
+ if (!evt.detail.topics.includes(this.topic)) {
5461
+ return;
5462
+ }
5463
+
5464
+ this.remoteBlocks.onReachable(evt.detail.from);
5465
+ this._replicationInfoBlockedPeers.delete(evt.detail.from.hashcode());
4457
5466
 
4458
- return this.handleSubscriptionChange(
4459
- evt.detail.from,
4460
- evt.detail.topics,
5467
+ return this.handleSubscriptionChange(
5468
+ evt.detail.from,
5469
+ evt.detail.topics,
4461
5470
  true,
4462
5471
  );
4463
5472
  }