@peerbit/shared-log 13.2.10 → 13.2.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -6,7 +6,11 @@ import {
6
6
  variant,
7
7
  } from "@dao-xyz/borsh";
8
8
  import type { AnyStore } from "@peerbit/any-store";
9
- import { AnyBlockStore, RemoteBlocks } from "@peerbit/blocks";
9
+ import {
10
+ AnyBlockStore,
11
+ type EagerBlocksSetting,
12
+ RemoteBlocks,
13
+ } from "@peerbit/blocks";
10
14
  import { cidifyString } from "@peerbit/blocks-interface";
11
15
  import { Cache } from "@peerbit/cache";
12
16
  import {
@@ -105,6 +109,7 @@ import {
105
109
  BACKGROUND_MESSAGE_PRIORITY,
106
110
  CONVERGENCE_MESSAGE_PRIORITY,
107
111
  DataMessage,
112
+ DeliveryError,
108
113
  MessageHeader,
109
114
  NotStartedError,
110
115
  type RouteHint,
@@ -278,10 +283,20 @@ type PendingIHave<T> = {
278
283
  resetTimeout: () => void;
279
284
  requesting: Set<string>;
280
285
  clear: () => void;
281
- callback: (entry: Entry<T>) => void;
286
+ callback: (entry: Entry<T>) => MaybePromise<void>;
282
287
  expiresAt?: number;
283
288
  };
284
289
 
290
+ type PeerReceiveLeaseBucket = {
291
+ active: number;
292
+ drain?: DeferredPromise<void>;
293
+ };
294
+
295
+ type PeerReceiveLeaseState = {
296
+ current: PeerReceiveLeaseBucket;
297
+ activeBuckets: Set<PeerReceiveLeaseBucket>;
298
+ };
299
+
285
300
  const toLocalPublicSignKey = (
286
301
  key: PublicSignKey | string,
287
302
  ): PublicSignKey | undefined => {
@@ -2307,6 +2322,64 @@ const isTransientReplicationAnnouncementError = (
2307
2322
  return constructorName === "TimeoutError" || name === "TimeoutError";
2308
2323
  };
2309
2324
 
2325
+ /**
2326
+ * Directed transport-delivery repair is allowed to retry explicit delivery
2327
+ * failures in addition to timeouts. A DirectStream ACK confirms receipt of the
2328
+ * signed envelope, not successful application by the receiver. Keep this
2329
+ * separate from the primary fanout classifier above so replicate() rejection
2330
+ * semantics remain unchanged for programming, serialization, and lifecycle
2331
+ * errors.
2332
+ */
2333
+ const isTransientReplicationAnnouncementRepairError = (
2334
+ error: unknown,
2335
+ seen = new Set<unknown>(),
2336
+ ): boolean => {
2337
+ if (
2338
+ error != null &&
2339
+ (typeof error === "object" || typeof error === "function")
2340
+ ) {
2341
+ if (seen.has(error)) {
2342
+ return false;
2343
+ }
2344
+ seen.add(error);
2345
+ }
2346
+
2347
+ if (error instanceof DeliveryError || error instanceof TimeoutError) {
2348
+ return true;
2349
+ }
2350
+
2351
+ const nested = (error as { errors?: unknown })?.errors;
2352
+ if (Array.isArray(nested) && nested.length > 0) {
2353
+ return nested.every((item) =>
2354
+ isTransientReplicationAnnouncementRepairError(item, new Set(seen)),
2355
+ );
2356
+ }
2357
+
2358
+ const cause = (error as { cause?: unknown })?.cause;
2359
+ if (
2360
+ cause != null &&
2361
+ isTransientReplicationAnnouncementRepairError(cause, seen)
2362
+ ) {
2363
+ return true;
2364
+ }
2365
+
2366
+ const constructorName =
2367
+ typeof (error as { constructor?: { name?: unknown } })?.constructor
2368
+ ?.name === "string"
2369
+ ? (error as { constructor: { name: string } }).constructor.name
2370
+ : "";
2371
+ const name =
2372
+ typeof (error as { name?: unknown })?.name === "string"
2373
+ ? (error as { name: string }).name
2374
+ : "";
2375
+ return (
2376
+ constructorName === "DeliveryError" ||
2377
+ name === "DeliveryError" ||
2378
+ constructorName === "TimeoutError" ||
2379
+ name === "TimeoutError"
2380
+ );
2381
+ };
2382
+
2310
2383
  interface IndexableDomain<R extends "u32" | "u64"> {
2311
2384
  numbers: Numbers<R>;
2312
2385
  constructorEntry: new (properties: {
@@ -2511,32 +2584,61 @@ export type SharedLogOptions<
2511
2584
  strictFullReplicaFallback?: boolean;
2512
2585
  compatibility?: number;
2513
2586
  domain?: ReplicationDomainConstructor<D>;
2514
- eagerBlocks?: boolean | { cacheSize?: number };
2587
+ eagerBlocks?: EagerBlocksSetting;
2515
2588
  fanout?: SharedLogFanoutOptions;
2516
2589
  };
2517
2590
 
2518
2591
  /**
2519
- * Native defaults a client can advertise for shared-log programs opened on
2520
- * it (the peerbit client's native network preset sets these; see
2521
- * `peerbit/rust`). They fill in open options the caller left undefined;
2522
- * explicit per-open options (including `false`) always win. Without the
2523
- * property on the client, behavior is unchanged.
2592
+ * Runtime defaults a client can advertise for shared-log programs opened on
2593
+ * it. The historical name is retained because the peerbit native network
2594
+ * preset introduced this hook. Defaults fill in open options the caller left
2595
+ * undefined; explicit per-open options (including `false`) always win.
2596
+ * Without the property on the client, behavior is unchanged.
2524
2597
  */
2525
2598
  export type SharedLogNativeDefaults = {
2526
2599
  nativeBackbone?: SharedLogOptions<any, any, any>["nativeBackbone"];
2527
2600
  nativeGraph?: LogProperties<any>["nativeGraph"];
2528
2601
  sync?: Pick<SyncOptions<any>, "rawExchangeHeads" | "nativeWireSync">;
2602
+ /**
2603
+ * Per-channel defaults applied only when the caller opts into SharedLog
2604
+ * fanout. Explicit per-open channel options take precedence.
2605
+ */
2606
+ fanout?: Pick<SharedLogFanoutOptions, "channel">;
2529
2607
  };
2530
2608
 
2531
2609
  type NodeWithSharedLogNativeDefaults = {
2532
2610
  sharedLogNativeDefaults?: SharedLogNativeDefaults;
2533
2611
  };
2534
2612
 
2613
+ type SharedLogFanoutChannelOptions = NonNullable<
2614
+ SharedLogFanoutOptions["channel"]
2615
+ >;
2616
+
2617
+ const mergeDefinedFanoutChannelOptions = (
2618
+ ...sources: Array<SharedLogFanoutChannelOptions | undefined>
2619
+ ): SharedLogFanoutChannelOptions | undefined => {
2620
+ let merged: Record<string, unknown> | undefined;
2621
+ for (const source of sources) {
2622
+ if (!source) {
2623
+ continue;
2624
+ }
2625
+ for (const [key, value] of Object.entries(source)) {
2626
+ if (value === undefined) {
2627
+ continue;
2628
+ }
2629
+ merged ??= {};
2630
+ merged[key] = value;
2631
+ }
2632
+ }
2633
+ return merged as SharedLogFanoutChannelOptions | undefined;
2634
+ };
2635
+
2535
2636
  const applySharedLogNativeDefaults = <
2536
2637
  O extends {
2537
2638
  nativeBackbone?: SharedLogOptions<any, any, any>["nativeBackbone"];
2538
2639
  nativeGraph?: LogProperties<any>["nativeGraph"];
2539
2640
  sync?: SyncOptions<any>;
2641
+ fanout?: SharedLogFanoutOptions;
2540
2642
  },
2541
2643
  >(
2542
2644
  options: O | undefined,
@@ -2555,11 +2657,21 @@ const applySharedLogNativeDefaults = <
2555
2657
  options?.sync?.nativeWireSync ?? defaults.sync?.nativeWireSync,
2556
2658
  }
2557
2659
  : undefined;
2660
+ const fanout = options?.fanout
2661
+ ? {
2662
+ ...options.fanout,
2663
+ channel: mergeDefinedFanoutChannelOptions(
2664
+ defaults.fanout?.channel,
2665
+ options.fanout.channel,
2666
+ ),
2667
+ }
2668
+ : undefined;
2558
2669
  return {
2559
2670
  ...options,
2560
2671
  nativeBackbone: options?.nativeBackbone ?? defaults.nativeBackbone,
2561
2672
  nativeGraph: options?.nativeGraph ?? defaults.nativeGraph,
2562
2673
  sync,
2674
+ fanout,
2563
2675
  } as O;
2564
2676
  };
2565
2677
 
@@ -2582,6 +2694,14 @@ const CHECKED_PRUNE_RETRY_MAX_DELAY_MS = 30_000;
2582
2694
  // DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
2583
2695
  const RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL = 1000;
2584
2696
  const REPLICATION_ANNOUNCEMENT_RETRY_INTERVAL = 1000;
2697
+ const REPLICATION_ANNOUNCEMENT_REPAIR_INTERVAL = 1000;
2698
+ const REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS = 3;
2699
+ // Repair one bounded cohort per mutation generation. The subscriber snapshot
2700
+ // is a best-effort cache and can contain thousands of entries, so attempting
2701
+ // the whole cache after every role mutation would turn convergence repair into
2702
+ // an unbounded burst of separately signed, acknowledged messages. A cursor
2703
+ // retained across generations rotates best-effort coverage over later changes.
2704
+ const REPLICATION_ANNOUNCEMENT_REPAIR_TARGETS_PER_GENERATION = 8;
2585
2705
  const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE = 0.01;
2586
2706
  const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT = 0.005;
2587
2707
  const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT = 0.001;
@@ -2618,6 +2738,13 @@ const NATIVE_ED25519_VERIFY_BATCH_MIN_ENTRIES = 16;
2618
2738
  const hasPreverifiedSignature = (entry: Entry<any>) =>
2619
2739
  (entry as { __peerbitSignatureVerified?: unknown })
2620
2740
  .__peerbitSignatureVerified === true;
2741
+
2742
+ type ReplicationAnnouncementRepairTarget = {
2743
+ key: PublicSignKey;
2744
+ generation: number;
2745
+ attempts: number;
2746
+ done: boolean;
2747
+ };
2621
2748
  // In sparse topologies (browser/relay), peers can learn about replicators via broadcast
2622
2749
  // replication announcements without having a direct connection that emits unsubscribe
2623
2750
  // on abrupt churn. Probe conservatively so a single missed ACK does not evict a
@@ -2631,9 +2758,13 @@ const REPLICATOR_LIVENESS_PROBE_FAILURES_TO_EVICT = 2;
2631
2758
  const CHURN_REPAIR_RETRY_SCHEDULE_MS = [
2632
2759
  0, 1_000, 3_000, 7_000, 15_000, 30_000, 45_000,
2633
2760
  ];
2761
+ // Preserve the bounded retry window for transient local misses, but serialize
2762
+ // delayed warmup sends per target so fixed snapshots cannot overlap and amplify
2763
+ // large transfers. Every queued pass re-checks current peer knowledge on entry.
2634
2764
  const JOIN_WARMUP_RETRY_SCHEDULE_MS = [
2635
2765
  0, 1_000, 3_000, 7_000, 15_000, 30_000, 60_000,
2636
2766
  ];
2767
+ const JOIN_WARMUP_SEND_SPACING_MS = 250;
2637
2768
  const JOIN_AUTHORITATIVE_RETRY_SCHEDULE_MS = [
2638
2769
  0, 1_000, 3_000, 7_000, 15_000, 30_000, 60_000,
2639
2770
  ];
@@ -2664,6 +2795,48 @@ type RepairMetricBucket = {
2664
2795
  };
2665
2796
  type RepairMetrics = Record<RepairDispatchMode, RepairMetricBucket>;
2666
2797
 
2798
+ type JoinWarmupSendState<R extends "u32" | "u64"> = {
2799
+ bypassKnownPeerHints: boolean;
2800
+ entries: Map<string, RepairDispatchEntry<R>>;
2801
+ generation: object;
2802
+ lastCompletedAt: number;
2803
+ pending: boolean;
2804
+ running: boolean;
2805
+ };
2806
+
2807
+ type JoinWarmupRetryTimer = {
2808
+ handle: ReturnType<typeof setTimeout>;
2809
+ resolve?: () => void;
2810
+ };
2811
+
2812
+ type JoinWarmupScheduledRetryBatch<R extends "u32" | "u64"> = {
2813
+ bypassKnownPeerHints: boolean;
2814
+ entries: Map<string, RepairDispatchEntry<R>>;
2815
+ remainingAttempts: number;
2816
+ };
2817
+
2818
+ type JoinWarmupScheduledRetryCohort<R extends "u32" | "u64"> = {
2819
+ batches: JoinWarmupScheduledRetryBatch<R>[];
2820
+ dueAt: number;
2821
+ };
2822
+
2823
+ type JoinWarmupScheduledRetrySlot<R extends "u32" | "u64"> = {
2824
+ cohorts: JoinWarmupScheduledRetryCohort<R>[];
2825
+ head: number;
2826
+ timer?: JoinWarmupRetryTimer;
2827
+ timerDueAt?: number;
2828
+ };
2829
+
2830
+ type JoinWarmupScheduledRetries<R extends "u32" | "u64"> = {
2831
+ generation: object;
2832
+ slotsByDelay: Map<number, JoinWarmupScheduledRetrySlot<R>>;
2833
+ };
2834
+
2835
+ type RepairSweepOptimisticPeerState = {
2836
+ count: number;
2837
+ generation: object;
2838
+ };
2839
+
2667
2840
  const REPAIR_DISPATCH_MODES: RepairDispatchMode[] = [
2668
2841
  "join-warmup",
2669
2842
  "join-authoritative",
@@ -2816,6 +2989,11 @@ export type DeliveryOptions = {
2816
2989
  reliability?: DeliveryReliability;
2817
2990
  minAcks?: number;
2818
2991
  requireRecipients?: boolean;
2992
+ /**
2993
+ * Transport priority for directed RPC delivery. Fanout unicast already uses
2994
+ * its control lane, so this only changes the direct/fallback RPC path.
2995
+ */
2996
+ priority?: number;
2819
2997
  timeout?: number;
2820
2998
  signal?: AbortSignal;
2821
2999
  };
@@ -3094,6 +3272,13 @@ export interface SharedLogEvents extends ProgramEvents {
3094
3272
  "replicator:mature": CustomEvent<ReplicatorMatureEvent>;
3095
3273
  }
3096
3274
 
3275
+ export type SharedLogRuntimeSnapshot = Readonly<{
3276
+ nativeGraph: Readonly<{
3277
+ active: boolean;
3278
+ useHeads: boolean;
3279
+ }>;
3280
+ }>;
3281
+
3097
3282
  @variant("shared_log")
3098
3283
  export class SharedLog<
3099
3284
  T,
@@ -3158,6 +3343,14 @@ export class SharedLog<
3158
3343
  private _subscriptionChangeCallbacks?: Set<Promise<void>>;
3159
3344
  private _acceptSubscriptionChangeCallbacks = false;
3160
3345
  private _replicationLifecycleController?: AbortController;
3346
+ private _activeReceiveHandlersByPeer!: Map<string, PeerReceiveLeaseState>;
3347
+ private _receiveHandlerDrainByPeer!: Map<string, Set<Promise<void>>>;
3348
+ private _receiveCleanupGateByPeer!: Map<string, number>;
3349
+ private _subscriptionOpeningEpochByPeer!: Map<string, object>;
3350
+ private _openingSyncCapabilitiesByPeer!: Map<
3351
+ string,
3352
+ { epoch: object; capabilities: number }
3353
+ >;
3161
3354
  private _onFanoutDataFn?: (arg: any) => void;
3162
3355
  private _onFanoutUnicastFn?: (arg: any) => void;
3163
3356
  private _fanoutChannel?: FanoutChannel;
@@ -3173,6 +3366,7 @@ export class SharedLog<
3173
3366
  private _closeController!: AbortController;
3174
3367
  private _respondToIHaveTimeout!: any;
3175
3368
  private _checkedPrune!: CheckedPruneCoordinator<T, R>;
3369
+ private _pendingIHaveCallbacks!: Set<Promise<void>>;
3176
3370
  private _pendingIHaveExpiryTimer?: ReturnType<typeof setTimeout>;
3177
3371
  private _pendingIHaveExpiryDeadline = Number.POSITIVE_INFINITY;
3178
3372
 
@@ -3204,6 +3398,18 @@ export class SharedLog<
3204
3398
  { attempts: number; timer?: ReturnType<typeof setTimeout> }
3205
3399
  >;
3206
3400
  private _replicationInfoApplyQueueByPeer!: Map<string, Promise<void>>;
3401
+ // Local receive generations fence replication-info handlers that were admitted
3402
+ // before a liveness eviction but reach the per-peer apply lane after it. Unlike
3403
+ // message timestamps, these tokens never compare clocks from different peers.
3404
+ private _replicationInfoReceiveEpochByPeer!: Map<string, object>;
3405
+ // Subscription callbacks can overlap because removing a replicator mutates the
3406
+ // replication index asynchronously. Keep that lifecycle separate from message
3407
+ // timestamps so a reconnect can synchronously revoke an older unsubscribe.
3408
+ private _subscriptionEpochByPeer!: Map<string, object>;
3409
+ // A superseded removal may be the queue item that actually observed an active
3410
+ // replicator. Carry that leave obligation to the transition that ultimately
3411
+ // wins, while a winning reconnect clears it without emitting a stale leave.
3412
+ private _pendingReplicatorLeaveByPeer!: Set<string>;
3207
3413
  private _replicatorLivenessSweepRunning!: boolean;
3208
3414
  private _replicatorLivenessTimer?: ReturnType<typeof setInterval>;
3209
3415
  private _replicatorLivenessTargets!: string[];
@@ -4175,6 +4381,20 @@ export class SharedLog<
4175
4381
  private _replicationAnnouncementRetryPending!: boolean;
4176
4382
  private _replicationAnnouncementRetryGeneration!: number;
4177
4383
  private _replicationAnnouncementRetryController!: AbortController;
4384
+ private replicationAnnouncementRepairDebounced:
4385
+ | ReturnType<typeof debounceFixedInterval>
4386
+ | undefined;
4387
+ private _replicationAnnouncementRepairPending!: boolean;
4388
+ private _replicationAnnouncementRepairGeneration!: number;
4389
+ private _replicationAnnouncementRepairGenerationController!: AbortController;
4390
+ private _replicationAnnouncementRepairTargets!: Map<
4391
+ string,
4392
+ ReplicationAnnouncementRepairTarget
4393
+ >;
4394
+ private _replicationAnnouncementRepairCohortSelected!: boolean;
4395
+ private _replicationAnnouncementRepairFairCursorHash!: string | undefined;
4396
+ private _replicationAnnouncementRepairMaxAttempts!: number;
4397
+ private _replicationAnnouncementRepairController!: AbortController;
4178
4398
 
4179
4399
  // A fn for debouncing the calls for pruning
4180
4400
  pruneDebouncedFn!: DebouncedAccumulatorMap<{
@@ -4210,6 +4430,7 @@ export class SharedLog<
4210
4430
  private _repairSweepRunning!: boolean;
4211
4431
  private _repairSweepPendingModes!: Set<RepairDispatchMode>;
4212
4432
  private _repairSweepPendingPeersByMode!: Map<RepairDispatchMode, Set<string>>;
4433
+ private _repairSweepJoinWarmupGenerationByTarget!: Map<string, object>;
4213
4434
  private _repairFrontierByMode!: Map<
4214
4435
  RepairDispatchMode,
4215
4436
  Map<string, Map<string, RepairDispatchEntry<R>>>
@@ -4222,10 +4443,24 @@ export class SharedLog<
4222
4443
  RepairDispatchMode,
4223
4444
  Set<string>
4224
4445
  >;
4446
+ private _joinWarmupGenerationByTarget!: Map<string, object>;
4447
+ private _joinWarmupSendStateByTarget!: Map<
4448
+ string,
4449
+ JoinWarmupSendState<R>
4450
+ >;
4451
+ private _joinWarmupRetryTimersByTarget!: Map<
4452
+ string,
4453
+ Set<JoinWarmupRetryTimer>
4454
+ >;
4455
+ private _joinWarmupScheduledRetriesByTarget!: Map<
4456
+ string,
4457
+ JoinWarmupScheduledRetries<R>
4458
+ >;
4225
4459
  private _repairSweepOptimisticGidPeersPending!: Map<
4226
4460
  string,
4227
- Map<string, number>
4461
+ Map<string, RepairSweepOptimisticPeerState>
4228
4462
  >;
4463
+ private _repairSweepOptimisticGidsByPeer!: Map<string, Set<string>>;
4229
4464
  private _entryKnownPeers!: Map<string, Set<string>>;
4230
4465
  private _entryKnownPeerObservedAt!: Map<string, Map<string, number>>;
4231
4466
  private _joinAuthoritativeRepairTimersByDelay!: Map<
@@ -4289,22 +4524,39 @@ export class SharedLog<
4289
4524
  this.rpc = new RPC();
4290
4525
  this._checkedPrune = new CheckedPruneCoordinator<T, R>();
4291
4526
  this._pendingIHave = new Map();
4527
+ this._pendingIHaveCallbacks = new Set();
4292
4528
  this.latestReplicationInfoMessage = new Map();
4293
4529
  this._replicationInfoBlockedPeers = new Set();
4294
4530
  this._replicationInfoRequestByPeer = new Map();
4295
4531
  this._replicationInfoApplyQueueByPeer = new Map();
4532
+ this._replicationInfoReceiveEpochByPeer = new Map();
4533
+ this._subscriptionEpochByPeer = new Map();
4534
+ this._pendingReplicatorLeaveByPeer = new Set();
4535
+ this._activeReceiveHandlersByPeer = new Map();
4536
+ this._receiveHandlerDrainByPeer = new Map();
4537
+ this._receiveCleanupGateByPeer = new Map();
4538
+ this._subscriptionOpeningEpochByPeer = new Map();
4539
+ this._openingSyncCapabilitiesByPeer = new Map();
4296
4540
  this._gidPeersHistory = new Map();
4297
4541
  this._repairRetryTimers = new Set();
4298
4542
  this._recentRepairDispatch = new Map();
4299
4543
  this._repairSweepRunning = false;
4300
4544
  this._repairSweepPendingModes = new Set();
4301
4545
  this._repairSweepPendingPeersByMode = createRepairPendingPeersByMode();
4546
+ this._repairSweepJoinWarmupGenerationByTarget = new Map();
4302
4547
  this._repairFrontierByMode = createRepairFrontierByMode() as Map<
4303
4548
  RepairDispatchMode,
4304
4549
  Map<string, Map<string, RepairDispatchEntry<R>>>
4305
4550
  >;
4306
4551
  this._repairFrontierActiveTargetsByMode = createRepairActiveTargetsByMode();
4552
+ this._repairFrontierBypassKnownPeersByMode =
4553
+ createRepairFrontierBypassKnownPeersByMode();
4554
+ this._joinWarmupGenerationByTarget = new Map();
4555
+ this._joinWarmupSendStateByTarget = new Map();
4556
+ this._joinWarmupRetryTimersByTarget = new Map();
4557
+ this._joinWarmupScheduledRetriesByTarget = new Map();
4307
4558
  this._repairSweepOptimisticGidPeersPending = new Map();
4559
+ this._repairSweepOptimisticGidsByPeer = new Map();
4308
4560
  this._entryKnownPeers = new Map();
4309
4561
  this._joinAuthoritativeRepairTimersByDelay = new Map();
4310
4562
  this._joinAuthoritativeRepairPeersByDelay = new Map();
@@ -4327,6 +4579,16 @@ export class SharedLog<
4327
4579
  this._replicationAnnouncementRetryPending = false;
4328
4580
  this._replicationAnnouncementRetryGeneration = 0;
4329
4581
  this._replicationAnnouncementRetryController = new AbortController();
4582
+ this._replicationAnnouncementRepairPending = false;
4583
+ this._replicationAnnouncementRepairGeneration = 0;
4584
+ this._replicationAnnouncementRepairGenerationController =
4585
+ new AbortController();
4586
+ this._replicationAnnouncementRepairTargets = new Map();
4587
+ this._replicationAnnouncementRepairCohortSelected = false;
4588
+ this._replicationAnnouncementRepairFairCursorHash = undefined;
4589
+ this._replicationAnnouncementRepairMaxAttempts =
4590
+ REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS;
4591
+ this._replicationAnnouncementRepairController = new AbortController();
4330
4592
  this.pendingMaturity = new Map();
4331
4593
  this._closeController = new AbortController();
4332
4594
  }
@@ -4689,9 +4951,11 @@ export class SharedLog<
4689
4951
  peer: string;
4690
4952
  message: ExchangeHeadsMessage<any>;
4691
4953
  payload: Uint8Array;
4954
+ priority?: number;
4692
4955
  fanoutUnicastOptions?: { timeoutMs?: number; signal?: AbortSignal };
4693
4956
  }): Promise<void> {
4694
- const { peer, message, payload, fanoutUnicastOptions } = properties;
4957
+ const { peer, message, payload, priority, fanoutUnicastOptions } =
4958
+ properties;
4695
4959
  const hints = await this._getSortedRouteHints(peer);
4696
4960
  const hasDirectHint = hints.some(
4697
4961
  (hint) => hint.kind === "directstream-ack",
@@ -4708,6 +4972,7 @@ export class SharedLog<
4708
4972
  redundancy: 1,
4709
4973
  to: [peer],
4710
4974
  }),
4975
+ priority,
4711
4976
  });
4712
4977
  return;
4713
4978
  } catch {
@@ -4746,6 +5011,7 @@ export class SharedLog<
4746
5011
  redundancy: 1,
4747
5012
  to: [peer],
4748
5013
  }),
5014
+ priority,
4749
5015
  });
4750
5016
  }
4751
5017
 
@@ -5146,6 +5412,7 @@ export class SharedLog<
5146
5412
  peer,
5147
5413
  message,
5148
5414
  payload,
5415
+ priority: delivery.priority,
5149
5416
  fanoutUnicastOptions,
5150
5417
  });
5151
5418
  })(),
@@ -5160,6 +5427,7 @@ export class SharedLog<
5160
5427
  redundancy: 1,
5161
5428
  to: nativeDeliveryPlan.silentTo,
5162
5429
  }),
5430
+ priority: delivery.priority,
5163
5431
  })
5164
5432
  .catch((error) => logger.error(error));
5165
5433
  }
@@ -5300,6 +5568,7 @@ export class SharedLog<
5300
5568
  peer,
5301
5569
  message,
5302
5570
  payload,
5571
+ priority: delivery.priority,
5303
5572
  fanoutUnicastOptions,
5304
5573
  });
5305
5574
  })(),
@@ -5311,6 +5580,7 @@ export class SharedLog<
5311
5580
  this.rpc
5312
5581
  .send(message, {
5313
5582
  mode: new SilentDelivery({ redundancy: 1, to: silentTo }),
5583
+ priority: delivery.priority,
5314
5584
  })
5315
5585
  .catch((error) => logger.error(error));
5316
5586
  }
@@ -5654,6 +5924,184 @@ export class SharedLog<
5654
5924
  }
5655
5925
  }
5656
5926
 
5927
+ private isPeerReceiveAdmissionOpen(
5928
+ peerHash: string,
5929
+ replicationLifecycleController: AbortController | undefined,
5930
+ subscriptionEpoch: object | null,
5931
+ options?: {
5932
+ allowReplicationInfoBlocked?: boolean;
5933
+ allowCleanupGate?: boolean;
5934
+ },
5935
+ ) {
5936
+ return (
5937
+ this.isReplicationLifecycleActive(replicationLifecycleController) &&
5938
+ this.isCurrentSubscriptionEpoch(peerHash, subscriptionEpoch) &&
5939
+ (options?.allowReplicationInfoBlocked === true ||
5940
+ !this._replicationInfoBlockedPeers.has(peerHash)) &&
5941
+ (options?.allowCleanupGate === true ||
5942
+ (this._receiveCleanupGateByPeer.get(peerHash) ?? 0) === 0)
5943
+ );
5944
+ }
5945
+
5946
+ private acquirePeerReceiveLease(
5947
+ peerHash: string,
5948
+ replicationLifecycleController: AbortController | undefined,
5949
+ subscriptionEpoch: object | null,
5950
+ options?: {
5951
+ allowReplicationInfoBlocked?: boolean;
5952
+ allowCleanupGate?: boolean;
5953
+ },
5954
+ ): (() => void) | undefined {
5955
+ if (
5956
+ !this.isPeerReceiveAdmissionOpen(
5957
+ peerHash,
5958
+ replicationLifecycleController,
5959
+ subscriptionEpoch,
5960
+ options,
5961
+ )
5962
+ ) {
5963
+ return;
5964
+ }
5965
+
5966
+ let state = this._activeReceiveHandlersByPeer.get(peerHash);
5967
+ if (!state) {
5968
+ const current: PeerReceiveLeaseBucket = { active: 0 };
5969
+ state = { current, activeBuckets: new Set() };
5970
+ this._activeReceiveHandlersByPeer.set(peerHash, state);
5971
+ }
5972
+ const bucket = state.current;
5973
+ bucket.active += 1;
5974
+ state.activeBuckets.add(bucket);
5975
+ let released = false;
5976
+ return () => {
5977
+ if (released) {
5978
+ return;
5979
+ }
5980
+ released = true;
5981
+ bucket.active -= 1;
5982
+ if (bucket.active > 0) {
5983
+ return;
5984
+ }
5985
+ state.activeBuckets.delete(bucket);
5986
+ bucket.drain?.resolve();
5987
+ if (
5988
+ state.activeBuckets.size === 0 &&
5989
+ state.current.active === 0 &&
5990
+ this._activeReceiveHandlersByPeer.get(peerHash) === state
5991
+ ) {
5992
+ this._activeReceiveHandlersByPeer.delete(peerHash);
5993
+ }
5994
+ };
5995
+ }
5996
+
5997
+ private async drainPeerReceiveHandlers(peerHash: string): Promise<void> {
5998
+ const state = this._activeReceiveHandlersByPeer.get(peerHash);
5999
+ if (!state || state.activeBuckets.size === 0) {
6000
+ return;
6001
+ }
6002
+
6003
+ // Rotate before awaiting so a reconnect/opening generation can keep receiving
6004
+ // sync traffic without joining the drain for the previous subscription. Cleanup
6005
+ // callers gate admission first; terminal callers also repeat until empty.
6006
+ const buckets = [...state.activeBuckets];
6007
+ state.current = { active: 0 };
6008
+ const drain = Promise.all(
6009
+ buckets.map((bucket) => {
6010
+ bucket.drain ??= pDefer<void>();
6011
+ return bucket.drain.promise;
6012
+ }),
6013
+ ).then(() => undefined);
6014
+ let drains = this._receiveHandlerDrainByPeer.get(peerHash);
6015
+ if (!drains) {
6016
+ drains = new Set();
6017
+ this._receiveHandlerDrainByPeer.set(peerHash, drains);
6018
+ }
6019
+ drains.add(drain);
6020
+ try {
6021
+ await drain;
6022
+ } finally {
6023
+ drains.delete(drain);
6024
+ if (drains.size === 0) {
6025
+ this._receiveHandlerDrainByPeer.delete(peerHash);
6026
+ }
6027
+ }
6028
+ }
6029
+
6030
+ private async drainReceiveHandlers(): Promise<void> {
6031
+ for (;;) {
6032
+ const peers = [...this._activeReceiveHandlersByPeer.keys()];
6033
+ if (peers.length === 0) {
6034
+ return;
6035
+ }
6036
+ await Promise.all(
6037
+ peers.map((peerHash) => this.drainPeerReceiveHandlers(peerHash)),
6038
+ );
6039
+ }
6040
+ }
6041
+
6042
+ private runPendingIHaveCallback(
6043
+ pending: PendingIHave<T>,
6044
+ entry: Entry<T>,
6045
+ ): void {
6046
+ const replicationLifecycleController =
6047
+ this._replicationLifecycleController;
6048
+ if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
6049
+ if (this._pendingIHave.get(entry.hash) === pending) {
6050
+ pending.clear();
6051
+ this._pendingIHave.delete(entry.hash);
6052
+ }
6053
+ return;
6054
+ }
6055
+
6056
+ // Register before invoking the callback so a synchronous terminal reentry
6057
+ // cannot make close/drop miss work that has already been admitted.
6058
+ const completion = pDefer<void>();
6059
+ const observed = completion.promise.catch((error) => {
6060
+ if (!(this.isTerminating() && isNotStartedError(error as Error))) {
6061
+ logger.error(error?.toString?.() ?? String(error));
6062
+ }
6063
+ });
6064
+ this._pendingIHaveCallbacks.add(observed);
6065
+ void observed.finally(() => {
6066
+ this._pendingIHaveCallbacks.delete(observed);
6067
+ if (this._pendingIHave.get(entry.hash) === pending) {
6068
+ pending.clear();
6069
+ this._pendingIHave.delete(entry.hash);
6070
+ }
6071
+ });
6072
+
6073
+ try {
6074
+ Promise.resolve(pending.callback(entry)).then(
6075
+ () => completion.resolve(),
6076
+ (error) => completion.reject(error),
6077
+ );
6078
+ } catch (error) {
6079
+ completion.reject(error);
6080
+ }
6081
+ }
6082
+
6083
+ private async drainPendingIHaveCallbacks(): Promise<void> {
6084
+ while (this._pendingIHaveCallbacks.size > 0) {
6085
+ await Promise.all([...this._pendingIHaveCallbacks]);
6086
+ }
6087
+ }
6088
+
6089
+ private blockPeerReceiveAdmission(peerHash: string) {
6090
+ this._receiveCleanupGateByPeer.set(
6091
+ peerHash,
6092
+ (this._receiveCleanupGateByPeer.get(peerHash) ?? 0) + 1,
6093
+ );
6094
+ }
6095
+
6096
+ private unblockPeerReceiveAdmission(peerHash: string) {
6097
+ const remaining = (this._receiveCleanupGateByPeer.get(peerHash) ?? 1) - 1;
6098
+ if (remaining > 0) {
6099
+ this._receiveCleanupGateByPeer.set(peerHash, remaining);
6100
+ } else {
6101
+ this._receiveCleanupGateByPeer.delete(peerHash);
6102
+ }
6103
+ }
6104
+
5657
6105
  private handleReplicationLifecycleSendError(
5658
6106
  error: unknown,
5659
6107
  controller = this._replicationLifecycleController,
@@ -5765,11 +6213,301 @@ export class SharedLog<
5765
6213
  );
5766
6214
  }
5767
6215
 
6216
+ private setupReplicationAnnouncementRepairFunction(
6217
+ interval = REPLICATION_ANNOUNCEMENT_REPAIR_INTERVAL,
6218
+ maxAttempts = REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS,
6219
+ ): void {
6220
+ if (!Number.isSafeInteger(maxAttempts) || maxAttempts <= 0) {
6221
+ throw new RangeError(
6222
+ "Replication announcement repair attempts must be positive",
6223
+ );
6224
+ }
6225
+ this.replicationAnnouncementRepairDebounced?.close();
6226
+ this._replicationAnnouncementRepairController?.abort();
6227
+ this._replicationAnnouncementRepairGenerationController?.abort();
6228
+ this._replicationAnnouncementRepairController = new AbortController();
6229
+ this._replicationAnnouncementRepairGenerationController =
6230
+ new AbortController();
6231
+ this._replicationAnnouncementRepairPending = false;
6232
+ this._replicationAnnouncementRepairGeneration =
6233
+ this._replicationAnnouncementRetryGeneration;
6234
+ this._replicationAnnouncementRepairTargets = new Map();
6235
+ this._replicationAnnouncementRepairCohortSelected = false;
6236
+ this._replicationAnnouncementRepairFairCursorHash = undefined;
6237
+ this._replicationAnnouncementRepairMaxAttempts = maxAttempts;
6238
+ this.replicationAnnouncementRepairDebounced = debounceFixedInterval(
6239
+ () => this.runCurrentReplicationStateAnnouncementRepair(),
6240
+ interval,
6241
+ {
6242
+ leading: false,
6243
+ // The wrapper catches worker failures while it still owns the generation
6244
+ // context. Keep this boundary visibility-only: it must never mutate a
6245
+ // possibly newer generation's pending state.
6246
+ onError: (error) => logger.error(error),
6247
+ },
6248
+ );
6249
+ }
6250
+
6251
+ private cancelCurrentReplicationStateAnnouncementRepair(): void {
6252
+ this._replicationAnnouncementRepairPending = false;
6253
+ this._replicationAnnouncementRepairController?.abort();
6254
+ this._replicationAnnouncementRepairGenerationController?.abort();
6255
+ this.replicationAnnouncementRepairDebounced?.close();
6256
+ this._replicationAnnouncementRepairTargets?.clear();
6257
+ }
6258
+
6259
+ private advanceCurrentReplicationStateAnnouncementRepairGeneration(): void {
6260
+ const generation = this._replicationAnnouncementRetryGeneration;
6261
+ if (generation === this._replicationAnnouncementRepairGeneration) {
6262
+ return;
6263
+ }
6264
+
6265
+ // Abort acknowledged sends carrying the old full-state snapshot before the
6266
+ // primary announcement for the new mutation waits on transport. Otherwise a
6267
+ // stale batch can hold the current state behind DirectStream's seek timeout.
6268
+ this._replicationAnnouncementRepairGenerationController?.abort();
6269
+ this._replicationAnnouncementRepairGenerationController =
6270
+ new AbortController();
6271
+ this._replicationAnnouncementRepairGeneration = generation;
6272
+ this._replicationAnnouncementRepairPending = false;
6273
+ this._replicationAnnouncementRepairTargets.clear();
6274
+ this._replicationAnnouncementRepairCohortSelected = false;
6275
+ }
6276
+
6277
+ private queueCurrentReplicationStateAnnouncementRepair(): void {
6278
+ if (
6279
+ this.closed ||
6280
+ this._closeController.signal.aborted ||
6281
+ this._replicationAnnouncementRepairController.signal.aborted ||
6282
+ !this.replicationAnnouncementRepairDebounced
6283
+ ) {
6284
+ return;
6285
+ }
6286
+
6287
+ this.advanceCurrentReplicationStateAnnouncementRepairGeneration();
6288
+ this._replicationAnnouncementRepairPending = true;
6289
+ void this.replicationAnnouncementRepairDebounced.call();
6290
+ }
6291
+
6292
+ private async runCurrentReplicationStateAnnouncementRepair(): Promise<void> {
6293
+ const generation = this._replicationAnnouncementRetryGeneration;
6294
+ const lifecycleController = this._replicationAnnouncementRepairController;
6295
+ const generationController =
6296
+ this._replicationAnnouncementRepairGenerationController;
6297
+ try {
6298
+ await this.repairCurrentReplicationStateAnnouncement({
6299
+ generation,
6300
+ lifecycleController,
6301
+ generationController,
6302
+ });
6303
+ } catch (error) {
6304
+ if (
6305
+ this.closed ||
6306
+ this._closeController.signal.aborted ||
6307
+ lifecycleController.signal.aborted ||
6308
+ generationController.signal.aborted ||
6309
+ generation !== this._replicationAnnouncementRetryGeneration ||
6310
+ generationController !==
6311
+ this._replicationAnnouncementRepairGenerationController
6312
+ ) {
6313
+ return;
6314
+ }
6315
+ if (isNotStartedError(error as Error)) {
6316
+ return;
6317
+ }
6318
+
6319
+ // Only the worker that still owns the current generation may conclude
6320
+ // that its repair failed. A stale worker must not clear a newer call's
6321
+ // pending flag or attribute its error to the new generation.
6322
+ this._replicationAnnouncementRepairPending = false;
6323
+ logger.error(error);
6324
+ }
6325
+ }
6326
+
6327
+ private async repairCurrentReplicationStateAnnouncement(context?: {
6328
+ generation: number;
6329
+ lifecycleController: AbortController;
6330
+ generationController: AbortController;
6331
+ }): Promise<void> {
6332
+ if (!this._replicationAnnouncementRepairPending) {
6333
+ return;
6334
+ }
6335
+ const generation =
6336
+ context?.generation ?? this._replicationAnnouncementRetryGeneration;
6337
+ const lifecycleController =
6338
+ context?.lifecycleController ??
6339
+ this._replicationAnnouncementRepairController;
6340
+ const generationController =
6341
+ context?.generationController ??
6342
+ this._replicationAnnouncementRepairGenerationController;
6343
+ const segments = (await this.getMyReplicationSegments()).map((range) =>
6344
+ range.toReplicationRange(),
6345
+ );
6346
+ if (
6347
+ this.closed ||
6348
+ this._closeController.signal.aborted ||
6349
+ lifecycleController.signal.aborted ||
6350
+ generationController.signal.aborted
6351
+ ) {
6352
+ return;
6353
+ }
6354
+ if (generation !== this._replicationAnnouncementRetryGeneration) {
6355
+ this.queueCurrentReplicationStateAnnouncementRepair();
6356
+ return;
6357
+ }
6358
+
6359
+ const subscribers =
6360
+ (await this.node.services.pubsub.getSubscribers(this.topic)) ?? [];
6361
+ if (
6362
+ this.closed ||
6363
+ this._closeController.signal.aborted ||
6364
+ lifecycleController.signal.aborted ||
6365
+ generationController.signal.aborted
6366
+ ) {
6367
+ return;
6368
+ }
6369
+ if (generation !== this._replicationAnnouncementRetryGeneration) {
6370
+ this.queueCurrentReplicationStateAnnouncementRepair();
6371
+ return;
6372
+ }
6373
+
6374
+ const selfHash = this.node.identity.publicKey.hashcode();
6375
+ const currentTargets = new Map<string, PublicSignKey>();
6376
+ for (const key of subscribers) {
6377
+ const hash = key.hashcode();
6378
+ if (
6379
+ hash !== selfHash &&
6380
+ !this._replicationInfoBlockedPeers.has(hash) &&
6381
+ !currentTargets.has(hash)
6382
+ ) {
6383
+ currentTargets.set(hash, key);
6384
+ }
6385
+ }
6386
+
6387
+ for (const [hash, target] of this._replicationAnnouncementRepairTargets) {
6388
+ if (target.generation !== generation || !currentTargets.has(hash)) {
6389
+ this._replicationAnnouncementRepairTargets.delete(hash);
6390
+ } else {
6391
+ target.key = currentTargets.get(hash)!;
6392
+ }
6393
+ }
6394
+ if (!this._replicationAnnouncementRepairCohortSelected) {
6395
+ const candidates = [...currentTargets.entries()].sort(([left], [right]) =>
6396
+ left.localeCompare(right),
6397
+ );
6398
+ const cursorIndex = this._replicationAnnouncementRepairFairCursorHash
6399
+ ? candidates.findIndex(
6400
+ ([hash]) =>
6401
+ hash.localeCompare(
6402
+ this._replicationAnnouncementRepairFairCursorHash!,
6403
+ ) > 0,
6404
+ )
6405
+ : 0;
6406
+ const fairStart = cursorIndex < 0 ? 0 : cursorIndex;
6407
+ const fairOrder = [
6408
+ ...candidates.slice(fairStart),
6409
+ ...candidates.slice(0, fairStart),
6410
+ ];
6411
+ const cohort = fairOrder.slice(
6412
+ 0,
6413
+ REPLICATION_ANNOUNCEMENT_REPAIR_TARGETS_PER_GENERATION,
6414
+ );
6415
+ for (const [hash, key] of cohort) {
6416
+ this._replicationAnnouncementRepairTargets.set(hash, {
6417
+ key,
6418
+ generation,
6419
+ attempts: 0,
6420
+ done: false,
6421
+ });
6422
+ }
6423
+ if (cohort.length > 0) {
6424
+ this._replicationAnnouncementRepairFairCursorHash =
6425
+ cohort[cohort.length - 1][0];
6426
+ }
6427
+ this._replicationAnnouncementRepairCohortSelected = true;
6428
+ }
6429
+
6430
+ const batch = [
6431
+ ...this._replicationAnnouncementRepairTargets.entries(),
6432
+ ].filter(([, target]) => !target.done);
6433
+ const snapshot = new AllReplicatingSegmentsMessage({ segments });
6434
+ const results = await Promise.allSettled(
6435
+ batch.map(([, target]) =>
6436
+ this.rpc.send(snapshot, {
6437
+ mode: new AcknowledgeDelivery({
6438
+ to: [target.key],
6439
+ redundancy: 1,
6440
+ }),
6441
+ priority: CONVERGENCE_MESSAGE_PRIORITY,
6442
+ signal: generationController.signal,
6443
+ }),
6444
+ ),
6445
+ );
6446
+ if (
6447
+ this.closed ||
6448
+ this._closeController.signal.aborted ||
6449
+ lifecycleController.signal.aborted ||
6450
+ generationController.signal.aborted
6451
+ ) {
6452
+ return;
6453
+ }
6454
+ if (generation !== this._replicationAnnouncementRetryGeneration) {
6455
+ this.queueCurrentReplicationStateAnnouncementRepair();
6456
+ return;
6457
+ }
6458
+
6459
+ for (const [index, result] of results.entries()) {
6460
+ const [hash, attemptedTarget] = batch[index];
6461
+ const target = this._replicationAnnouncementRepairTargets.get(hash);
6462
+ if (target !== attemptedTarget || target.generation !== generation) {
6463
+ continue;
6464
+ }
6465
+ if (result.status === "fulfilled") {
6466
+ // DirectStream ACKs confirm that the signed transport envelope reached
6467
+ // the target. Applying the contained replication state remains a
6468
+ // receiver-local, best-effort operation.
6469
+ target.done = true;
6470
+ continue;
6471
+ }
6472
+
6473
+ target.attempts += 1;
6474
+ if (!isTransientReplicationAnnouncementRepairError(result.reason)) {
6475
+ target.done = true;
6476
+ logger.error(result.reason);
6477
+ } else if (
6478
+ target.attempts >= this._replicationAnnouncementRepairMaxAttempts
6479
+ ) {
6480
+ target.done = true;
6481
+ logger.trace(
6482
+ "Acknowledged replication announcement repair exhausted for %s",
6483
+ hash,
6484
+ );
6485
+ }
6486
+ }
6487
+
6488
+ if (generation !== this._replicationAnnouncementRetryGeneration) {
6489
+ this.queueCurrentReplicationStateAnnouncementRepair();
6490
+ return;
6491
+ }
6492
+ if (
6493
+ [...this._replicationAnnouncementRepairTargets.values()].some(
6494
+ (target) => !target.done,
6495
+ )
6496
+ ) {
6497
+ void this.replicationAnnouncementRepairDebounced?.call();
6498
+ return;
6499
+ }
6500
+
6501
+ this._replicationAnnouncementRepairPending = false;
6502
+ this._replicationAnnouncementRepairTargets.clear();
6503
+ }
6504
+
5768
6505
  private cancelCurrentReplicationStateAnnouncementRetry(): void {
5769
6506
  this._replicationAnnouncementRetryGeneration += 1;
5770
6507
  this._replicationAnnouncementRetryPending = false;
5771
6508
  this._replicationAnnouncementRetryController?.abort();
5772
6509
  this.replicationAnnouncementRetryDebounced?.close();
6510
+ this.cancelCurrentReplicationStateAnnouncementRepair();
5773
6511
  }
5774
6512
 
5775
6513
  private async sendReplicationAnnouncement(
@@ -5783,10 +6521,12 @@ export class SharedLog<
5783
6521
  // local state; the generation mismatch forces one more current snapshot
5784
6522
  // after that stale send settles.
5785
6523
  this._replicationAnnouncementRetryGeneration += 1;
6524
+ this.advanceCurrentReplicationStateAnnouncementRepairGeneration();
5786
6525
  try {
5787
6526
  await this.rpc.send(message, {
5788
6527
  priority: CONVERGENCE_MESSAGE_PRIORITY,
5789
6528
  });
6529
+ this.queueCurrentReplicationStateAnnouncementRepair();
5790
6530
  } catch (error) {
5791
6531
  // The local replication-index mutation precedes all calls to this
5792
6532
  // wrapper. Preserve the explicit caller's rejection, but independently
@@ -5820,6 +6560,7 @@ export class SharedLog<
5820
6560
  priority: CONVERGENCE_MESSAGE_PRIORITY,
5821
6561
  signal: controller.signal,
5822
6562
  });
6563
+ this.queueCurrentReplicationStateAnnouncementRepair();
5823
6564
  } catch (error) {
5824
6565
  if (
5825
6566
  this.closed ||
@@ -6363,76 +7104,167 @@ export class SharedLog<
6363
7104
 
6364
7105
  private async removeReplicator(
6365
7106
  key: PublicSignKey | string,
6366
- options?: { noEvent?: boolean },
6367
- ) {
7107
+ options?: {
7108
+ cleanupIfSubscriptionSuperseded?: boolean;
7109
+ expectedJoinWarmupGeneration?: object | null;
7110
+ noEvent?: boolean;
7111
+ onRemoved?: (state: { wasReplicator: boolean }) => void;
7112
+ replicationLifecycleController?: AbortController;
7113
+ shouldRemove?: () => boolean;
7114
+ subscriptionEpoch?: object | null;
7115
+ },
7116
+ ): Promise<boolean> {
6368
7117
  const keyHash = typeof key === "string" ? key : key.hashcode();
6369
- const deleted = await this.replicationIndex
6370
- .iterate({
6371
- query: { hash: keyHash },
6372
- })
6373
- .all();
7118
+ const expectedJoinWarmupGeneration =
7119
+ options?.expectedJoinWarmupGeneration !== undefined
7120
+ ? options.expectedJoinWarmupGeneration
7121
+ : (this._joinWarmupGenerationByTarget.get(keyHash) ?? null);
7122
+ const ownsSubscriptionEpoch = () =>
7123
+ options?.subscriptionEpoch === undefined ||
7124
+ this.isCurrentSubscriptionEpoch(keyHash, options.subscriptionEpoch);
7125
+ const ownsReplicationLifecycle = () =>
7126
+ options?.replicationLifecycleController === undefined ||
7127
+ this.isReplicationLifecycleActive(
7128
+ options.replicationLifecycleController,
7129
+ );
7130
+ const cancelExpectedJoinWarmupTarget = () => {
7131
+ if (
7132
+ expectedJoinWarmupGeneration !== null &&
7133
+ this._joinWarmupGenerationByTarget.get(keyHash) ===
7134
+ expectedJoinWarmupGeneration
7135
+ ) {
7136
+ this.cancelJoinWarmupTarget(keyHash);
7137
+ }
7138
+ };
7139
+ const isMe = this.node.identity.publicKey.hashcode() === keyHash;
7140
+ let receiveAdmissionBlocked = false;
7141
+ const blockAndDrainPeerReceives = async () => {
7142
+ if (!receiveAdmissionBlocked) {
7143
+ this.blockPeerReceiveAdmission(keyHash);
7144
+ receiveAdmissionBlocked = true;
7145
+ }
7146
+ await this.drainPeerReceiveHandlers(keyHash);
7147
+ };
7148
+ const cleanupDisconnectedPeer = async () => {
7149
+ await blockAndDrainPeerReceives();
7150
+ this.removePeerFromGidPeerHistory(keyHash);
7151
+ this.cleanupPeerDisconnectTracking(keyHash);
7152
+ this.removeRepairFrontierTarget(keyHash, {
7153
+ expectedJoinWarmupGeneration,
7154
+ });
7155
+ this._recentRepairDispatch.delete(keyHash);
7156
+ if (!isMe) {
7157
+ await this.syncronizer.onPeerDisconnected(keyHash);
7158
+ }
7159
+ };
7160
+ let removed = false;
6374
7161
 
6375
- this.uniqueReplicators.delete(keyHash);
6376
- this._replicatorJoinEmitted.delete(keyHash);
6377
- await this.replicationIndex.del({ query: { hash: keyHash } });
6378
- for (const result of deleted) {
6379
- this.deleteNativeReplicationRange(result.value);
6380
- }
7162
+ // Replication-info updates already serialize per peer. Put the hash-wide
7163
+ // removal on the same queue so a newer reset cannot be deleted underneath
7164
+ // itself by an older unsubscribe callback.
7165
+ await this.withReplicationInfoApplyQueue(keyHash, async () => {
7166
+ if (!ownsReplicationLifecycle()) {
7167
+ return;
7168
+ }
7169
+ if (!ownsSubscriptionEpoch()) {
7170
+ // A reconnect may supersede an unsubscribe before its destructive
7171
+ // removal starts. Still retire the old connection's sync/request state
7172
+ // in lane order so the reconnect barrier cannot inherit stale caches.
7173
+ if (options?.cleanupIfSubscriptionSuperseded) {
7174
+ await cleanupDisconnectedPeer();
7175
+ }
7176
+ return;
7177
+ }
7178
+ if (options?.shouldRemove && !options.shouldRemove()) {
7179
+ return;
7180
+ }
7181
+ const wasReplicator = this.uniqueReplicators.has(keyHash);
6381
7182
 
6382
- await this.updateOldestTimestampFromIndex();
7183
+ const deleted = await this.replicationIndex
7184
+ .iterate({
7185
+ query: { hash: keyHash },
7186
+ })
7187
+ .all();
7188
+ // Liveness evidence can arrive without a subscription transition while
7189
+ // this scan is pending. Stop new receives, drain those already admitted,
7190
+ // then revalidate immediately before the first destructive mutation.
7191
+ await blockAndDrainPeerReceives();
7192
+ if (options?.shouldRemove && !options.shouldRemove()) {
7193
+ return;
7194
+ }
7195
+ // Liveness removal must not cancel a still-current warmup until fresh
7196
+ // activity has been rechecked at the destructive boundary.
7197
+ cancelExpectedJoinWarmupTarget();
6383
7198
 
6384
- const isMe = this.node.identity.publicKey.hashcode() === keyHash;
6385
- if (isMe) {
6386
- // announce that we are no longer replicating
7199
+ await this.replicationIndex.del({ query: { hash: keyHash } });
7200
+ for (const result of deleted) {
7201
+ this.deleteNativeReplicationRange(result.value);
7202
+ }
6387
7203
 
6388
- await this.sendReplicationAnnouncement(
6389
- new AllReplicatingSegmentsMessage({ segments: [] }),
6390
- );
6391
- }
7204
+ // Once the range indexes have removed the peer, make membership agree
7205
+ // before any later awaited bookkeeping can fail.
7206
+ this.uniqueReplicators.delete(keyHash);
7207
+ this._replicatorJoinEmitted.delete(keyHash);
6392
7208
 
6393
- if (options?.noEvent !== true) {
6394
- const publicKey = toLocalPublicSignKey(key);
6395
- if (publicKey) {
6396
- this.events.dispatchEvent(
6397
- new CustomEvent<ReplicationChangeEvent>("replication:change", {
6398
- detail: { publicKey },
6399
- }),
7209
+ await this.updateOldestTimestampFromIndex();
7210
+
7211
+ if (isMe) {
7212
+ // announce that we are no longer replicating
7213
+
7214
+ await this.sendReplicationAnnouncement(
7215
+ new AllReplicatingSegmentsMessage({ segments: [] }),
6400
7216
  );
6401
- } else {
6402
- throw new Error("Key was not a PublicSignKey");
6403
7217
  }
6404
- }
6405
7218
 
6406
- const timestamp = BigInt(+new Date());
6407
- for (const x of deleted) {
6408
- this.replicationChangeDebounceFn.add({
6409
- range: x.value,
6410
- type: "removed",
6411
- timestamp,
6412
- });
6413
- }
7219
+ if (options?.noEvent !== true) {
7220
+ const publicKey = toLocalPublicSignKey(key);
7221
+ if (publicKey) {
7222
+ this.events.dispatchEvent(
7223
+ new CustomEvent<ReplicationChangeEvent>("replication:change", {
7224
+ detail: { publicKey },
7225
+ }),
7226
+ );
7227
+ } else {
7228
+ throw new Error("Key was not a PublicSignKey");
7229
+ }
7230
+ }
7231
+ const timestamp = BigInt(+new Date());
7232
+ for (const x of deleted) {
7233
+ this.replicationChangeDebounceFn.add({
7234
+ range: x.value,
7235
+ type: "removed",
7236
+ timestamp,
7237
+ });
7238
+ }
6414
7239
 
6415
- const pendingMaturity = this.pendingMaturity.get(keyHash);
6416
- if (pendingMaturity) {
6417
- for (const [_k, v] of pendingMaturity) {
6418
- clearTimeout(v.timeout);
7240
+ const pendingMaturity = this.pendingMaturity.get(keyHash);
7241
+ if (pendingMaturity) {
7242
+ for (const [_k, v] of pendingMaturity) {
7243
+ clearTimeout(v.timeout);
7244
+ }
7245
+ this.pendingMaturity.delete(keyHash);
6419
7246
  }
6420
- this.pendingMaturity.delete(keyHash);
6421
- }
6422
7247
 
6423
- // Keep local sync/prune state consistent even when a peer disappears
6424
- // through replication-info updates without a topic unsubscribe event.
6425
- this.removePeerFromGidPeerHistory(keyHash);
6426
- this._checkedPrune.cleanupPeer(keyHash);
6427
- this.removeRepairFrontierTarget(keyHash);
6428
- this._recentRepairDispatch.delete(keyHash);
6429
- if (!isMe) {
6430
- this.syncronizer.onPeerDisconnected(keyHash);
6431
- }
7248
+ // Keep local sync/prune state consistent even when a peer disappears
7249
+ // through replication-info updates without a topic unsubscribe event.
7250
+ await cleanupDisconnectedPeer();
6432
7251
 
6433
- if (!isMe) {
6434
- this.rebalanceParticipationDebounced?.call();
6435
- }
7252
+ if (!isMe) {
7253
+ // Replication-info handlers release their receive lease before joining
7254
+ // this lane. Fence every handler queued behind this successful removal,
7255
+ // regardless of whether it came from liveness, startup pruning, or an
7256
+ // unsubscribe transition.
7257
+ this.advanceReplicationInfoRecoveryEpoch(keyHash);
7258
+ this.rebalanceParticipationDebounced?.call();
7259
+ }
7260
+ removed = true;
7261
+ options?.onRemoved?.({ wasReplicator });
7262
+ }).finally(() => {
7263
+ if (receiveAdmissionBlocked) {
7264
+ this.unblockPeerReceiveAdmission(keyHash);
7265
+ }
7266
+ });
7267
+ return removed;
6436
7268
  }
6437
7269
 
6438
7270
  private async updateOldestTimestampFromIndex() {
@@ -6541,6 +7373,10 @@ export class SharedLog<
6541
7373
  if (this._isTrustedReplicator && !(await this._isTrustedReplicator(from))) {
6542
7374
  return undefined;
6543
7375
  }
7376
+ // Preserve what the peer announced before duplicate filtering can empty the
7377
+ // working array. A repeated authoritative/non-empty announcement is still
7378
+ // proof of live membership after this process reopens.
7379
+ const announcedReplication = ranges.length > 0;
6544
7380
  let isNewReplicator = false;
6545
7381
  let timestamp = BigInt(ts ?? +new Date());
6546
7382
  rebalance = rebalance == null ? true : rebalance;
@@ -6678,13 +7514,13 @@ export class SharedLog<
6678
7514
  // means "nothing new", not "stopped". Removing the peer here would let
6679
7515
  // uniqueReplicators diverge from replicationIndex while the peer's
6680
7516
  // ranges stay indexed.
6681
- const announcedStopped = reset === true && ranges.length === 0;
7517
+ const announcedStopped = reset === true && !announcedReplication;
6682
7518
  const stoppedTransition = announcedStopped
6683
7519
  ? this.uniqueReplicators.delete(fromHash)
6684
7520
  : false;
6685
7521
  if (announcedStopped) {
6686
7522
  this._replicatorJoinEmitted.delete(fromHash);
6687
- } else if (ranges.length > 0) {
7523
+ } else if (announcedReplication) {
6688
7524
  this.uniqueReplicators.add(fromHash);
6689
7525
  }
6690
7526
 
@@ -6822,6 +7658,18 @@ export class SharedLog<
6822
7658
  this.rebalanceParticipationDebounced?.call();
6823
7659
  }
6824
7660
  }
7661
+ if (
7662
+ announcedReplication &&
7663
+ !from.equals(this.node.identity.publicKey) &&
7664
+ !this._replicatorJoinEmitted.has(fromHash)
7665
+ ) {
7666
+ this._replicatorJoinEmitted.add(fromHash);
7667
+ this.events.dispatchEvent(
7668
+ new CustomEvent<ReplicatorJoinEvent>("replicator:join", {
7669
+ detail: { publicKey: from },
7670
+ }),
7671
+ );
7672
+ }
6825
7673
  return diffs;
6826
7674
  }
6827
7675
 
@@ -7068,21 +7916,50 @@ export class SharedLog<
7068
7916
  return observedAt != null && Date.now() - observedAt <= maxAgeMs;
7069
7917
  }
7070
7918
 
7071
- private markRepairSweepOptimisticPeer(gid: string, peer: string) {
7919
+ private markRepairSweepOptimisticPeer(
7920
+ gid: string,
7921
+ peer: string,
7922
+ generation: object,
7923
+ ) {
7072
7924
  let peers = this._repairSweepOptimisticGidPeersPending.get(gid);
7073
7925
  if (!peers) {
7074
7926
  peers = new Map();
7075
7927
  this._repairSweepOptimisticGidPeersPending.set(gid, peers);
7076
7928
  }
7077
- peers.set(peer, (peers.get(peer) || 0) + 1);
7929
+ const current = peers.get(peer);
7930
+ peers.set(peer, {
7931
+ count: current?.generation === generation ? current.count + 1 : 1,
7932
+ generation,
7933
+ });
7934
+ let gids = this._repairSweepOptimisticGidsByPeer.get(peer);
7935
+ if (!gids) {
7936
+ gids = new Set();
7937
+ this._repairSweepOptimisticGidsByPeer.set(peer, gids);
7938
+ }
7939
+ gids.add(gid);
7078
7940
  }
7079
7941
 
7080
7942
  private hasPendingRepairSweepOptimisticPeer(gid: string, peer: string) {
7081
7943
  return (
7082
- (this._repairSweepOptimisticGidPeersPending.get(gid)?.get(peer) || 0) > 0
7944
+ (this._repairSweepOptimisticGidPeersPending.get(gid)?.get(peer)?.count ||
7945
+ 0) > 0
7083
7946
  );
7084
7947
  }
7085
7948
 
7949
+ private clearRepairSweepOptimisticPeer(peer: string) {
7950
+ for (const gid of this._repairSweepOptimisticGidsByPeer.get(peer) ?? []) {
7951
+ const peers = this._repairSweepOptimisticGidPeersPending.get(gid);
7952
+ if (!peers) {
7953
+ continue;
7954
+ }
7955
+ peers.delete(peer);
7956
+ if (peers.size === 0) {
7957
+ this._repairSweepOptimisticGidPeersPending.delete(gid);
7958
+ }
7959
+ }
7960
+ this._repairSweepOptimisticGidsByPeer.delete(peer);
7961
+ }
7962
+
7086
7963
  private createEntryReplicatedForRepair(properties: {
7087
7964
  entry: Entry<T>;
7088
7965
  coordinates: NumberFromType<R>[];
@@ -7133,54 +8010,309 @@ export class SharedLog<
7133
8010
  const timer = setTimeout(() => {
7134
8011
  this._repairRetryTimers.delete(timer);
7135
8012
  resolve();
7136
- }, delayMs);
7137
- timer.unref?.();
7138
- this._repairRetryTimers.add(timer);
8013
+ }, delayMs);
8014
+ timer.unref?.();
8015
+ this._repairRetryTimers.add(timer);
8016
+ });
8017
+ }
8018
+
8019
+ private queueRepairFrontierEntries(
8020
+ mode: RepairDispatchMode,
8021
+ target: string,
8022
+ entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
8023
+ options?: { bypassKnownPeerHints?: boolean },
8024
+ ) {
8025
+ let targets = this._repairFrontierByMode.get(mode);
8026
+ if (!targets) {
8027
+ targets = new Map();
8028
+ this._repairFrontierByMode.set(mode, targets);
8029
+ }
8030
+ let pending = targets.get(target);
8031
+ if (!pending) {
8032
+ pending = new Map();
8033
+ targets.set(target, pending);
8034
+ }
8035
+ for (const [hash, entry] of entries) {
8036
+ pending.set(hash, entry);
8037
+ }
8038
+ if (options?.bypassKnownPeerHints === true) {
8039
+ this._repairFrontierBypassKnownPeersByMode.get(mode)?.add(target);
8040
+ }
8041
+ }
8042
+
8043
+ private clearRepairFrontierHashes(target: string, hashes: Iterable<string>) {
8044
+ const hashList = [...hashes];
8045
+ if (hashList.length === 0) {
8046
+ return;
8047
+ }
8048
+ for (const mode of REPAIR_DISPATCH_MODES) {
8049
+ const pending = this._repairFrontierByMode.get(mode)?.get(target);
8050
+ if (!pending) {
8051
+ continue;
8052
+ }
8053
+ for (const hash of hashList) {
8054
+ pending.delete(hash);
8055
+ }
8056
+ if (pending.size === 0) {
8057
+ this._repairFrontierByMode.get(mode)?.delete(target);
8058
+ this._repairFrontierBypassKnownPeersByMode.get(mode)?.delete(target);
8059
+ }
8060
+ }
8061
+ }
8062
+
8063
+ private getJoinWarmupGeneration(target: string) {
8064
+ let generation = this._joinWarmupGenerationByTarget.get(target);
8065
+ if (!generation) {
8066
+ generation = {};
8067
+ this._joinWarmupGenerationByTarget.set(target, generation);
8068
+ }
8069
+ return generation;
8070
+ }
8071
+
8072
+ private trackJoinWarmupTimer(
8073
+ target: string,
8074
+ timer: JoinWarmupRetryTimer,
8075
+ ) {
8076
+ let timers = this._joinWarmupRetryTimersByTarget.get(target);
8077
+ if (!timers) {
8078
+ timers = new Set();
8079
+ this._joinWarmupRetryTimersByTarget.set(target, timers);
8080
+ }
8081
+ timers.add(timer);
8082
+ this._repairRetryTimers.add(timer.handle);
8083
+ }
8084
+
8085
+ private untrackJoinWarmupTimer(
8086
+ target: string,
8087
+ timer: JoinWarmupRetryTimer,
8088
+ ) {
8089
+ this._repairRetryTimers.delete(timer.handle);
8090
+ const timers = this._joinWarmupRetryTimersByTarget.get(target);
8091
+ if (!timers) {
8092
+ return;
8093
+ }
8094
+ timers.delete(timer);
8095
+ if (timers.size === 0) {
8096
+ this._joinWarmupRetryTimersByTarget.delete(target);
8097
+ }
8098
+ }
8099
+
8100
+ private cancelJoinWarmupTimers(target: string) {
8101
+ const timers = this._joinWarmupRetryTimersByTarget.get(target);
8102
+ if (!timers) {
8103
+ return;
8104
+ }
8105
+ for (const timer of [...timers]) {
8106
+ clearTimeout(timer.handle);
8107
+ timer.resolve?.();
8108
+ this.untrackJoinWarmupTimer(target, timer);
8109
+ }
8110
+ }
8111
+
8112
+ private cancelJoinWarmupTarget(target: string) {
8113
+ this._joinWarmupGenerationByTarget.delete(target);
8114
+ const pendingWarmupPeers =
8115
+ this._repairSweepPendingPeersByMode.get("join-warmup");
8116
+ pendingWarmupPeers?.delete(target);
8117
+ if (pendingWarmupPeers?.size === 0) {
8118
+ this._repairSweepPendingModes.delete("join-warmup");
8119
+ }
8120
+ this._repairSweepJoinWarmupGenerationByTarget.delete(target);
8121
+ this.clearRepairSweepOptimisticPeer(target);
8122
+ this.cancelJoinWarmupTimers(target);
8123
+ this._joinWarmupScheduledRetriesByTarget.delete(target);
8124
+ const state = this._joinWarmupSendStateByTarget.get(target);
8125
+ if (!state) {
8126
+ return;
8127
+ }
8128
+ state.bypassKnownPeerHints = false;
8129
+ state.entries.clear();
8130
+ state.pending = false;
8131
+ if (!state.running) {
8132
+ this._joinWarmupSendStateByTarget.delete(target);
8133
+ }
8134
+ }
8135
+
8136
+ private cancelAllJoinWarmupTargets() {
8137
+ const targets = new Set([
8138
+ ...this._joinWarmupGenerationByTarget.keys(),
8139
+ ...this._joinWarmupRetryTimersByTarget.keys(),
8140
+ ...this._joinWarmupScheduledRetriesByTarget.keys(),
8141
+ ...this._joinWarmupSendStateByTarget.keys(),
8142
+ ]);
8143
+ for (const target of targets) {
8144
+ this.cancelJoinWarmupTarget(target);
8145
+ }
8146
+ }
8147
+
8148
+ private async sleepJoinWarmupTracked(target: string, delayMs: number) {
8149
+ if (delayMs <= 0) {
8150
+ return;
8151
+ }
8152
+ await new Promise<void>((resolve) => {
8153
+ let settled = false;
8154
+ let trackedTimer!: JoinWarmupRetryTimer;
8155
+ const settle = () => {
8156
+ if (settled) {
8157
+ return;
8158
+ }
8159
+ settled = true;
8160
+ this.untrackJoinWarmupTimer(target, trackedTimer);
8161
+ resolve();
8162
+ };
8163
+ const handle = setTimeout(settle, delayMs);
8164
+ handle.unref?.();
8165
+ trackedTimer = { handle, resolve: settle };
8166
+ this.trackJoinWarmupTimer(target, trackedTimer);
7139
8167
  });
7140
8168
  }
7141
8169
 
7142
- private queueRepairFrontierEntries(
7143
- mode: RepairDispatchMode,
8170
+ private scheduleJoinWarmupRetries(
7144
8171
  target: string,
8172
+ generation: object,
8173
+ delaysMs: Iterable<number>,
7145
8174
  entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
7146
- options?: { bypassKnownPeerHints?: boolean },
8175
+ bypassKnownPeerHints: boolean,
7147
8176
  ) {
7148
- let targets = this._repairFrontierByMode.get(mode);
7149
- if (!targets) {
7150
- targets = new Map();
7151
- this._repairFrontierByMode.set(mode, targets);
8177
+ if (
8178
+ this.closed ||
8179
+ this._joinWarmupGenerationByTarget.get(target) !== generation
8180
+ ) {
8181
+ return;
7152
8182
  }
7153
- let pending = targets.get(target);
7154
- if (!pending) {
7155
- pending = new Map();
7156
- targets.set(target, pending);
8183
+ let scheduled = this._joinWarmupScheduledRetriesByTarget.get(target);
8184
+ if (scheduled?.generation !== generation) {
8185
+ this.cancelJoinWarmupTimers(target);
8186
+ this._joinWarmupScheduledRetriesByTarget.delete(target);
8187
+ scheduled = undefined;
7157
8188
  }
7158
- for (const [hash, entry] of entries) {
7159
- pending.set(hash, entry);
8189
+ if (!scheduled) {
8190
+ scheduled = {
8191
+ generation,
8192
+ slotsByDelay: new Map(),
8193
+ };
8194
+ this._joinWarmupScheduledRetriesByTarget.set(target, scheduled);
7160
8195
  }
7161
- if (options?.bypassKnownPeerHints === true) {
7162
- this._repairFrontierBypassKnownPeersByMode.get(mode)?.add(target);
8196
+ const delays = [...new Set(delaysMs)];
8197
+ const batch: JoinWarmupScheduledRetryBatch<R> = {
8198
+ bypassKnownPeerHints,
8199
+ entries: new Map(entries),
8200
+ remainingAttempts: delays.length,
8201
+ };
8202
+ const now = Date.now();
8203
+ for (const delayMs of delays) {
8204
+ let slot = scheduled.slotsByDelay.get(delayMs);
8205
+ if (!slot) {
8206
+ slot = { cohorts: [], head: 0 };
8207
+ scheduled.slotsByDelay.set(delayMs, slot);
8208
+ }
8209
+ const tail = slot.cohorts.at(-1);
8210
+ const dueAt = Math.max(tail?.dueAt ?? 0, now + delayMs);
8211
+ if (tail?.dueAt === dueAt) {
8212
+ tail.batches.push(batch);
8213
+ } else {
8214
+ slot.cohorts.push({
8215
+ batches: [batch],
8216
+ dueAt,
8217
+ });
8218
+ }
8219
+ this.armJoinWarmupRetrySlot(
8220
+ target,
8221
+ scheduled,
8222
+ delayMs,
8223
+ slot,
8224
+ );
7163
8225
  }
7164
8226
  }
7165
8227
 
7166
- private clearRepairFrontierHashes(target: string, hashes: Iterable<string>) {
7167
- const hashList = [...hashes];
7168
- if (hashList.length === 0) {
8228
+ private armJoinWarmupRetrySlot(
8229
+ target: string,
8230
+ scheduled: JoinWarmupScheduledRetries<R>,
8231
+ delayMs: number,
8232
+ slot: JoinWarmupScheduledRetrySlot<R>,
8233
+ ) {
8234
+ const nextDueAt = slot.cohorts[slot.head]?.dueAt;
8235
+ if (nextDueAt == null) {
7169
8236
  return;
7170
8237
  }
7171
- for (const mode of REPAIR_DISPATCH_MODES) {
7172
- const pending = this._repairFrontierByMode.get(mode)?.get(target);
7173
- if (!pending) {
7174
- continue;
7175
- }
7176
- for (const hash of hashList) {
7177
- pending.delete(hash);
7178
- }
7179
- if (pending.size === 0) {
7180
- this._repairFrontierByMode.get(mode)?.delete(target);
7181
- this._repairFrontierBypassKnownPeersByMode.get(mode)?.delete(target);
7182
- }
8238
+ if (slot.timer && slot.timerDueAt === nextDueAt) {
8239
+ return;
8240
+ }
8241
+ if (slot.timer) {
8242
+ clearTimeout(slot.timer.handle);
8243
+ this.untrackJoinWarmupTimer(target, slot.timer);
7183
8244
  }
8245
+ let trackedTimer!: JoinWarmupRetryTimer;
8246
+ const handle = setTimeout(
8247
+ () => {
8248
+ this.untrackJoinWarmupTimer(target, trackedTimer);
8249
+ if (slot.timer !== trackedTimer) {
8250
+ return;
8251
+ }
8252
+ slot.timer = undefined;
8253
+ slot.timerDueAt = undefined;
8254
+ const current = this._joinWarmupScheduledRetriesByTarget.get(target);
8255
+ if (
8256
+ current !== scheduled ||
8257
+ current.slotsByDelay.get(delayMs) !== slot
8258
+ ) {
8259
+ return;
8260
+ }
8261
+
8262
+ const dueEntries = new Map<string, RepairDispatchEntry<R>>();
8263
+ let bypassKnownPeerHints = false;
8264
+ const now = Date.now();
8265
+ while (
8266
+ slot.head < slot.cohorts.length &&
8267
+ slot.cohorts[slot.head].dueAt <= now
8268
+ ) {
8269
+ const cohort = slot.cohorts[slot.head++];
8270
+ for (const batch of cohort.batches) {
8271
+ for (const [hash, entry] of batch.entries) {
8272
+ dueEntries.set(hash, entry);
8273
+ }
8274
+ bypassKnownPeerHints ||=
8275
+ batch.bypassKnownPeerHints;
8276
+ batch.remainingAttempts -= 1;
8277
+ if (batch.remainingAttempts === 0) {
8278
+ batch.entries.clear();
8279
+ }
8280
+ }
8281
+ cohort.batches.length = 0;
8282
+ }
8283
+ if (
8284
+ dueEntries.size > 0 &&
8285
+ !this.closed &&
8286
+ this._joinWarmupGenerationByTarget.get(target) ===
8287
+ scheduled.generation
8288
+ ) {
8289
+ this.queueJoinWarmupSend(
8290
+ target,
8291
+ scheduled.generation,
8292
+ dueEntries,
8293
+ bypassKnownPeerHints,
8294
+ );
8295
+ }
8296
+ if (slot.head === slot.cohorts.length) {
8297
+ current.slotsByDelay.delete(delayMs);
8298
+ if (current.slotsByDelay.size === 0) {
8299
+ this._joinWarmupScheduledRetriesByTarget.delete(target);
8300
+ }
8301
+ return;
8302
+ }
8303
+ if (slot.head >= 1_024 && slot.head * 2 >= slot.cohorts.length) {
8304
+ slot.cohorts = slot.cohorts.slice(slot.head);
8305
+ slot.head = 0;
8306
+ }
8307
+ this.armJoinWarmupRetrySlot(target, current, delayMs, slot);
8308
+ },
8309
+ Math.max(0, nextDueAt - Date.now()),
8310
+ );
8311
+ handle.unref?.();
8312
+ trackedTimer = { handle };
8313
+ slot.timer = trackedTimer;
8314
+ slot.timerDueAt = nextDueAt;
8315
+ this.trackJoinWarmupTimer(target, trackedTimer);
7184
8316
  }
7185
8317
 
7186
8318
  private async getFullReplicaRepairCandidates(
@@ -7216,7 +8348,18 @@ export class SharedLog<
7216
8348
  return candidates;
7217
8349
  }
7218
8350
 
7219
- private removeRepairFrontierTarget(target: string) {
8351
+ private removeRepairFrontierTarget(
8352
+ target: string,
8353
+ options?: { expectedJoinWarmupGeneration?: object | null },
8354
+ ) {
8355
+ if (
8356
+ options?.expectedJoinWarmupGeneration === undefined ||
8357
+ (options.expectedJoinWarmupGeneration !== null &&
8358
+ this._joinWarmupGenerationByTarget.get(target) ===
8359
+ options.expectedJoinWarmupGeneration)
8360
+ ) {
8361
+ this.cancelJoinWarmupTarget(target);
8362
+ }
7220
8363
  for (const mode of REPAIR_DISPATCH_MODES) {
7221
8364
  this._repairFrontierByMode.get(mode)?.delete(target);
7222
8365
  this._repairFrontierActiveTargetsByMode.get(mode)?.delete(target);
@@ -7329,6 +8472,110 @@ export class SharedLog<
7329
8472
  });
7330
8473
  }
7331
8474
 
8475
+ private queueJoinWarmupSend(
8476
+ target: string,
8477
+ generation: object,
8478
+ entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
8479
+ bypassKnownPeerHints: boolean,
8480
+ ) {
8481
+ if (
8482
+ this.closed ||
8483
+ this._joinWarmupGenerationByTarget.get(target) !== generation
8484
+ ) {
8485
+ return;
8486
+ }
8487
+ let state = this._joinWarmupSendStateByTarget.get(target);
8488
+ if (!state) {
8489
+ state = {
8490
+ bypassKnownPeerHints: false,
8491
+ entries: new Map(),
8492
+ generation,
8493
+ lastCompletedAt: Number.NEGATIVE_INFINITY,
8494
+ pending: false,
8495
+ running: false,
8496
+ };
8497
+ this._joinWarmupSendStateByTarget.set(target, state);
8498
+ } else if (state.generation !== generation) {
8499
+ state.bypassKnownPeerHints = false;
8500
+ state.entries.clear();
8501
+ state.pending = false;
8502
+ }
8503
+ for (const [hash, entry] of entries) {
8504
+ state.entries.set(hash, entry);
8505
+ }
8506
+ state.bypassKnownPeerHints ||= bypassKnownPeerHints;
8507
+ state.generation = generation;
8508
+ state.pending = true;
8509
+ if (state.running) {
8510
+ return;
8511
+ }
8512
+ void this.drainJoinWarmupSends(target, state).catch((error: any) =>
8513
+ logger.error(error),
8514
+ );
8515
+ }
8516
+
8517
+ private async drainJoinWarmupSends(
8518
+ target: string,
8519
+ state: JoinWarmupSendState<R>,
8520
+ ) {
8521
+ if (state.running) {
8522
+ return;
8523
+ }
8524
+ state.running = true;
8525
+ try {
8526
+ while (state.pending) {
8527
+ state.pending = false;
8528
+ const generation = state.generation;
8529
+ const entries = new Map(state.entries);
8530
+ state.entries.clear();
8531
+ const bypassKnownPeerHints = state.bypassKnownPeerHints;
8532
+ state.bypassKnownPeerHints = false;
8533
+ const spacingMs = Math.max(
8534
+ 0,
8535
+ state.lastCompletedAt + JOIN_WARMUP_SEND_SPACING_MS - Date.now(),
8536
+ );
8537
+ await this.sleepJoinWarmupTracked(target, spacingMs);
8538
+ if (
8539
+ this.closed ||
8540
+ state.generation !== generation ||
8541
+ this._joinWarmupGenerationByTarget.get(target) !== generation
8542
+ ) {
8543
+ continue;
8544
+ }
8545
+ if (entries.size === 0) {
8546
+ continue;
8547
+ }
8548
+ this._repairMetrics["join-warmup"].simpleFallbackPasses += 1;
8549
+ try {
8550
+ await this.sendRepairEntriesWithTransport(
8551
+ target,
8552
+ entries,
8553
+ "simple",
8554
+ {
8555
+ bypassKnownPeers: bypassKnownPeerHints,
8556
+ bypassRecentKnownPeers: bypassKnownPeerHints,
8557
+ },
8558
+ );
8559
+ } catch (error: any) {
8560
+ logger.error(error);
8561
+ } finally {
8562
+ state.lastCompletedAt = Date.now();
8563
+ }
8564
+ }
8565
+ } finally {
8566
+ state.running = false;
8567
+ if (this._joinWarmupSendStateByTarget.get(target) === state) {
8568
+ if (state.pending) {
8569
+ void this.drainJoinWarmupSends(target, state).catch((error: any) =>
8570
+ logger.error(error),
8571
+ );
8572
+ } else if (!this._joinWarmupGenerationByTarget.has(target)) {
8573
+ this._joinWarmupSendStateByTarget.delete(target);
8574
+ }
8575
+ }
8576
+ }
8577
+ }
8578
+
7332
8579
  private async sendMaybeMissingEntriesNow(
7333
8580
  target: string,
7334
8581
  entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
@@ -7616,18 +8863,34 @@ export class SharedLog<
7616
8863
  const bucket = this._repairMetrics[options.mode];
7617
8864
  bucket.dispatches += 1;
7618
8865
  bucket.entries += filteredEntries.size;
8866
+ const joinWarmupGeneration =
8867
+ options.mode === "join-warmup"
8868
+ ? this.getJoinWarmupGeneration(target)
8869
+ : undefined;
8870
+ const bypassKnownPeerHints = this.shouldBypassKnownPeerHints(
8871
+ options.mode,
8872
+ options.bypassKnownPeerHints,
8873
+ );
7619
8874
 
7620
8875
  const run = (transport: RepairTransportMode) => {
7621
- if (transport === "simple") {
7622
- bucket.simpleFallbackPasses += 1;
7623
- } else {
8876
+ if (
8877
+ transport === "simple" &&
8878
+ options.mode === "join-warmup" &&
8879
+ joinWarmupGeneration
8880
+ ) {
8881
+ this.queueJoinWarmupSend(
8882
+ target,
8883
+ joinWarmupGeneration,
8884
+ filteredEntries,
8885
+ bypassKnownPeerHints,
8886
+ );
8887
+ return;
8888
+ }
8889
+ if (transport === "rateless") {
7624
8890
  bucket.ratelessFirstPasses += 1;
8891
+ } else {
8892
+ bucket.simpleFallbackPasses += 1;
7625
8893
  }
7626
- const bypassKnownPeerHints = this.shouldBypassKnownPeerHints(
7627
- options.mode,
7628
- options.bypassKnownPeerHints,
7629
- );
7630
-
7631
8894
  return Promise.resolve(
7632
8895
  this.sendRepairEntriesWithTransport(
7633
8896
  target,
@@ -7641,12 +8904,21 @@ export class SharedLog<
7641
8904
  ).catch((error: any) => logger.error(error));
7642
8905
  };
7643
8906
 
8907
+ const delayedJoinWarmupRetries: number[] = [];
7644
8908
  retrySchedule.forEach((delayMs, index) => {
7645
8909
  const transport = getRepairTransportForAttempt(options.mode, index);
7646
8910
  if (delayMs === 0) {
7647
8911
  void run(transport);
7648
8912
  return;
7649
8913
  }
8914
+ if (
8915
+ options.mode === "join-warmup" &&
8916
+ joinWarmupGeneration &&
8917
+ transport === "simple"
8918
+ ) {
8919
+ delayedJoinWarmupRetries.push(delayMs);
8920
+ return;
8921
+ }
7650
8922
  const timer = setTimeout(() => {
7651
8923
  this._repairRetryTimers.delete(timer);
7652
8924
  if (this.closed) {
@@ -7657,19 +8929,46 @@ export class SharedLog<
7657
8929
  timer.unref?.();
7658
8930
  this._repairRetryTimers.add(timer);
7659
8931
  });
8932
+ if (joinWarmupGeneration && delayedJoinWarmupRetries.length > 0) {
8933
+ this.scheduleJoinWarmupRetries(
8934
+ target,
8935
+ joinWarmupGeneration,
8936
+ delayedJoinWarmupRetries,
8937
+ filteredEntries,
8938
+ bypassKnownPeerHints,
8939
+ );
8940
+ }
7660
8941
  }
7661
8942
 
7662
8943
  private scheduleRepairSweep(options: {
7663
8944
  mode: RepairDispatchMode;
7664
8945
  peers?: Iterable<string>;
8946
+ joinWarmupGenerations?: ReadonlyMap<string, object>;
7665
8947
  }) {
7666
- this._repairSweepPendingModes.add(options.mode);
7667
8948
  const pendingPeers = this._repairSweepPendingPeersByMode.get(options.mode);
7668
8949
  if (pendingPeers) {
7669
8950
  for (const peer of options.peers ?? []) {
8951
+ if (options.mode === "join-warmup") {
8952
+ const generation =
8953
+ options.joinWarmupGenerations?.get(peer) ??
8954
+ this.getJoinWarmupGeneration(peer);
8955
+ if (
8956
+ this._joinWarmupGenerationByTarget.get(peer) !== generation
8957
+ ) {
8958
+ continue;
8959
+ }
8960
+ this._repairSweepJoinWarmupGenerationByTarget.set(
8961
+ peer,
8962
+ generation,
8963
+ );
8964
+ }
7670
8965
  pendingPeers.add(peer);
7671
8966
  }
7672
8967
  }
8968
+ if (!pendingPeers || pendingPeers.size === 0) {
8969
+ return;
8970
+ }
8971
+ this._repairSweepPendingModes.add(options.mode);
7673
8972
  if (!this._repairSweepRunning && !this.closed) {
7674
8973
  this._repairSweepRunning = true;
7675
8974
  void this.runRepairSweep();
@@ -7731,10 +9030,31 @@ export class SharedLog<
7731
9030
  const pendingPeersByMode = cloneRepairPendingPeersByMode(
7732
9031
  this._repairSweepPendingPeersByMode,
7733
9032
  );
9033
+ const pendingJoinWarmupGenerations = new Map(
9034
+ this._repairSweepJoinWarmupGenerationByTarget,
9035
+ );
7734
9036
  this._repairSweepPendingModes.clear();
7735
9037
  for (const peers of this._repairSweepPendingPeersByMode.values()) {
7736
9038
  peers.clear();
7737
9039
  }
9040
+ this._repairSweepJoinWarmupGenerationByTarget.clear();
9041
+ const pendingJoinWarmupPeers =
9042
+ pendingPeersByMode.get("join-warmup");
9043
+ const pruneStaleJoinWarmupPeers = () => {
9044
+ for (const peer of [...(pendingJoinWarmupPeers ?? [])]) {
9045
+ if (
9046
+ this._joinWarmupGenerationByTarget.get(peer) !==
9047
+ pendingJoinWarmupGenerations.get(peer)
9048
+ ) {
9049
+ pendingJoinWarmupPeers?.delete(peer);
9050
+ }
9051
+ }
9052
+ if (pendingJoinWarmupPeers?.size === 0) {
9053
+ pendingModes.delete("join-warmup");
9054
+ }
9055
+ return pendingModes.size > 0;
9056
+ };
9057
+ pruneStaleJoinWarmupPeers();
7738
9058
 
7739
9059
  if (pendingModes.size === 0) {
7740
9060
  return;
@@ -7746,7 +9066,7 @@ export class SharedLog<
7746
9066
  >();
7747
9067
  const optimisticGidPeersConsumedByMode = new Map<
7748
9068
  RepairDispatchMode,
7749
- Map<string, Map<string, number>>
9069
+ Map<string, Map<string, RepairSweepOptimisticPeerState>>
7750
9070
  >();
7751
9071
  for (const mode of pendingModes) {
7752
9072
  const modePeers = pendingPeersByMode.get(mode);
@@ -7756,20 +9076,22 @@ export class SharedLog<
7756
9076
  const optimisticGidPeers = new Map<string, Set<string>>();
7757
9077
  const optimisticGidPeersConsumed = new Map<
7758
9078
  string,
7759
- Map<string, number>
9079
+ Map<string, RepairSweepOptimisticPeerState>
7760
9080
  >();
7761
9081
  for (const [gid, peerCounts] of this
7762
9082
  ._repairSweepOptimisticGidPeersPending) {
7763
9083
  let matchedPeers: Set<string> | undefined;
7764
- let matchedCounts: Map<string, number> | undefined;
7765
- for (const [peer, count] of peerCounts) {
9084
+ let matchedCounts:
9085
+ | Map<string, RepairSweepOptimisticPeerState>
9086
+ | undefined;
9087
+ for (const [peer, state] of peerCounts) {
7766
9088
  if (!modePeers.has(peer)) {
7767
9089
  continue;
7768
9090
  }
7769
9091
  matchedPeers ||= new Set();
7770
9092
  matchedCounts ||= new Map();
7771
9093
  matchedPeers.add(peer);
7772
- matchedCounts.set(peer, count);
9094
+ matchedCounts.set(peer, { ...state });
7773
9095
  }
7774
9096
  if (matchedPeers && matchedCounts) {
7775
9097
  optimisticGidPeers.set(gid, matchedPeers);
@@ -7799,6 +9121,7 @@ export class SharedLog<
7799
9121
  await this.getFullReplicaRepairCandidates(pendingRepairPeers, {
7800
9122
  includeSubscribers: false,
7801
9123
  });
9124
+ pruneStaleJoinWarmupPeers();
7802
9125
  const fullReplicaRepairCandidateCount = Math.max(
7803
9126
  1,
7804
9127
  fullReplicaRepairCandidates.size,
@@ -7816,6 +9139,18 @@ export class SharedLog<
7816
9139
  if (!entries || entries.size === 0) {
7817
9140
  return;
7818
9141
  }
9142
+ if (
9143
+ mode === "join-warmup" &&
9144
+ this._joinWarmupGenerationByTarget.get(target) !==
9145
+ pendingJoinWarmupGenerations.get(target)
9146
+ ) {
9147
+ targets?.delete(target);
9148
+ pendingJoinWarmupPeers?.delete(target);
9149
+ if (pendingJoinWarmupPeers?.size === 0) {
9150
+ pendingModes.delete("join-warmup");
9151
+ }
9152
+ return;
9153
+ }
7819
9154
  this.dispatchMaybeMissingEntries(target, entries, {
7820
9155
  bypassRecentDedupe: true,
7821
9156
  bypassKnownPeerHints:
@@ -7832,6 +9167,17 @@ export class SharedLog<
7832
9167
  target: string,
7833
9168
  entry: RepairDispatchEntry<any>,
7834
9169
  ) => {
9170
+ if (
9171
+ mode === "join-warmup" &&
9172
+ this._joinWarmupGenerationByTarget.get(target) !==
9173
+ pendingJoinWarmupGenerations.get(target)
9174
+ ) {
9175
+ pendingJoinWarmupPeers?.delete(target);
9176
+ if (pendingJoinWarmupPeers?.size === 0) {
9177
+ pendingModes.delete("join-warmup");
9178
+ }
9179
+ return;
9180
+ }
7835
9181
  const sweepTargets = nextFrontierByMode.get(mode);
7836
9182
  if (sweepTargets) {
7837
9183
  let sweepSet = sweepTargets.get(target);
@@ -7862,16 +9208,17 @@ export class SharedLog<
7862
9208
  residentEntriesByHash &&
7863
9209
  !this.hasCustomFindLeaders()
7864
9210
  ) {
7865
- const repairDispatchPlan = await this.planResidentRepairDispatchBatch(
7866
- {
9211
+ const repairDispatchPlan = pruneStaleJoinWarmupPeers()
9212
+ ? await this.planResidentRepairDispatchBatch({
7867
9213
  pendingModes,
7868
9214
  pendingPeersByMode,
7869
9215
  optimisticGidPeersByMode,
7870
9216
  fullReplicaRepairCandidates,
7871
9217
  fullReplicaRepairCandidateCount,
7872
9218
  selfHash: this.node.identity.publicKey.hashcode(),
7873
- },
7874
- );
9219
+ })
9220
+ : new Map();
9221
+ pruneStaleJoinWarmupPeers();
7875
9222
  for (const [mode, targets] of repairDispatchPlan) {
7876
9223
  for (const [target, hashes] of targets) {
7877
9224
  for (const hash of hashes) {
@@ -7882,13 +9229,20 @@ export class SharedLog<
7882
9229
  }
7883
9230
  }
7884
9231
  }
7885
- } else {
9232
+ } else if (pruneStaleJoinWarmupPeers()) {
7886
9233
  const iterator = this.entryCoordinatesIndex.iterate({});
7887
9234
  try {
7888
- while (!this.closed && !iterator.done()) {
9235
+ while (
9236
+ !this.closed &&
9237
+ !iterator.done() &&
9238
+ pruneStaleJoinWarmupPeers()
9239
+ ) {
7889
9240
  const entries = await iterator.next(
7890
9241
  REPAIR_SWEEP_ENTRY_BATCH_SIZE,
7891
9242
  );
9243
+ if (!pruneStaleJoinWarmupPeers()) {
9244
+ break;
9245
+ }
7892
9246
  const entryReplicatedBatch = entries.map((entry) => entry.value);
7893
9247
  const requestedReplicasBatch = entryReplicatedBatch.map((entry) =>
7894
9248
  decodeReplicas(entry).getValue(this),
@@ -7903,6 +9257,9 @@ export class SharedLog<
7903
9257
  fullReplicaRepairCandidateCount,
7904
9258
  selfHash: this.node.identity.publicKey.hashcode(),
7905
9259
  });
9260
+ if (!pruneStaleJoinWarmupPeers()) {
9261
+ break;
9262
+ }
7906
9263
  const entriesByHash = new Map(
7907
9264
  entryReplicatedBatch.map((entry) => [entry.hash, entry]),
7908
9265
  );
@@ -7932,13 +9289,28 @@ export class SharedLog<
7932
9289
  if (!pendingPeerCounts) {
7933
9290
  continue;
7934
9291
  }
7935
- for (const [peer, count] of peerCounts) {
7936
- const current = pendingPeerCounts.get(peer) || 0;
7937
- const next = current - count;
9292
+ for (const [peer, consumed] of peerCounts) {
9293
+ const current = pendingPeerCounts.get(peer);
9294
+ if (
9295
+ !current ||
9296
+ current.generation !== consumed.generation
9297
+ ) {
9298
+ continue;
9299
+ }
9300
+ const next = current.count - consumed.count;
7938
9301
  if (next > 0) {
7939
- pendingPeerCounts.set(peer, next);
9302
+ pendingPeerCounts.set(peer, {
9303
+ count: next,
9304
+ generation: current.generation,
9305
+ });
7940
9306
  } else {
7941
9307
  pendingPeerCounts.delete(peer);
9308
+ const gids =
9309
+ this._repairSweepOptimisticGidsByPeer.get(peer);
9310
+ gids?.delete(gid);
9311
+ if (gids?.size === 0) {
9312
+ this._repairSweepOptimisticGidsByPeer.delete(peer);
9313
+ }
7942
9314
  }
7943
9315
  }
7944
9316
  if (pendingPeerCounts.size === 0) {
@@ -8513,10 +9885,22 @@ export class SharedLog<
8513
9885
  return;
8514
9886
  }
8515
9887
 
8516
- void Promise.allSettled(this.prune(toPrune));
9888
+ const pruneTasks = this.prune(toPrune);
9889
+ const confirmationTasks: Promise<void>[] = [];
8517
9890
  for (const hash of responseStillApplies) {
8518
- void this._checkedPrune.getPendingDelete(hash)?.resolve(publicKeyHash);
9891
+ const pendingDelete = this._checkedPrune.getPendingDelete(hash);
9892
+ if (pendingDelete) {
9893
+ confirmationTasks.push(
9894
+ Promise.resolve(pendingDelete.resolve(publicKeyHash)),
9895
+ );
9896
+ }
8519
9897
  }
9898
+ // The restarted prune session owns its own bounded timeout and revalidates
9899
+ // before deletion. Only the peer-derived confirmations must remain inside
9900
+ // this receive lease; waiting for the whole prune session could stall close
9901
+ // or disconnect until its background timeout.
9902
+ void Promise.allSettled(pruneTasks);
9903
+ await Promise.allSettled(confirmationTasks);
8520
9904
  }
8521
9905
 
8522
9906
  async append(
@@ -12111,15 +13495,27 @@ export class SharedLog<
12111
13495
  this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 2e4;
12112
13496
  this._checkedPrune = new CheckedPruneCoordinator<T, R>();
12113
13497
  this._pendingIHave = new Map();
13498
+ this._pendingIHaveCallbacks = new Set();
12114
13499
  this.latestReplicationInfoMessage = new Map();
12115
13500
  this._replicationInfoBlockedPeers = new Set();
12116
13501
  this._replicationInfoRequestByPeer = new Map();
13502
+ // Terminal close/drop drains the previous lifecycle before another open can
13503
+ // install fresh lanes and opaque per-subscription ownership tokens.
12117
13504
  this._replicationInfoApplyQueueByPeer = new Map();
13505
+ this._replicationInfoReceiveEpochByPeer = new Map();
13506
+ this._subscriptionEpochByPeer = new Map();
13507
+ this._pendingReplicatorLeaveByPeer = new Set();
13508
+ this._activeReceiveHandlersByPeer = new Map();
13509
+ this._receiveHandlerDrainByPeer = new Map();
13510
+ this._receiveCleanupGateByPeer = new Map();
13511
+ this._subscriptionOpeningEpochByPeer = new Map();
13512
+ this._openingSyncCapabilitiesByPeer = new Map();
12118
13513
  this._repairRetryTimers = new Set();
12119
13514
  this._recentRepairDispatch = new Map();
12120
13515
  this._repairSweepRunning = false;
12121
13516
  this._repairSweepPendingModes = new Set();
12122
13517
  this._repairSweepPendingPeersByMode = createRepairPendingPeersByMode();
13518
+ this._repairSweepJoinWarmupGenerationByTarget = new Map();
12123
13519
  this._repairFrontierByMode = createRepairFrontierByMode() as Map<
12124
13520
  RepairDispatchMode,
12125
13521
  Map<string, Map<string, RepairDispatchEntry<R>>>
@@ -12127,7 +13523,12 @@ export class SharedLog<
12127
13523
  this._repairFrontierActiveTargetsByMode = createRepairActiveTargetsByMode();
12128
13524
  this._repairFrontierBypassKnownPeersByMode =
12129
13525
  createRepairFrontierBypassKnownPeersByMode();
13526
+ this._joinWarmupGenerationByTarget = new Map();
13527
+ this._joinWarmupSendStateByTarget ??= new Map();
13528
+ this._joinWarmupRetryTimersByTarget = new Map();
13529
+ this._joinWarmupScheduledRetriesByTarget = new Map();
12130
13530
  this._repairSweepOptimisticGidPeersPending = new Map();
13531
+ this._repairSweepOptimisticGidsByPeer = new Map();
12131
13532
  this._entryKnownPeers = new Map();
12132
13533
  this._entryKnownPeerObservedAt = new Map();
12133
13534
  this._joinAuthoritativeRepairTimersByDelay = new Map();
@@ -12216,6 +13617,7 @@ export class SharedLog<
12216
13617
 
12217
13618
  this._closeController = new AbortController();
12218
13619
  this.setupReplicationAnnouncementRetryFunction();
13620
+ this.setupReplicationAnnouncementRepairFunction();
12219
13621
  this._closeController.signal.addEventListener("abort", () => {
12220
13622
  for (const [_peer, state] of this._replicationInfoRequestByPeer) {
12221
13623
  if (state.timer) clearTimeout(state.timer);
@@ -12322,7 +13724,9 @@ export class SharedLog<
12322
13724
  this.rpc.send(new BlocksMessage(message), options),
12323
13725
  waitFor: this.rpc.waitFor.bind(this.rpc),
12324
13726
  publicKey: this.node.identity.publicKey,
12325
- eagerBlocks: options?.eagerBlocks ?? true,
13727
+ // Unsolicited block retention is opt-in. Explicit `true` retains the
13728
+ // compatible eager path with bounded validation and storage budgets.
13729
+ eagerBlocks: options?.eagerBlocks ?? false,
12326
13730
  resolveProviders: async (cid, opts) => {
12327
13731
  // 1) tracker-backed provider directory (best-effort, bounded)
12328
13732
  try {
@@ -13280,11 +14684,17 @@ export class SharedLog<
13280
14684
  async afterOpen(): Promise<void> {
13281
14685
  await super.afterOpen();
13282
14686
  const existingSubscribersPromise = this._getTopicSubscribers(this.topic);
14687
+ const replicationLifecycleController =
14688
+ this._replicationLifecycleController;
13283
14689
 
13284
14690
  // We do this here, because these calls requires this.closed == false
13285
14691
  void this.pruneOfflineReplicators()
13286
14692
  .then(() => {
13287
- this._replicatorsReconciled = true;
14693
+ if (
14694
+ this.isReplicationLifecycleActive(replicationLifecycleController)
14695
+ ) {
14696
+ this._replicatorsReconciled = true;
14697
+ }
13288
14698
  })
13289
14699
  .catch((error) => {
13290
14700
  if (isNotStartedError(error as Error)) {
@@ -13318,13 +14728,25 @@ export class SharedLog<
13318
14728
  async pruneOfflineReplicators() {
13319
14729
  // Go through all segments and wait for replicators to become reachable;
13320
14730
  // otherwise prune them away from the local membership view.
14731
+ const replicationLifecycleController =
14732
+ this._replicationLifecycleController;
13321
14733
  try {
14734
+ if (
14735
+ !replicationLifecycleController ||
14736
+ !this.isReplicationLifecycleActive(replicationLifecycleController)
14737
+ ) {
14738
+ return;
14739
+ }
13322
14740
  const promises: Promise<any>[] = [];
13323
14741
  const iterator = this.replicationIndex.iterate();
13324
14742
  const checkedIsAlive = new Set<string>();
13325
14743
 
13326
14744
  while (!iterator.done()) {
13327
- for (const segment of await iterator.next(1000)) {
14745
+ const segments = await iterator.next(1000);
14746
+ if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
14747
+ return;
14748
+ }
14749
+ for (const segment of segments) {
13328
14750
  if (
13329
14751
  checkedIsAlive.has(segment.value.hash) ||
13330
14752
  this.node.identity.publicKey.hashcode() === segment.value.hash
@@ -13334,50 +14756,97 @@ export class SharedLog<
13334
14756
  }
13335
14757
 
13336
14758
  checkedIsAlive.add(segment.value.hash);
14759
+ const peerHash = segment.value.hash;
14760
+ const subscriptionEpoch = this.getSubscriptionEpoch(peerHash);
13337
14761
 
13338
14762
  promises.push(
13339
- waitForSubscribers(this.node, segment.value.hash, this.rpc.topic, {
14763
+ waitForSubscribers(this.node, peerHash, this.rpc.topic, {
13340
14764
  timeout: this.waitForReplicatorTimeout,
13341
14765
  signal: this._closeController.signal,
13342
14766
  })
13343
14767
  .then(async () => {
13344
- const key = await this._resolvePublicKeyFromHash(
13345
- segment.value.hash,
13346
- );
14768
+ if (
14769
+ !this.isReplicationLifecycleActive(
14770
+ replicationLifecycleController,
14771
+ )
14772
+ ) {
14773
+ return;
14774
+ }
14775
+ const key = await this._resolvePublicKeyFromHash(peerHash);
13347
14776
  if (!key) {
13348
14777
  throw new Error(
13349
14778
  "Failed to resolve public key from hash: " +
13350
- segment.value.hash,
14779
+ peerHash,
13351
14780
  );
13352
14781
  }
13353
14782
 
13354
14783
  const keyHash = key.hashcode();
13355
- this.uniqueReplicators.add(keyHash);
13356
-
13357
- if (!this._replicatorJoinEmitted.has(keyHash)) {
13358
- this._replicatorJoinEmitted.add(keyHash);
13359
- this.events.dispatchEvent(
13360
- new CustomEvent<ReplicatorJoinEvent>("replicator:join", {
13361
- detail: { publicKey: key },
13362
- }),
13363
- );
13364
- this.events.dispatchEvent(
13365
- new CustomEvent<ReplicationChangeEvent>(
13366
- "replication:change",
13367
- {
13368
- detail: { publicKey: key },
13369
- },
13370
- ),
13371
- );
14784
+ if (keyHash !== peerHash) {
14785
+ return;
13372
14786
  }
14787
+ return this.withReplicationInfoApplyQueue(
14788
+ keyHash,
14789
+ async () => {
14790
+ // A successful reachability check may legitimately span a
14791
+ // subscribe event during startup. The current lane's blocked
14792
+ // state plus an extant index row are the authoritative guard;
14793
+ // only the destructive catch path remains tied to the old token.
14794
+ if (
14795
+ !this.isReplicationLifecycleActive(
14796
+ replicationLifecycleController,
14797
+ ) ||
14798
+ this.closed ||
14799
+ this._replicationInfoBlockedPeers.has(keyHash)
14800
+ ) {
14801
+ return;
14802
+ }
14803
+ const hasReplicationRange =
14804
+ (await this.replicationIndex.count({
14805
+ query: { hash: keyHash },
14806
+ })) > 0;
14807
+ if (
14808
+ !hasReplicationRange ||
14809
+ !this.isReplicationLifecycleActive(
14810
+ replicationLifecycleController,
14811
+ ) ||
14812
+ this._replicationInfoBlockedPeers.has(keyHash)
14813
+ ) {
14814
+ return;
14815
+ }
14816
+ this.uniqueReplicators.add(keyHash);
14817
+
14818
+ if (!this._replicatorJoinEmitted.has(keyHash)) {
14819
+ this._replicatorJoinEmitted.add(keyHash);
14820
+ this.events.dispatchEvent(
14821
+ new CustomEvent<ReplicatorJoinEvent>(
14822
+ "replicator:join",
14823
+ { detail: { publicKey: key } },
14824
+ ),
14825
+ );
14826
+ this.events.dispatchEvent(
14827
+ new CustomEvent<ReplicationChangeEvent>(
14828
+ "replication:change",
14829
+ { detail: { publicKey: key } },
14830
+ ),
14831
+ );
14832
+ }
14833
+ },
14834
+ );
13373
14835
  })
13374
14836
  .catch(async (error) => {
13375
- if (isNotStartedError(error as Error)) {
14837
+ if (
14838
+ isNotStartedError(error as Error) ||
14839
+ !this.isReplicationLifecycleActive(
14840
+ replicationLifecycleController,
14841
+ )
14842
+ ) {
13376
14843
  return;
13377
14844
  }
13378
14845
 
13379
- return this.removeReplicator(segment.value.hash, {
14846
+ return this.removeReplicator(peerHash, {
13380
14847
  noEvent: true,
14848
+ replicationLifecycleController,
14849
+ subscriptionEpoch,
13381
14850
  });
13382
14851
  }),
13383
14852
  );
@@ -13386,7 +14855,12 @@ export class SharedLog<
13386
14855
 
13387
14856
  return Promise.all(promises);
13388
14857
  } catch (error) {
13389
- if (isNotStartedError(error as Error)) {
14858
+ if (
14859
+ isNotStartedError(error as Error) ||
14860
+ !this.isReplicationLifecycleActive(
14861
+ replicationLifecycleController,
14862
+ )
14863
+ ) {
13390
14864
  return;
13391
14865
  }
13392
14866
  throw error;
@@ -13458,11 +14932,28 @@ export class SharedLog<
13458
14932
  this._replicatorLivenessFailures.delete(peerHash);
13459
14933
  this._replicatorLastActivityAt.delete(peerHash);
13460
14934
  this._peerSyncCapabilities.delete(peerHash);
14935
+ this.cleanupPendingIHavePeer(peerHash);
13461
14936
  this._checkedPrune.cleanupPeer(peerHash);
13462
14937
  }
13463
14938
 
14939
+ private cleanupPendingIHavePeer(peerHash: string) {
14940
+ for (const [hash, pending] of this._pendingIHave) {
14941
+ pending.requesting.delete(peerHash);
14942
+ if (pending.requesting.size === 0) {
14943
+ pending.clear();
14944
+ this._pendingIHave.delete(hash);
14945
+ }
14946
+ }
14947
+ }
14948
+
13464
14949
  private markReplicatorActivity(peerHash: string, now = Date.now()) {
13465
14950
  this._replicatorLastActivityAt.set(peerHash, now);
14951
+ // Any recent authenticated activity is positive liveness evidence. Reset the
14952
+ // consecutive miss streak immediately, including while an eviction is
14953
+ // waiting in the per-peer mutation lane.
14954
+ if (Date.now() - now < REPLICATOR_LIVENESS_IDLE_THRESHOLD_MS) {
14955
+ this._replicatorLivenessFailures.delete(peerHash);
14956
+ }
13466
14957
  }
13467
14958
 
13468
14959
  private hasRecentReplicatorActivity(peerHash: string, now = Date.now()) {
@@ -13477,39 +14968,65 @@ export class SharedLog<
13477
14968
  return false;
13478
14969
  }
13479
14970
 
14971
+ private advanceReplicationInfoRecoveryEpoch(peerHash: string) {
14972
+ // Handlers admitted before a successful peer removal must not restore state
14973
+ // when they eventually reach the apply lane. Reset the sender's
14974
+ // ordering watermark with the local epoch so a later arrival can be
14975
+ // accepted without comparing its clock to this receiver's clock.
14976
+ this.advanceReplicationInfoReceiveEpoch(peerHash);
14977
+ this.latestReplicationInfoMessage.delete(peerHash);
14978
+ }
14979
+
13480
14980
  private async evictReplicatorFromLiveness(
13481
14981
  peerHash: string,
13482
14982
  publicKey: PublicSignKey,
14983
+ replicationLifecycleController: AbortController,
14984
+ subscriptionEpoch: object | null,
14985
+ observedActivityAt: number | undefined,
13483
14986
  ) {
13484
- const wasReplicator = this.uniqueReplicators.has(peerHash);
13485
- const watermark = BigInt(+new Date());
13486
- const previousWatermark = this.latestReplicationInfoMessage.get(peerHash);
13487
- if (!previousWatermark || previousWatermark < watermark) {
13488
- this.latestReplicationInfoMessage.set(peerHash, watermark);
13489
- }
14987
+ try {
14988
+ await this.removeReplicator(publicKey, {
14989
+ noEvent: true,
14990
+ replicationLifecycleController,
14991
+ shouldRemove: () =>
14992
+ this._replicatorLastActivityAt.get(peerHash) === observedActivityAt,
14993
+ subscriptionEpoch,
14994
+ onRemoved: ({ wasReplicator }) => {
14995
+ if (wasReplicator) {
14996
+ this._pendingReplicatorLeaveByPeer.add(peerHash);
14997
+ }
14998
+ // A newer subscription/lifecycle may have started while the admitted
14999
+ // removal was completing. Its reconnect barrier owns all later effects.
15000
+ if (
15001
+ !this.isReplicationLifecycleActive(
15002
+ replicationLifecycleController,
15003
+ ) ||
15004
+ !this.isCurrentSubscriptionEpoch(peerHash, subscriptionEpoch)
15005
+ ) {
15006
+ return;
15007
+ }
15008
+ if (this._pendingReplicatorLeaveByPeer.delete(peerHash)) {
15009
+ this.events.dispatchEvent(
15010
+ new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
15011
+ detail: { publicKey },
15012
+ }),
15013
+ );
15014
+ }
13490
15015
 
13491
- try {
13492
- await this.removeReplicator(publicKey, { noEvent: true });
15016
+ if (!this._replicationInfoBlockedPeers.has(peerHash)) {
15017
+ this.scheduleReplicationInfoRequests(
15018
+ publicKey,
15019
+ replicationLifecycleController,
15020
+ );
15021
+ }
15022
+ this._replicatorLivenessTargetsSize = -1;
15023
+ },
15024
+ });
13493
15025
  } catch (error) {
13494
15026
  if (!isNotStartedError(error as Error)) {
13495
15027
  throw error;
13496
15028
  }
13497
15029
  }
13498
-
13499
- this.cleanupPeerDisconnectTracking(peerHash);
13500
-
13501
- if (wasReplicator) {
13502
- this.events.dispatchEvent(
13503
- new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
13504
- detail: { publicKey },
13505
- }),
13506
- );
13507
- }
13508
-
13509
- if (!this._replicationInfoBlockedPeers.has(peerHash)) {
13510
- this.scheduleReplicationInfoRequests(publicKey);
13511
- }
13512
- this._replicatorLivenessTargetsSize = -1;
13513
15030
  }
13514
15031
 
13515
15032
  private async resolveCandidatePeersForHash(
@@ -13589,7 +15106,13 @@ export class SharedLog<
13589
15106
  }
13590
15107
 
13591
15108
  private async runReplicatorLivenessSweep() {
13592
- if (this.closed || this._closeController.signal.aborted) {
15109
+ const replicationLifecycleController =
15110
+ this._replicationLifecycleController;
15111
+ if (
15112
+ this.closed ||
15113
+ this._closeController.signal.aborted ||
15114
+ !this.isReplicationLifecycleActive(replicationLifecycleController)
15115
+ ) {
13593
15116
  return;
13594
15117
  }
13595
15118
  if (this._replicatorLivenessSweepRunning) {
@@ -13615,14 +15138,30 @@ export class SharedLog<
13615
15138
  logger.error((error as any)?.toString?.() ?? String(error));
13616
15139
  }
13617
15140
  } finally {
13618
- this._replicatorLivenessSweepRunning = false;
15141
+ if (
15142
+ this._replicationLifecycleController ===
15143
+ replicationLifecycleController
15144
+ ) {
15145
+ this._replicatorLivenessSweepRunning = false;
15146
+ }
13619
15147
  }
13620
15148
  }
13621
15149
 
13622
15150
  private async probeReplicatorLiveness(peerHash: string) {
13623
- if (this.closed || this._closeController.signal.aborted) {
15151
+ const replicationLifecycleController =
15152
+ this._replicationLifecycleController;
15153
+ if (
15154
+ this.closed ||
15155
+ this._closeController.signal.aborted ||
15156
+ !replicationLifecycleController ||
15157
+ !this.isReplicationLifecycleActive(replicationLifecycleController)
15158
+ ) {
13624
15159
  return;
13625
15160
  }
15161
+ const subscriptionEpoch = this.getSubscriptionEpoch(peerHash);
15162
+ const ownsProbe = () =>
15163
+ this.isReplicationLifecycleActive(replicationLifecycleController) &&
15164
+ this.isCurrentSubscriptionEpoch(peerHash, subscriptionEpoch);
13626
15165
  if (!this.uniqueReplicators.has(peerHash)) {
13627
15166
  this._replicatorLivenessFailures.delete(peerHash);
13628
15167
  return;
@@ -13630,18 +15169,36 @@ export class SharedLog<
13630
15169
  if (this.hasRecentReplicatorActivity(peerHash)) {
13631
15170
  return;
13632
15171
  }
15172
+ const observedActivityAt = this._replicatorLastActivityAt.get(peerHash);
13633
15173
 
13634
15174
  const publicKey = await this._resolvePublicKeyFromHash(peerHash);
15175
+ if (!ownsProbe()) {
15176
+ return;
15177
+ }
15178
+ if (this.hasRecentReplicatorActivity(peerHash)) {
15179
+ return;
15180
+ }
13635
15181
  if (!publicKey) {
13636
15182
  try {
13637
- await this.removeReplicator(peerHash, { noEvent: true });
15183
+ await this.removeReplicator(peerHash, {
15184
+ noEvent: true,
15185
+ replicationLifecycleController,
15186
+ shouldRemove: () =>
15187
+ this._replicatorLastActivityAt.get(peerHash) ===
15188
+ observedActivityAt,
15189
+ subscriptionEpoch,
15190
+ onRemoved: () => {
15191
+ if (!ownsProbe()) {
15192
+ return;
15193
+ }
15194
+ this._replicatorLivenessTargetsSize = -1;
15195
+ },
15196
+ });
13638
15197
  } catch (error) {
13639
15198
  if (!isNotStartedError(error as Error)) {
13640
15199
  throw error;
13641
15200
  }
13642
15201
  }
13643
- this.cleanupPeerDisconnectTracking(peerHash);
13644
- this._replicatorLivenessTargetsSize = -1;
13645
15202
  return;
13646
15203
  }
13647
15204
 
@@ -13653,6 +15210,9 @@ export class SharedLog<
13653
15210
  priority: ACK_CONTROL_PRIORITY,
13654
15211
  responsePriority: ACK_CONTROL_PRIORITY,
13655
15212
  });
15213
+ if (!ownsProbe()) {
15214
+ return;
15215
+ }
13656
15216
  this.markReplicatorActivity(peerHash);
13657
15217
  this._replicatorLivenessFailures.delete(peerHash);
13658
15218
  return;
@@ -13661,29 +15221,53 @@ export class SharedLog<
13661
15221
  return;
13662
15222
  }
13663
15223
  }
15224
+ if (!ownsProbe()) {
15225
+ return;
15226
+ }
15227
+ if (this.hasRecentReplicatorActivity(peerHash)) {
15228
+ return;
15229
+ }
13664
15230
 
13665
15231
  // Relay-backed prod paths can keep a peer subscribed/reachable even if an
13666
15232
  // ACKed liveness ping gets delayed or dropped under load. Treat observed
13667
15233
  // topic presence as a positive liveness signal before evicting the peer.
13668
15234
  if (await this.confirmReplicatorSubscriberPresence(peerHash)) {
15235
+ if (!ownsProbe()) {
15236
+ return;
15237
+ }
13669
15238
  this.markReplicatorActivity(peerHash);
13670
15239
  this._replicatorLivenessFailures.delete(peerHash);
13671
15240
  return;
13672
15241
  }
15242
+ if (!ownsProbe()) {
15243
+ return;
15244
+ }
15245
+ if (this.hasRecentReplicatorActivity(peerHash)) {
15246
+ return;
15247
+ }
13673
15248
 
13674
15249
  const failures = (this._replicatorLivenessFailures.get(peerHash) ?? 0) + 1;
13675
15250
  this._replicatorLivenessFailures.set(peerHash, failures);
13676
- this.scheduleReplicationInfoRequests(publicKey);
15251
+ this.scheduleReplicationInfoRequests(
15252
+ publicKey,
15253
+ replicationLifecycleController,
15254
+ );
13677
15255
 
13678
15256
  if (failures < REPLICATOR_LIVENESS_PROBE_FAILURES_TO_EVICT) {
13679
15257
  return;
13680
15258
  }
13681
- if (!this.uniqueReplicators.has(peerHash)) {
15259
+ if (!ownsProbe() || !this.uniqueReplicators.has(peerHash)) {
13682
15260
  this._replicatorLivenessFailures.delete(peerHash);
13683
15261
  return;
13684
15262
  }
13685
15263
 
13686
- await this.evictReplicatorFromLiveness(peerHash, publicKey);
15264
+ await this.evictReplicatorFromLiveness(
15265
+ peerHash,
15266
+ publicKey,
15267
+ replicationLifecycleController,
15268
+ subscriptionEpoch,
15269
+ observedActivityAt,
15270
+ );
13687
15271
  }
13688
15272
 
13689
15273
  private async confirmReplicatorSubscriberPresence(peerHash: string) {
@@ -13722,6 +15306,26 @@ export class SharedLog<
13722
15306
  /* ((await this.log.entryIndex?.getMemoryUsage()) || 0) */ // + (await this.log.blocks.size())
13723
15307
  }
13724
15308
 
15309
+ /** Return a detached snapshot of effective shared-log runtime settings. */
15310
+ getRuntimeSnapshot(): SharedLogRuntimeSnapshot {
15311
+ const nativeGraph = this.log.entryIndex.properties.nativeGraph;
15312
+ const active = nativeGraph?.graph != null;
15313
+ return Object.freeze({
15314
+ nativeGraph: Object.freeze({
15315
+ active,
15316
+ useHeads: active && nativeGraph?.useHeads === true,
15317
+ }),
15318
+ });
15319
+ }
15320
+
15321
+ /**
15322
+ * Return a detached snapshot of the optional eager-response cache.
15323
+ * Undefined means eager response retention is disabled for this log.
15324
+ */
15325
+ getEagerBlockCacheTelemetry() {
15326
+ return this.remoteBlocks?.getEagerBlockCacheTelemetry();
15327
+ }
15328
+
13725
15329
  private clampReplicas(value: number) {
13726
15330
  const lower = this.replicas.min?.getValue(this) || 1;
13727
15331
  const higher = this.replicas.max?.getValue(this) ?? Number.MAX_SAFE_INTEGER;
@@ -14477,6 +16081,7 @@ export class SharedLog<
14477
16081
  ),
14478
16082
  );
14479
16083
  captureSync(() => {
16084
+ this.cancelAllJoinWarmupTargets();
14480
16085
  for (const timer of this._repairRetryTimers ?? []) clearTimeout(timer);
14481
16086
  this._repairRetryTimers?.clear();
14482
16087
  this._recentRepairDispatch?.clear();
@@ -14484,7 +16089,9 @@ export class SharedLog<
14484
16089
  this._repairSweepPendingModes?.clear();
14485
16090
  for (const peers of this._repairSweepPendingPeersByMode?.values() ?? [])
14486
16091
  peers.clear();
16092
+ this._repairSweepJoinWarmupGenerationByTarget?.clear();
14487
16093
  this._repairSweepOptimisticGidPeersPending?.clear();
16094
+ this._repairSweepOptimisticGidsByPeer?.clear();
14488
16095
  this._entryKnownPeers?.clear();
14489
16096
  this._entryKnownPeerObservedAt?.clear();
14490
16097
  this._nativeSharedLogState?.clearEntryKnownPeers();
@@ -14521,7 +16128,14 @@ export class SharedLog<
14521
16128
  }
14522
16129
  captureSync(() => {
14523
16130
  this._pendingIHave?.clear();
16131
+ this._pendingIHaveCallbacks?.clear();
14524
16132
  this.latestReplicationInfoMessage?.clear();
16133
+ this._replicationInfoReceiveEpochByPeer?.clear();
16134
+ this._activeReceiveHandlersByPeer?.clear();
16135
+ this._receiveHandlerDrainByPeer?.clear();
16136
+ this._receiveCleanupGateByPeer?.clear();
16137
+ this._subscriptionOpeningEpochByPeer?.clear();
16138
+ this._openingSyncCapabilitiesByPeer?.clear();
14525
16139
  this._gidPeersHistory?.clear();
14526
16140
  this._peerSyncCapabilities?.clear();
14527
16141
  this._liveRawGossipBatches?.clear();
@@ -14594,7 +16208,14 @@ export class SharedLog<
14594
16208
  // SharedLog-specific await or observable teardown can admit a new owner.
14595
16209
  this.preventParentAttachments();
14596
16210
  this.stopSubscriptionChangeCallbackAdmission();
16211
+ this.cancelAllJoinWarmupTargets();
14597
16212
  await this.drainSubscriptionChangeCallbacks();
16213
+ // An already-admitted subscription callback can create a fresh warmup
16214
+ // generation while the first cancellation is draining.
16215
+ this.cancelAllJoinWarmupTargets();
16216
+ await this.drainReceiveHandlers();
16217
+ await this.drainReplicationInfoApplyQueues();
16218
+ await this.drainPendingIHaveCallbacks();
14598
16219
  this.ensureNativeDurabilityRuntimeState();
14599
16220
  this.cancelCurrentReplicationStateAnnouncementRetry();
14600
16221
  // Best-effort: announce that we are going offline before tearing down
@@ -14715,7 +16336,14 @@ export class SharedLog<
14715
16336
  // the terminal fence only after that precondition succeeds.
14716
16337
  this.preventParentAttachments();
14717
16338
  this.stopSubscriptionChangeCallbackAdmission();
16339
+ this.cancelAllJoinWarmupTargets();
14718
16340
  await this.drainSubscriptionChangeCallbacks();
16341
+ // An already-admitted subscription callback can create a fresh warmup
16342
+ // generation while the first cancellation is draining.
16343
+ this.cancelAllJoinWarmupTargets();
16344
+ await this.drainReceiveHandlers();
16345
+ await this.drainReplicationInfoApplyQueues();
16346
+ await this.drainPendingIHaveCallbacks();
14719
16347
  this.cancelCurrentReplicationStateAnnouncementRetry();
14720
16348
  // Best-effort: announce that we are going offline before tearing down
14721
16349
  // RPC/subscription state (same reasoning as in `close()`).
@@ -14932,13 +16560,45 @@ export class SharedLog<
14932
16560
  const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
14933
16561
  ? msg
14934
16562
  : undefined;
16563
+ let releasePeerReceiveLease: (() => void) | undefined;
14935
16564
  try {
14936
16565
  this.throwIfNativeDurableCommitFailed();
14937
16566
  if (!context.from) {
14938
16567
  throw new Error("Missing from in update role message");
14939
16568
  }
16569
+ // Snapshot receive ownership before any async handler gets a chance to
16570
+ // yield. Replication-info messages reach their branch only after the
16571
+ // synchronizer declines them, and a U/S transition can happen meanwhile.
16572
+ const receiveFromHash = context.from.hashcode();
16573
+ const receiveReplicationLifecycleController =
16574
+ this._replicationLifecycleController;
16575
+ const receiveSubscriptionEpoch =
16576
+ this.getSubscriptionEpoch(receiveFromHash);
16577
+ const isOpeningSubscriptionReceive =
16578
+ this._subscriptionOpeningEpochByPeer.get(receiveFromHash) ===
16579
+ receiveSubscriptionEpoch;
16580
+ const isOpeningCapabilityAdvertisement =
16581
+ msg instanceof SyncCapabilitiesMessage && isOpeningSubscriptionReceive;
16582
+ releasePeerReceiveLease = this.acquirePeerReceiveLease(
16583
+ receiveFromHash,
16584
+ receiveReplicationLifecycleController,
16585
+ receiveSubscriptionEpoch,
16586
+ {
16587
+ // The replication-info fence existed before receive leases and is
16588
+ // intentionally narrower than the subscription itself. Keep admitting
16589
+ // sync negotiation/data while the new subscription drains the previous
16590
+ // apply generation; replication-info branches recheck the fence below.
16591
+ allowReplicationInfoBlocked: isOpeningSubscriptionReceive,
16592
+ allowCleanupGate: isOpeningCapabilityAdvertisement,
16593
+ },
16594
+ );
16595
+ if (!releasePeerReceiveLease) {
16596
+ return;
16597
+ }
16598
+ const receiveReplicationInfoReceiveEpoch =
16599
+ this.getReplicationInfoReceiveEpoch(receiveFromHash);
14940
16600
  if (!context.from.equals(this.node.identity.publicKey)) {
14941
- this.markReplicatorActivity(context.from.hashcode());
16601
+ this.markReplicatorActivity(receiveFromHash);
14942
16602
  }
14943
16603
 
14944
16604
  if (msg instanceof ResponseRoleMessage) {
@@ -16269,6 +17929,7 @@ export class SharedLog<
16269
17929
  );
16270
17930
  return allToMergeMaterializedEntries;
16271
17931
  };
17932
+ let admittedMergeHashes: ReadonlySet<string> = new Set();
16272
17933
  let nativePreparedCommittedHashes: Set<string> | undefined;
16273
17934
  if (allToMerge.length > 0) {
16274
17935
  const validateStartedAt = syncProfileStart(syncProfile);
@@ -16377,9 +18038,9 @@ export class SharedLog<
16377
18038
  const entriesToPersist = usedNativeAllKeptJoinPlan
16378
18039
  ? allToMergeShallowEntries
16379
18040
  : joinPlans.flatMap((plan) => plan.toPersist);
16380
- const coordinatePersistFallbackEntries: ShallowOrFullEntry<any>[] =
18041
+ let coordinatePersistFallbackEntries: ShallowOrFullEntry<any>[] =
16381
18042
  [];
16382
- const reusableCoordinatePersistItems: CoordinatePersistBatchItem<R>[] =
18043
+ let reusableCoordinatePersistItems: CoordinatePersistBatchItem<R>[] =
16383
18044
  [];
16384
18045
  for (const entry of entriesToPersist) {
16385
18046
  const reusablePlan = reusableCoordinatePlans.get(entry.hash);
@@ -16397,8 +18058,6 @@ export class SharedLog<
16397
18058
  prepared: reusablePlan.prepared,
16398
18059
  });
16399
18060
  }
16400
- const reusableCoordinatePersistItemCount =
16401
- reusableCoordinatePersistItems.length;
16402
18061
  let nativePreparedCoordinateBatch:
16403
18062
  | NativeBackboneReceiveCoordinateBatch<R>
16404
18063
  | undefined;
@@ -16501,6 +18160,34 @@ export class SharedLog<
16501
18160
  __peerbitProfile: syncProfile,
16502
18161
  });
16503
18162
  }
18163
+ // A recursive lower-log join can resolve successfully while declining
18164
+ // an individual top-level entry (for example, when one of its parents
18165
+ // is temporarily unavailable). The public Log.join() API intentionally
18166
+ // does not expose that per-entry result, so make local index presence the
18167
+ // authority before publishing any SharedLog-side effects. A successful
18168
+ // prepared-facts batch is atomic and already proves every input hash.
18169
+ const admittedHashes = joinedPreparedFacts
18170
+ ? new Set(allToMergeHashes)
18171
+ : await this.log.hasMany(allToMergeHashes);
18172
+ admittedMergeHashes = admittedHashes;
18173
+ const admittedShallowEntries =
18174
+ admittedHashes.size === allToMergeShallowEntries.length
18175
+ ? allToMergeShallowEntries
18176
+ : allToMergeShallowEntries.filter((entry) =>
18177
+ admittedHashes.has(entry.hash),
18178
+ );
18179
+ if (!joinedPreparedFacts) {
18180
+ reusableCoordinatePersistItems =
18181
+ reusableCoordinatePersistItems.filter((item) =>
18182
+ admittedHashes.has(item.entry.hash),
18183
+ );
18184
+ coordinatePersistFallbackEntries =
18185
+ coordinatePersistFallbackEntries.filter((entry) =>
18186
+ admittedHashes.has(entry.hash),
18187
+ );
18188
+ }
18189
+ const reusableCoordinatePersistItemCount =
18190
+ reusableCoordinatePersistItems.length;
16504
18191
  if (syncProfile) {
16505
18192
  emitSyncProfileDuration(syncProfile, lowerLogJoinStartedAt, {
16506
18193
  name: "sharedLog.receive.lowerLogJoin",
@@ -16512,6 +18199,7 @@ export class SharedLog<
16512
18199
  batchHashOnlyEntryAdded,
16513
18200
  programOnChange,
16514
18201
  joinedPreparedFacts,
18202
+ admittedEntries: admittedHashes.size,
16515
18203
  nativePreparedCoordinatesFinished,
16516
18204
  },
16517
18205
  });
@@ -16586,11 +18274,11 @@ export class SharedLog<
16586
18274
  },
16587
18275
  });
16588
18276
  }
16589
- for (const hash of allToMergeHashes) {
18277
+ for (const hash of admittedHashes) {
16590
18278
  confirmedHashes.add(hash);
16591
18279
  }
16592
18280
  const checkedPruneStartedAt = syncProfileStart(syncProfile);
16593
- await this.pruneJoinedEntriesNoLongerLed(allToMergeShallowEntries, {
18281
+ await this.pruneJoinedEntriesNoLongerLed(admittedShallowEntries, {
16594
18282
  decodedReplicaCounts: receiveReplicaCounts,
16595
18283
  reusableLeaderPlans: reusableCoordinatePlans,
16596
18284
  profile: syncProfile,
@@ -16605,15 +18293,17 @@ export class SharedLog<
16605
18293
  }
16606
18294
 
16607
18295
  for (const plan of joinPlans) {
16608
- plan.toDelete?.map((x) =>
16609
- this.pruneDebouncedFnAddIfNotKeeping({
16610
- key: x.hash,
16611
- value: {
16612
- entry: x,
16613
- leaders: plan.leaders as Map<string, any>,
16614
- },
16615
- }),
16616
- );
18296
+ plan.toDelete
18297
+ ?.filter((entry) => admittedMergeHashes.has(entry.hash))
18298
+ .map((entry) =>
18299
+ this.pruneDebouncedFnAddIfNotKeeping({
18300
+ key: entry.hash,
18301
+ value: {
18302
+ entry,
18303
+ leaders: plan.leaders as Map<string, any>,
18304
+ },
18305
+ }),
18306
+ );
16617
18307
  }
16618
18308
  this.rebalanceParticipationDebounced?.call();
16619
18309
  }
@@ -16623,17 +18313,23 @@ export class SharedLog<
16623
18313
  continue;
16624
18314
  }
16625
18315
  for (const entries of plan.maybeDelete) {
18316
+ const admittedEntries = entries.filter((entry) =>
18317
+ admittedMergeHashes.has(getExchangeHeadHash(entry)),
18318
+ );
18319
+ if (admittedEntries.length === 0) {
18320
+ continue;
18321
+ }
16626
18322
  const minReplicas = await this.getMaxReplicasFromHeads(
16627
- this.getEntryGid(entries[0].entry),
18323
+ this.getEntryGid(admittedEntries[0].entry),
16628
18324
  );
16629
18325
  if (minReplicas != null) {
16630
18326
  const isLeader = await this.isLeader({
16631
- entry: entries[0].entry,
18327
+ entry: admittedEntries[0].entry,
16632
18328
  replicas: minReplicas,
16633
18329
  });
16634
18330
 
16635
18331
  if (!isLeader) {
16636
- for (const x of entries) {
18332
+ for (const x of admittedEntries) {
16637
18333
  this.pruneDebouncedFnAddIfNotKeeping({
16638
18334
  key: x.entry.hash,
16639
18335
  value: {
@@ -16877,6 +18573,13 @@ export class SharedLog<
16877
18573
  let pendingIHaveExtended = 0;
16878
18574
  const leaderResponseHashes: string[] = [];
16879
18575
  for (let i = 0; i < msg.hashes.length; i++) {
18576
+ if (
18577
+ !this.isReplicationLifecycleActive(
18578
+ receiveReplicationLifecycleController,
18579
+ )
18580
+ ) {
18581
+ return;
18582
+ }
16880
18583
  const hash = msg.hashes[i]!;
16881
18584
 
16882
18585
  const nativeEntryGid = nativeLeaderHints.nativeEntryGids?.[i];
@@ -17012,8 +18715,21 @@ export class SharedLog<
17012
18715
  resetTimeout: () => this.resetPendingIHaveTimeout(pendingIHave),
17013
18716
  clear: () => this.clearPendingIHaveTimeout(pendingIHave),
17014
18717
  callback: async (entry: Entry<T>) => {
17015
- this.removePeerFromGidPeerHistory(from, entry.meta.gid);
17016
- this.removePruneRequestSent(entry.hash, from);
18718
+ if (
18719
+ !this.isReplicationLifecycleActive(
18720
+ receiveReplicationLifecycleController,
18721
+ ) ||
18722
+ requesting.size === 0
18723
+ ) {
18724
+ return;
18725
+ }
18726
+ for (const requester of requesting) {
18727
+ this.removePeerFromGidPeerHistory(
18728
+ requester,
18729
+ entry.meta.gid,
18730
+ );
18731
+ this.removePruneRequestSent(entry.hash, requester);
18732
+ }
17017
18733
  let isLeader = false;
17018
18734
  await this._waitForEntryReplicators(
17019
18735
  entry,
@@ -17032,12 +18748,19 @@ export class SharedLog<
17032
18748
  },
17033
18749
  },
17034
18750
  );
18751
+ if (
18752
+ !this.isReplicationLifecycleActive(
18753
+ receiveReplicationLifecycleController,
18754
+ ) ||
18755
+ requesting.size === 0
18756
+ ) {
18757
+ return;
18758
+ }
17035
18759
  if (isLeader) {
17036
18760
  this.responseToPruneDebouncedFn.add({
17037
18761
  hashes: [entry.hash],
17038
- peers: requesting,
18762
+ peers: new Set(requesting),
17039
18763
  });
17040
- this._pendingIHave.delete(hash);
17041
18764
  }
17042
18765
  },
17043
18766
  };
@@ -17080,19 +18803,32 @@ export class SharedLog<
17080
18803
  }
17081
18804
  } else if (msg instanceof ResponseIPrune) {
17082
18805
  const lateResponses: string[] = [];
18806
+ const responseTasks: Promise<void>[] = [];
17083
18807
  for (const hash of msg.hashes) {
17084
18808
  const pendingDelete = this._checkedPrune.getPendingDelete(hash);
17085
18809
  if (pendingDelete) {
17086
- void pendingDelete.resolve(context.from.hashcode());
18810
+ responseTasks.push(
18811
+ Promise.resolve(
18812
+ pendingDelete.resolve(context.from.hashcode()),
18813
+ ),
18814
+ );
17087
18815
  } else {
17088
18816
  lateResponses.push(hash);
17089
18817
  }
17090
18818
  }
17091
18819
  if (lateResponses.length > 0) {
17092
- void this.recoverCheckedPruneFromLateResponses(
17093
- lateResponses,
17094
- context.from.hashcode(),
17095
- ).catch((error) => logger.error(error.toString()));
18820
+ responseTasks.push(
18821
+ this.recoverCheckedPruneFromLateResponses(
18822
+ lateResponses,
18823
+ context.from.hashcode(),
18824
+ ),
18825
+ );
18826
+ }
18827
+ const results = await Promise.allSettled(responseTasks);
18828
+ for (const result of results) {
18829
+ if (result.status === "rejected") {
18830
+ logger.error(result.reason?.toString?.() ?? String(result.reason));
18831
+ }
17096
18832
  }
17097
18833
  } else if (msg instanceof ConfirmEntriesMessage) {
17098
18834
  this.markEntriesKnownByPeer(msg.hashes, context.from.hashcode());
@@ -17100,10 +18836,25 @@ export class SharedLog<
17100
18836
  return;
17101
18837
  } else if (msg instanceof SyncCapabilitiesMessage) {
17102
18838
  if (!context.from.equals(this.node.identity.publicKey)) {
17103
- this._peerSyncCapabilities.set(
17104
- context.from.hashcode(),
17105
- msg.capabilities,
17106
- );
18839
+ const openingEpoch =
18840
+ this._subscriptionOpeningEpochByPeer.get(receiveFromHash);
18841
+ if (
18842
+ this._replicationInfoBlockedPeers.has(receiveFromHash) &&
18843
+ openingEpoch === receiveSubscriptionEpoch
18844
+ ) {
18845
+ // A prior unsubscribe cleanup may still be ahead of this reconnect
18846
+ // barrier. Stage the new generation's one-shot advertisement so that
18847
+ // cleanup cannot erase it before the opening transition commits.
18848
+ this._openingSyncCapabilitiesByPeer.set(receiveFromHash, {
18849
+ epoch: openingEpoch,
18850
+ capabilities: msg.capabilities,
18851
+ });
18852
+ } else {
18853
+ this._peerSyncCapabilities.set(
18854
+ receiveFromHash,
18855
+ msg.capabilities,
18856
+ );
18857
+ }
17107
18858
  }
17108
18859
  return;
17109
18860
  } else if (await this.syncronizer.onMessage(msg, context)) {
@@ -17120,10 +18871,14 @@ export class SharedLog<
17120
18871
  return;
17121
18872
  }
17122
18873
  const replicationLifecycleController =
17123
- this._replicationLifecycleController;
18874
+ receiveReplicationLifecycleController;
17124
18875
  if (
17125
18876
  !replicationLifecycleController ||
17126
- !this.isReplicationLifecycleActive(replicationLifecycleController)
18877
+ !this.isPeerReceiveAdmissionOpen(
18878
+ receiveFromHash,
18879
+ replicationLifecycleController,
18880
+ receiveSubscriptionEpoch,
18881
+ )
17127
18882
  ) {
17128
18883
  return;
17129
18884
  }
@@ -17133,8 +18888,10 @@ export class SharedLog<
17133
18888
  replicationSegments = await this.getMyReplicationSegments();
17134
18889
  } catch (error) {
17135
18890
  if (
17136
- !this.isReplicationLifecycleActive(
18891
+ !this.isPeerReceiveAdmissionOpen(
18892
+ receiveFromHash,
17137
18893
  replicationLifecycleController,
18894
+ receiveSubscriptionEpoch,
17138
18895
  ) &&
17139
18896
  isNotStartedError(error as Error)
17140
18897
  ) {
@@ -17143,7 +18900,11 @@ export class SharedLog<
17143
18900
  throw error;
17144
18901
  }
17145
18902
  if (
17146
- !this.isReplicationLifecycleActive(replicationLifecycleController)
18903
+ !this.isPeerReceiveAdmissionOpen(
18904
+ receiveFromHash,
18905
+ replicationLifecycleController,
18906
+ receiveSubscriptionEpoch,
18907
+ )
17147
18908
  ) {
17148
18909
  return;
17149
18910
  }
@@ -17164,7 +18925,11 @@ export class SharedLog<
17164
18925
  ),
17165
18926
  );
17166
18927
  if (
17167
- !this.isReplicationLifecycleActive(replicationLifecycleController)
18928
+ !this.isPeerReceiveAdmissionOpen(
18929
+ receiveFromHash,
18930
+ replicationLifecycleController,
18931
+ receiveSubscriptionEpoch,
18932
+ )
17168
18933
  ) {
17169
18934
  return;
17170
18935
  }
@@ -17214,14 +18979,38 @@ export class SharedLog<
17214
18979
  // (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
17215
18980
  const from = context.from!;
17216
18981
  const fromHash = from.hashcode();
17217
- if (this._replicationInfoBlockedPeers.has(fromHash)) {
18982
+ if (
18983
+ !this.isReplicationLifecycleActive(
18984
+ receiveReplicationLifecycleController,
18985
+ ) ||
18986
+ !this.isCurrentReplicationInfoReceiveEpoch(
18987
+ receiveFromHash,
18988
+ receiveReplicationInfoReceiveEpoch,
18989
+ ) ||
18990
+ this._replicationInfoBlockedPeers.has(fromHash)
18991
+ ) {
17218
18992
  return;
17219
18993
  }
17220
18994
  const messageTimestamp = context.message.header.timestamp;
18995
+ releasePeerReceiveLease?.();
18996
+ releasePeerReceiveLease = undefined;
17221
18997
  await this.withReplicationInfoApplyQueue(fromHash, async () => {
17222
18998
  try {
17223
18999
  // The peer may have unsubscribed after this message was queued.
17224
- if (this._replicationInfoBlockedPeers.has(fromHash)) {
19000
+ if (
19001
+ !this.isReplicationLifecycleActive(
19002
+ receiveReplicationLifecycleController,
19003
+ ) ||
19004
+ !this.isCurrentSubscriptionEpoch(
19005
+ fromHash,
19006
+ receiveSubscriptionEpoch,
19007
+ ) ||
19008
+ !this.isCurrentReplicationInfoReceiveEpoch(
19009
+ fromHash,
19010
+ receiveReplicationInfoReceiveEpoch,
19011
+ ) ||
19012
+ this._replicationInfoBlockedPeers.has(fromHash)
19013
+ ) {
17225
19014
  return;
17226
19015
  }
17227
19016
 
@@ -17269,29 +19058,72 @@ export class SharedLog<
17269
19058
  }
17270
19059
  });
17271
19060
  } else if (msg instanceof StoppedReplicating) {
17272
- if (context.from.equals(this.node.identity.publicKey)) {
19061
+ const from = context.from!;
19062
+ const segmentIds = msg.segmentIds;
19063
+ if (from.equals(this.node.identity.publicKey)) {
17273
19064
  return;
17274
19065
  }
17275
- const fromHash = context.from.hashcode();
17276
- if (this._replicationInfoBlockedPeers.has(fromHash)) {
19066
+ const fromHash = from.hashcode();
19067
+ if (
19068
+ !this.isReplicationLifecycleActive(
19069
+ receiveReplicationLifecycleController,
19070
+ ) ||
19071
+ !this.isCurrentReplicationInfoReceiveEpoch(
19072
+ receiveFromHash,
19073
+ receiveReplicationInfoReceiveEpoch,
19074
+ ) ||
19075
+ this._replicationInfoBlockedPeers.has(fromHash)
19076
+ ) {
17277
19077
  return;
17278
19078
  }
17279
- this._replicatorLivenessFailures.delete(fromHash);
19079
+ const messageTimestamp = context.message.header.timestamp;
19080
+ releasePeerReceiveLease?.();
19081
+ releasePeerReceiveLease = undefined;
19082
+ await this.withReplicationInfoApplyQueue(fromHash, async () => {
19083
+ if (
19084
+ !this.isReplicationLifecycleActive(
19085
+ receiveReplicationLifecycleController,
19086
+ ) ||
19087
+ !this.isCurrentSubscriptionEpoch(
19088
+ fromHash,
19089
+ receiveSubscriptionEpoch,
19090
+ ) ||
19091
+ !this.isCurrentReplicationInfoReceiveEpoch(
19092
+ fromHash,
19093
+ receiveReplicationInfoReceiveEpoch,
19094
+ ) ||
19095
+ this._replicationInfoBlockedPeers.has(fromHash)
19096
+ ) {
19097
+ return;
19098
+ }
17280
19099
 
17281
- const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(
17282
- msg.segmentIds,
17283
- context.from,
17284
- );
19100
+ const previousTimestamp =
19101
+ this.latestReplicationInfoMessage.get(fromHash);
19102
+ if (previousTimestamp && previousTimestamp > messageTimestamp) {
19103
+ return;
19104
+ }
19105
+ this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
19106
+ this._replicatorLivenessFailures.delete(fromHash);
19107
+ if (this.closed) {
19108
+ return;
19109
+ }
17285
19110
 
17286
- await this.removeReplicationRanges(rangesToRemove, context.from);
17287
- const timestamp = BigInt(+new Date());
17288
- for (const range of rangesToRemove) {
17289
- this.replicationChangeDebounceFn.add({
17290
- range,
17291
- type: "removed",
17292
- timestamp,
17293
- });
17294
- }
19111
+ const rangesToRemove =
19112
+ await this.resolveReplicationRangesFromIdsAndKey(
19113
+ segmentIds,
19114
+ from,
19115
+ );
19116
+
19117
+ await this.removeReplicationRanges(rangesToRemove, from);
19118
+ const timestamp = BigInt(+new Date());
19119
+ for (const range of rangesToRemove) {
19120
+ this.replicationChangeDebounceFn.add({
19121
+ range,
19122
+ type: "removed",
19123
+ timestamp,
19124
+ });
19125
+ }
19126
+ });
17295
19127
  } else {
17296
19128
  throw new Error("Unexpected message");
17297
19129
  }
@@ -17345,6 +19177,8 @@ export class SharedLog<
17345
19177
  // Every return and every locally swallowed receive error passes this
17346
19178
  // boundary. Release a native wire stash exactly once first, then surface
17347
19179
  // any durable mutation poison that arose while handling the message.
19180
+ releasePeerReceiveLease?.();
19181
+ releasePeerReceiveLease = undefined;
17348
19182
  this.throwIfNativeDurableCommitFailed();
17349
19183
  }
17350
19184
  }
@@ -17987,30 +19821,35 @@ export class SharedLog<
17987
19821
  checkLeaders: (options: WaitForReplicatorsOptions<R>) => Promise<LeaderMap>,
17988
19822
  ): Promise<LeaderMap | false> {
17989
19823
  const timeout = options.timeout ?? this.waitForReplicatorTimeout;
19824
+ const closeSignal = this._closeController.signal;
19825
+ const replicationLifecycleSignal =
19826
+ this._replicationLifecycleController?.signal;
17990
19827
 
17991
19828
  return new Promise((resolve, reject) => {
17992
19829
  let settled = false;
19830
+ const checks = new Set<Promise<void>>();
17993
19831
  const removeListeners = () => {
17994
19832
  this.events.removeEventListener("replication:change", roleListener);
17995
19833
  this.events.removeEventListener("replicator:mature", roleListener);
17996
- this._closeController.signal.removeEventListener(
17997
- "abort",
17998
- abortListener,
17999
- );
19834
+ closeSignal.removeEventListener("abort", abortListener);
19835
+ replicationLifecycleSignal?.removeEventListener("abort", abortListener);
18000
19836
  };
18001
19837
  const settleResolve = (value: LeaderMap | false) => {
18002
19838
  if (settled) return;
18003
19839
  settled = true;
18004
19840
  removeListeners();
18005
19841
  clearTimeout(timer);
18006
- resolve(value);
19842
+ // Leader planning may persist coordinates. Keep the caller (and any
19843
+ // receive lease it owns) alive until checks admitted before this
19844
+ // timeout/abort have finished their local side effects.
19845
+ void Promise.allSettled([...checks]).then(() => resolve(value));
18007
19846
  };
18008
19847
  const settleReject = (error: unknown) => {
18009
19848
  if (settled) return;
18010
19849
  settled = true;
18011
19850
  removeListeners();
18012
19851
  clearTimeout(timer);
18013
- reject(error);
19852
+ void Promise.allSettled([...checks]).then(() => reject(error));
18014
19853
  };
18015
19854
  const abortListener = () => {
18016
19855
  settleResolve(false);
@@ -18044,9 +19883,14 @@ export class SharedLog<
18044
19883
  settleResolve(leaders);
18045
19884
  };
18046
19885
  const runCheck = () => {
18047
- void check().catch((error) => {
18048
- settleReject(error);
18049
- });
19886
+ if (settled) return;
19887
+ let running!: Promise<void>;
19888
+ running = check()
19889
+ .catch((error) => {
19890
+ settleReject(error);
19891
+ })
19892
+ .finally(() => checks.delete(running));
19893
+ checks.add(running);
18050
19894
  };
18051
19895
 
18052
19896
  const roleListener = () => {
@@ -18055,7 +19899,15 @@ export class SharedLog<
18055
19899
 
18056
19900
  this.events.addEventListener("replication:change", roleListener);
18057
19901
  this.events.addEventListener("replicator:mature", roleListener);
18058
- this._closeController.signal.addEventListener("abort", abortListener);
19902
+ closeSignal.addEventListener("abort", abortListener);
19903
+ replicationLifecycleSignal?.addEventListener("abort", abortListener);
19904
+ // AbortSignal does not replay an abort event to listeners added after it
19905
+ // fired. Recheck after registration so work started concurrently with the
19906
+ // terminal fence cannot wait for the full leader-selection timeout.
19907
+ if (closeSignal.aborted || replicationLifecycleSignal?.aborted) {
19908
+ abortListener();
19909
+ return;
19910
+ }
18059
19911
  runCheck();
18060
19912
  });
18061
19913
  }
@@ -21059,6 +22911,53 @@ export class SharedLog<
21059
22911
  });
21060
22912
  }
21061
22913
 
22914
+ private async drainReplicationInfoApplyQueues(): Promise<void> {
22915
+ for (;;) {
22916
+ const tails = [...(this._replicationInfoApplyQueueByPeer?.values() ?? [])];
22917
+ if (tails.length === 0) {
22918
+ return;
22919
+ }
22920
+ await Promise.allSettled(tails);
22921
+ // Queue cleanup runs in `finally`; give it a microtask before checking for
22922
+ // tails admitted while the previous snapshot was settling.
22923
+ await Promise.resolve();
22924
+ }
22925
+ }
22926
+
22927
+ private advanceReplicationInfoReceiveEpoch(peerHash: string): object {
22928
+ const next = {};
22929
+ this._replicationInfoReceiveEpochByPeer.set(peerHash, next);
22930
+ return next;
22931
+ }
22932
+
22933
+ private getReplicationInfoReceiveEpoch(peerHash: string): object | null {
22934
+ return this._replicationInfoReceiveEpochByPeer.get(peerHash) ?? null;
22935
+ }
22936
+
22937
+ private isCurrentReplicationInfoReceiveEpoch(
22938
+ peerHash: string,
22939
+ epoch: object | null,
22940
+ ): boolean {
22941
+ return this.getReplicationInfoReceiveEpoch(peerHash) === epoch;
22942
+ }
22943
+
22944
+ private advanceSubscriptionEpoch(peerHash: string): object {
22945
+ const next = {};
22946
+ this._subscriptionEpochByPeer.set(peerHash, next);
22947
+ return next;
22948
+ }
22949
+
22950
+ private getSubscriptionEpoch(peerHash: string): object | null {
22951
+ return this._subscriptionEpochByPeer.get(peerHash) ?? null;
22952
+ }
22953
+
22954
+ private isCurrentSubscriptionEpoch(
22955
+ peerHash: string,
22956
+ epoch: object | null,
22957
+ ): boolean {
22958
+ return this.getSubscriptionEpoch(peerHash) === epoch;
22959
+ }
22960
+
21062
22961
  private cancelReplicationInfoRequests(peerHash: string) {
21063
22962
  const state = this._replicationInfoRequestByPeer.get(peerHash);
21064
22963
  if (!state) return;
@@ -21147,6 +23046,7 @@ export class SharedLog<
21147
23046
  publicKey: PublicSignKey,
21148
23047
  topics: string[],
21149
23048
  subscribed: boolean,
23049
+ subscriptionEpoch?: object,
21150
23050
  ) {
21151
23051
  if (!topics.includes(this.topic)) {
21152
23052
  return;
@@ -21160,8 +23060,76 @@ export class SharedLog<
21160
23060
  }
21161
23061
 
21162
23062
  const peerHash = publicKey.hashcode();
23063
+ const expectedSubscriptionEpoch =
23064
+ subscriptionEpoch ?? this.advanceSubscriptionEpoch(peerHash);
23065
+ const ownsSubscriptionEpoch = () =>
23066
+ this.isCurrentSubscriptionEpoch(peerHash, expectedSubscriptionEpoch);
23067
+ if (!ownsSubscriptionEpoch()) {
23068
+ return;
23069
+ }
23070
+ if (subscribed) {
23071
+ const pendingOpeningCapabilities =
23072
+ this._openingSyncCapabilitiesByPeer.get(peerHash);
23073
+ if (
23074
+ pendingOpeningCapabilities &&
23075
+ pendingOpeningCapabilities.epoch !== expectedSubscriptionEpoch
23076
+ ) {
23077
+ this._openingSyncCapabilitiesByPeer.delete(peerHash);
23078
+ }
23079
+ this._subscriptionOpeningEpochByPeer.set(
23080
+ peerHash,
23081
+ expectedSubscriptionEpoch,
23082
+ );
23083
+ // Fence new messages immediately, drain handlers admitted by the previous
23084
+ // subscription, then wait behind every queued replication mutation. A
23085
+ // reconnect must not inherit metadata or ranges from the old connection.
23086
+ try {
23087
+ this._replicationInfoBlockedPeers.add(peerHash);
23088
+ await this.drainPeerReceiveHandlers(peerHash);
23089
+ await this.withReplicationInfoApplyQueue(peerHash, async () => {});
23090
+ if (
23091
+ !this.isReplicationLifecycleActive(replicationLifecycleController) ||
23092
+ !ownsSubscriptionEpoch()
23093
+ ) {
23094
+ return;
23095
+ }
23096
+ // The timestamp watermark belongs to the previous subscription epoch.
23097
+ // Sender clocks are not synchronized, so carrying a local unsubscribe
23098
+ // timestamp forward could reject every valid announcement after reconnect.
23099
+ this.latestReplicationInfoMessage.delete(peerHash);
23100
+ this._pendingReplicatorLeaveByPeer.delete(peerHash);
23101
+ const openingCapabilities =
23102
+ this._openingSyncCapabilitiesByPeer.get(peerHash);
23103
+ if (openingCapabilities?.epoch === expectedSubscriptionEpoch) {
23104
+ this._peerSyncCapabilities.set(
23105
+ peerHash,
23106
+ openingCapabilities.capabilities,
23107
+ );
23108
+ this._openingSyncCapabilitiesByPeer.delete(peerHash);
23109
+ }
23110
+ this._replicationInfoBlockedPeers.delete(peerHash);
23111
+ } finally {
23112
+ if (
23113
+ this._openingSyncCapabilitiesByPeer.get(peerHash)?.epoch ===
23114
+ expectedSubscriptionEpoch
23115
+ ) {
23116
+ this._openingSyncCapabilitiesByPeer.delete(peerHash);
23117
+ }
23118
+ if (
23119
+ this._subscriptionOpeningEpochByPeer.get(peerHash) ===
23120
+ expectedSubscriptionEpoch
23121
+ ) {
23122
+ this._subscriptionOpeningEpochByPeer.delete(peerHash);
23123
+ }
23124
+ }
23125
+ }
21163
23126
  if (!subscribed) {
23127
+ this._subscriptionOpeningEpochByPeer.delete(peerHash);
23128
+ this._openingSyncCapabilitiesByPeer.delete(peerHash);
21164
23129
  this._replicationInfoBlockedPeers.add(peerHash);
23130
+ const disconnectedJoinWarmupGeneration =
23131
+ this._joinWarmupGenerationByTarget.get(peerHash) ?? null;
23132
+ this.cancelJoinWarmupTarget(peerHash);
21165
23133
 
21166
23134
  const now = BigInt(+new Date());
21167
23135
  const previous = this.latestReplicationInfoMessage.get(peerHash);
@@ -21169,24 +23137,37 @@ export class SharedLog<
21169
23137
  this.latestReplicationInfoMessage.set(peerHash, now);
21170
23138
  }
21171
23139
 
21172
- const wasReplicator = this.uniqueReplicators.has(peerHash);
23140
+ let removed = false;
21173
23141
  try {
21174
23142
  // Unsubscribe can race with the peer's final replication reset message.
21175
23143
  // Proactively evict its ranges so leader selection doesn't keep stale owners.
21176
- await this.removeReplicator(publicKey, { noEvent: true });
23144
+ removed = await this.removeReplicator(publicKey, {
23145
+ cleanupIfSubscriptionSuperseded: true,
23146
+ expectedJoinWarmupGeneration:
23147
+ disconnectedJoinWarmupGeneration,
23148
+ noEvent: true,
23149
+ onRemoved: ({ wasReplicator }) => {
23150
+ if (wasReplicator) {
23151
+ this._pendingReplicatorLeaveByPeer.add(peerHash);
23152
+ }
23153
+ },
23154
+ replicationLifecycleController,
23155
+ subscriptionEpoch: expectedSubscriptionEpoch,
23156
+ });
21177
23157
  } catch (error) {
21178
23158
  if (!isNotStartedError(error as Error)) {
21179
23159
  throw error;
21180
23160
  }
21181
23161
  }
21182
- if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
23162
+ if (
23163
+ !this.isReplicationLifecycleActive(replicationLifecycleController) ||
23164
+ !ownsSubscriptionEpoch() ||
23165
+ !removed
23166
+ ) {
21183
23167
  return;
21184
23168
  }
21185
23169
 
21186
- this._replicatorJoinEmitted.delete(peerHash);
21187
- this.cleanupPeerDisconnectTracking(peerHash);
21188
-
21189
- if (wasReplicator) {
23170
+ if (this._pendingReplicatorLeaveByPeer.delete(peerHash)) {
21190
23171
  this.events.dispatchEvent(
21191
23172
  new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
21192
23173
  detail: { publicKey },
@@ -21234,7 +23215,10 @@ export class SharedLog<
21234
23215
  }
21235
23216
  throw error;
21236
23217
  }
21237
- if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
23218
+ if (
23219
+ !this.isReplicationLifecycleActive(replicationLifecycleController) ||
23220
+ !ownsSubscriptionEpoch()
23221
+ ) {
21238
23222
  return;
21239
23223
  }
21240
23224
  if (replicationSegments.length > 0) {
@@ -21254,7 +23238,10 @@ export class SharedLog<
21254
23238
  replicationLifecycleController,
21255
23239
  ),
21256
23240
  );
21257
- if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
23241
+ if (
23242
+ !this.isReplicationLifecycleActive(replicationLifecycleController) ||
23243
+ !ownsSubscriptionEpoch()
23244
+ ) {
21258
23245
  return;
21259
23246
  }
21260
23247
 
@@ -21285,7 +23272,10 @@ export class SharedLog<
21285
23272
  // Request the remote peer's replication info. This makes joins resilient to
21286
23273
  // timing-sensitive delivery/order issues where we may miss their initial
21287
23274
  // replication announcement.
21288
- if (this.isReplicationLifecycleActive(replicationLifecycleController)) {
23275
+ if (
23276
+ this.isReplicationLifecycleActive(replicationLifecycleController) &&
23277
+ ownsSubscriptionEpoch()
23278
+ ) {
21289
23279
  this.scheduleReplicationInfoRequests(
21290
23280
  publicKey,
21291
23281
  replicationLifecycleController,
@@ -21718,14 +23708,23 @@ export class SharedLog<
21718
23708
  if (this.closed) {
21719
23709
  return;
21720
23710
  }
21721
-
21722
- await this.log.trim();
21723
-
21724
23711
  const batchedChanges = Array.isArray(changeOrChanges[0])
21725
23712
  ? (changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>[])
21726
23713
  : [changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>];
21727
23714
  const changes = batchedChanges.flat();
21728
23715
  const selfHash = this.node.identity.publicKey.hashcode();
23716
+ const joinWarmupGenerations = new Map<string, object>();
23717
+ for (const change of changes) {
23718
+ if (change.type === "added" && change.range.hash !== selfHash) {
23719
+ joinWarmupGenerations.set(
23720
+ change.range.hash,
23721
+ this.getJoinWarmupGeneration(change.range.hash),
23722
+ );
23723
+ }
23724
+ }
23725
+
23726
+ await this.log.trim();
23727
+
21729
23728
  // On removed ranges (peer leaves / shrink), gid-level history can hide
21730
23729
  // per-entry gaps. Force a fresh delivery pass for reassigned entries.
21731
23730
  const forceFreshDelivery = changes.some(
@@ -21803,6 +23802,12 @@ export class SharedLog<
21803
23802
  ),
21804
23803
  )
21805
23804
  : changes;
23805
+ const isCurrentJoinWarmupTarget = (target: string) =>
23806
+ warmupPeers.has(target) &&
23807
+ this._joinWarmupGenerationByTarget.get(target) ===
23808
+ joinWarmupGenerations.get(target);
23809
+ const areJoinWarmupGenerationsCurrent = () =>
23810
+ [...warmupPeers].every(isCurrentJoinWarmupTarget);
21806
23811
 
21807
23812
  try {
21808
23813
  const uncheckedDeliver: Map<
@@ -21815,6 +23820,10 @@ export class SharedLog<
21815
23820
  return;
21816
23821
  }
21817
23822
  const isWarmupTarget = warmupPeers.has(target);
23823
+ if (isWarmupTarget && !isCurrentJoinWarmupTarget(target)) {
23824
+ uncheckedDeliver.delete(target);
23825
+ return;
23826
+ }
21818
23827
  const mode: RepairDispatchMode = forceFreshDelivery
21819
23828
  ? "churn"
21820
23829
  : isWarmupTarget
@@ -21839,6 +23848,9 @@ export class SharedLog<
21839
23848
  target: string,
21840
23849
  entry: EntryReplicated<any>,
21841
23850
  ) => {
23851
+ if (warmupPeers.has(target) && !isCurrentJoinWarmupTarget(target)) {
23852
+ return;
23853
+ }
21842
23854
  churnRepairPeers.add(target);
21843
23855
  let set = uncheckedDeliver.get(target);
21844
23856
  if (!set) {
@@ -21863,7 +23875,11 @@ export class SharedLog<
21863
23875
  forceFresh: forceFreshDelivery || useJoinWarmupFastPath,
21864
23876
  },
21865
23877
  )) {
21866
- if (this.closed) {
23878
+ if (
23879
+ this.closed ||
23880
+ (useJoinWarmupFastPath &&
23881
+ !areJoinWarmupGenerationsCurrent())
23882
+ ) {
21867
23883
  break;
21868
23884
  }
21869
23885
 
@@ -21883,7 +23899,9 @@ export class SharedLog<
21883
23899
 
21884
23900
  const candidatePeers = new Set<string>([selfHash]);
21885
23901
  for (const target of warmupPeers) {
21886
- candidatePeers.add(target);
23902
+ if (isCurrentJoinWarmupTarget(target)) {
23903
+ candidatePeers.add(target);
23904
+ }
21887
23905
  }
21888
23906
  if (oldPeersSet) {
21889
23907
  for (const oldPeer of oldPeersSet) {
@@ -21900,6 +23918,9 @@ export class SharedLog<
21900
23918
  persist: false,
21901
23919
  },
21902
23920
  );
23921
+ if (!areJoinWarmupGenerationsCurrent()) {
23922
+ continue;
23923
+ }
21903
23924
 
21904
23925
  if (oldPeersSet) {
21905
23926
  for (const oldPeer of oldPeersSet) {
@@ -21910,14 +23931,18 @@ export class SharedLog<
21910
23931
  }
21911
23932
 
21912
23933
  for (const [peer] of currentPeers) {
21913
- if (warmupPeers.has(peer)) {
21914
- this.markRepairSweepOptimisticPeer(entryReplicated.gid, peer);
23934
+ if (isCurrentJoinWarmupTarget(peer)) {
23935
+ this.markRepairSweepOptimisticPeer(
23936
+ entryReplicated.gid,
23937
+ peer,
23938
+ joinWarmupGenerations.get(peer)!,
23939
+ );
21915
23940
  }
21916
23941
  }
21917
23942
 
21918
23943
  const authoritativePeers = [...currentPeers.keys()].filter(
21919
23944
  (peer) =>
21920
- !warmupPeers.has(peer) &&
23945
+ !isCurrentJoinWarmupTarget(peer) &&
21921
23946
  !this.hasPendingRepairSweepOptimisticPeer(
21922
23947
  entryReplicated.gid,
21923
23948
  peer,
@@ -21983,8 +24008,12 @@ export class SharedLog<
21983
24008
  }
21984
24009
 
21985
24010
  for (const [peer] of currentPeers) {
21986
- if (addedPeers.has(peer)) {
21987
- this.markRepairSweepOptimisticPeer(entryReplicated.gid, peer);
24011
+ if (isCurrentJoinWarmupTarget(peer)) {
24012
+ this.markRepairSweepOptimisticPeer(
24013
+ entryReplicated.gid,
24014
+ peer,
24015
+ joinWarmupGenerations.get(peer)!,
24016
+ );
21988
24017
  }
21989
24018
  }
21990
24019
 
@@ -22036,6 +24065,7 @@ export class SharedLog<
22036
24065
  this.scheduleRepairSweep({
22037
24066
  mode: "join-warmup",
22038
24067
  peers,
24068
+ joinWarmupGenerations,
22039
24069
  });
22040
24070
  }, 250);
22041
24071
  timer.unref?.();
@@ -22110,6 +24140,7 @@ export class SharedLog<
22110
24140
  }
22111
24141
 
22112
24142
  const fromHash = evt.detail.from.hashcode();
24143
+ const subscriptionEpoch = this.advanceSubscriptionEpoch(fromHash);
22113
24144
  this._replicationInfoBlockedPeers.add(fromHash);
22114
24145
  this._recentRepairDispatch.delete(fromHash);
22115
24146
 
@@ -22127,6 +24158,7 @@ export class SharedLog<
22127
24158
  evt.detail.from,
22128
24159
  evt.detail.topics,
22129
24160
  false,
24161
+ subscriptionEpoch,
22130
24162
  );
22131
24163
  }
22132
24164
 
@@ -22140,14 +24172,17 @@ export class SharedLog<
22140
24172
  return;
22141
24173
  }
22142
24174
 
24175
+ const fromHash = evt.detail.from.hashcode();
24176
+ const subscriptionEpoch = this.advanceSubscriptionEpoch(fromHash);
22143
24177
  this.remoteBlocks.onReachable(evt.detail.from);
22144
- this._replicationInfoBlockedPeers.delete(evt.detail.from.hashcode());
24178
+ this._replicationInfoBlockedPeers.add(fromHash);
22145
24179
  this.invalidateSharedLogTopicSubscribersCache();
22146
24180
 
22147
24181
  await this.handleSubscriptionChange(
22148
24182
  evt.detail.from,
22149
24183
  evt.detail.topics,
22150
24184
  true,
24185
+ subscriptionEpoch,
22151
24186
  );
22152
24187
  }
22153
24188
 
@@ -22342,7 +24377,7 @@ export class SharedLog<
22342
24377
 
22343
24378
  if (ih) {
22344
24379
  ih.clear();
22345
- ih.callback(entry);
24380
+ this.runPendingIHaveCallback(ih, entry);
22346
24381
  }
22347
24382
 
22348
24383
  this.syncronizer.onEntryAdded(entry);
@@ -22356,7 +24391,7 @@ export class SharedLog<
22356
24391
  }
22357
24392
  const entry = materializeEntry();
22358
24393
  ih.clear();
22359
- ih.callback(entry);
24394
+ this.runPendingIHaveCallback(ih, entry);
22360
24395
  this.syncronizer.onEntryAdded(entry);
22361
24396
  return;
22362
24397
  }