@peerbit/shared-log 13.2.10 → 13.2.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/src/index.js CHANGED
@@ -33,7 +33,7 @@ var __runInitializers = (this && this.__runInitializers) || function (thisArg, i
33
33
  return useValue ? value : void 0;
34
34
  };
35
35
  import { BorshError, deserialize, field, serialize, variant, } from "@dao-xyz/borsh";
36
- import { AnyBlockStore, RemoteBlocks } from "@peerbit/blocks";
36
+ import { AnyBlockStore, RemoteBlocks, } from "@peerbit/blocks";
37
37
  import { cidifyString } from "@peerbit/blocks-interface";
38
38
  import { Cache } from "@peerbit/cache";
39
39
  import { AccessError, Ed25519Keypair, Ed25519PublicKey, PublicSignKey, Secp256k1PublicKey, getPublicKeyFromPeerId, sha256Base64Sync, sha256Sync, toHexString, } from "@peerbit/crypto";
@@ -44,7 +44,7 @@ import { ClosedError, Program, TerminalOperationNotStartedError, } from "@peerbi
44
44
  import { FanoutChannel, waitForSubscribers, } from "@peerbit/pubsub";
45
45
  import { SubscriptionEvent, UnsubcriptionEvent, } from "@peerbit/pubsub-interface";
46
46
  import { RPC } from "@peerbit/rpc";
47
- import { ACK_CONTROL_PRIORITY, AcknowledgeDelivery, AnyWhere, BACKGROUND_MESSAGE_PRIORITY, CONVERGENCE_MESSAGE_PRIORITY, DataMessage, MessageHeader, NotStartedError, SilentDelivery, createRequestTransportContext, } from "@peerbit/stream-interface";
47
+ import { ACK_CONTROL_PRIORITY, AcknowledgeDelivery, AnyWhere, BACKGROUND_MESSAGE_PRIORITY, CONVERGENCE_MESSAGE_PRIORITY, DataMessage, DeliveryError, MessageHeader, NotStartedError, SilentDelivery, createRequestTransportContext, } from "@peerbit/stream-interface";
48
48
  import { AbortError, TimeoutError, debounceAccumulator, debounceFixedInterval, waitFor, } from "@peerbit/time";
49
49
  import pDefer, {} from "p-defer";
50
50
  import PQueue from "p-queue";
@@ -1381,6 +1381,46 @@ const isTransientReplicationAnnouncementError = (error, seen = new Set()) => {
1381
1381
  : "";
1382
1382
  return constructorName === "TimeoutError" || name === "TimeoutError";
1383
1383
  };
1384
+ /**
1385
+ * Directed transport-delivery repair is allowed to retry explicit delivery
1386
+ * failures in addition to timeouts. A DirectStream ACK confirms receipt of the
1387
+ * signed envelope, not successful application by the receiver. Keep this
1388
+ * separate from the primary fanout classifier above so replicate() rejection
1389
+ * semantics remain unchanged for programming, serialization, and lifecycle
1390
+ * errors.
1391
+ */
1392
+ const isTransientReplicationAnnouncementRepairError = (error, seen = new Set()) => {
1393
+ if (error != null &&
1394
+ (typeof error === "object" || typeof error === "function")) {
1395
+ if (seen.has(error)) {
1396
+ return false;
1397
+ }
1398
+ seen.add(error);
1399
+ }
1400
+ if (error instanceof DeliveryError || error instanceof TimeoutError) {
1401
+ return true;
1402
+ }
1403
+ const nested = error?.errors;
1404
+ if (Array.isArray(nested) && nested.length > 0) {
1405
+ return nested.every((item) => isTransientReplicationAnnouncementRepairError(item, new Set(seen)));
1406
+ }
1407
+ const cause = error?.cause;
1408
+ if (cause != null &&
1409
+ isTransientReplicationAnnouncementRepairError(cause, seen)) {
1410
+ return true;
1411
+ }
1412
+ const constructorName = typeof error?.constructor
1413
+ ?.name === "string"
1414
+ ? error.constructor.name
1415
+ : "";
1416
+ const name = typeof error?.name === "string"
1417
+ ? error.name
1418
+ : "";
1419
+ return (constructorName === "DeliveryError" ||
1420
+ name === "DeliveryError" ||
1421
+ constructorName === "TimeoutError" ||
1422
+ name === "TimeoutError");
1423
+ };
1384
1424
  const EMPTY_HASHES = [];
1385
1425
  const normalizedHashValues = (hashes) => {
1386
1426
  if (Array.isArray(hashes)) {
@@ -1442,6 +1482,22 @@ const createIndexableDomainFromResolution = (resolution) => {
1442
1482
  }
1443
1483
  throw new Error("Unsupported resolution");
1444
1484
  };
1485
+ const mergeDefinedFanoutChannelOptions = (...sources) => {
1486
+ let merged;
1487
+ for (const source of sources) {
1488
+ if (!source) {
1489
+ continue;
1490
+ }
1491
+ for (const [key, value] of Object.entries(source)) {
1492
+ if (value === undefined) {
1493
+ continue;
1494
+ }
1495
+ merged ??= {};
1496
+ merged[key] = value;
1497
+ }
1498
+ }
1499
+ return merged;
1500
+ };
1445
1501
  const applySharedLogNativeDefaults = (options, defaults) => {
1446
1502
  if (!defaults) {
1447
1503
  return options;
@@ -1453,11 +1509,18 @@ const applySharedLogNativeDefaults = (options, defaults) => {
1453
1509
  nativeWireSync: options?.sync?.nativeWireSync ?? defaults.sync?.nativeWireSync,
1454
1510
  }
1455
1511
  : undefined;
1512
+ const fanout = options?.fanout
1513
+ ? {
1514
+ ...options.fanout,
1515
+ channel: mergeDefinedFanoutChannelOptions(defaults.fanout?.channel, options.fanout.channel),
1516
+ }
1517
+ : undefined;
1456
1518
  return {
1457
1519
  ...options,
1458
1520
  nativeBackbone: options?.nativeBackbone ?? defaults.nativeBackbone,
1459
1521
  nativeGraph: options?.nativeGraph ?? defaults.nativeGraph,
1460
1522
  sync,
1523
+ fanout,
1461
1524
  };
1462
1525
  };
1463
1526
  export const DEFAULT_MIN_REPLICAS = 2;
@@ -1478,6 +1541,14 @@ const CHECKED_PRUNE_RETRY_MAX_DELAY_MS = 30_000;
1478
1541
  // DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
1479
1542
  const RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL = 1000;
1480
1543
  const REPLICATION_ANNOUNCEMENT_RETRY_INTERVAL = 1000;
1544
+ const REPLICATION_ANNOUNCEMENT_REPAIR_INTERVAL = 1000;
1545
+ const REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS = 3;
1546
+ // Repair one bounded cohort per mutation generation. The subscriber snapshot
1547
+ // is a best-effort cache and can contain thousands of entries, so attempting
1548
+ // the whole cache after every role mutation would turn convergence repair into
1549
+ // an unbounded burst of separately signed, acknowledged messages. A cursor
1550
+ // retained across generations rotates best-effort coverage over later changes.
1551
+ const REPLICATION_ANNOUNCEMENT_REPAIR_TARGETS_PER_GENERATION = 8;
1481
1552
  const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE = 0.01;
1482
1553
  const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT = 0.005;
1483
1554
  const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT = 0.001;
@@ -1517,9 +1588,13 @@ const REPLICATOR_LIVENESS_PROBE_FAILURES_TO_EVICT = 2;
1517
1588
  const CHURN_REPAIR_RETRY_SCHEDULE_MS = [
1518
1589
  0, 1_000, 3_000, 7_000, 15_000, 30_000, 45_000,
1519
1590
  ];
1591
+ // Preserve the bounded retry window for transient local misses, but serialize
1592
+ // delayed warmup sends per target so fixed snapshots cannot overlap and amplify
1593
+ // large transfers. Every queued pass re-checks current peer knowledge on entry.
1520
1594
  const JOIN_WARMUP_RETRY_SCHEDULE_MS = [
1521
1595
  0, 1_000, 3_000, 7_000, 15_000, 30_000, 60_000,
1522
1596
  ];
1597
+ const JOIN_WARMUP_SEND_SPACING_MS = 250;
1523
1598
  const JOIN_AUTHORITATIVE_RETRY_SCHEDULE_MS = [
1524
1599
  0, 1_000, 3_000, 7_000, 15_000, 30_000, 60_000,
1525
1600
  ];
@@ -1686,6 +1761,11 @@ let SharedLog = (() => {
1686
1761
  _subscriptionChangeCallbacks;
1687
1762
  _acceptSubscriptionChangeCallbacks = false;
1688
1763
  _replicationLifecycleController;
1764
+ _activeReceiveHandlersByPeer;
1765
+ _receiveHandlerDrainByPeer;
1766
+ _receiveCleanupGateByPeer;
1767
+ _subscriptionOpeningEpochByPeer;
1768
+ _openingSyncCapabilitiesByPeer;
1689
1769
  _onFanoutDataFn;
1690
1770
  _onFanoutUnicastFn;
1691
1771
  _fanoutChannel;
@@ -1695,6 +1775,7 @@ let SharedLog = (() => {
1695
1775
  _closeController;
1696
1776
  _respondToIHaveTimeout;
1697
1777
  _checkedPrune;
1778
+ _pendingIHaveCallbacks;
1698
1779
  _pendingIHaveExpiryTimer;
1699
1780
  _pendingIHaveExpiryDeadline = Number.POSITIVE_INFINITY;
1700
1781
  get _pendingDeletes() {
@@ -1710,6 +1791,18 @@ let SharedLog = (() => {
1710
1791
  _replicationInfoBlockedPeers;
1711
1792
  _replicationInfoRequestByPeer;
1712
1793
  _replicationInfoApplyQueueByPeer;
1794
+ // Local receive generations fence replication-info handlers that were admitted
1795
+ // before a liveness eviction but reach the per-peer apply lane after it. Unlike
1796
+ // message timestamps, these tokens never compare clocks from different peers.
1797
+ _replicationInfoReceiveEpochByPeer;
1798
+ // Subscription callbacks can overlap because removing a replicator mutates the
1799
+ // replication index asynchronously. Keep that lifecycle separate from message
1800
+ // timestamps so a reconnect can synchronously revoke an older unsubscribe.
1801
+ _subscriptionEpochByPeer;
1802
+ // A superseded removal may be the queue item that actually observed an active
1803
+ // replicator. Carry that leave obligation to the transition that ultimately
1804
+ // wins, while a winning reconnect clears it without emitting a stale leave.
1805
+ _pendingReplicatorLeaveByPeer;
1713
1806
  _replicatorLivenessSweepRunning;
1714
1807
  _replicatorLivenessTimer;
1715
1808
  _replicatorLivenessTargets;
@@ -2461,6 +2554,15 @@ let SharedLog = (() => {
2461
2554
  _replicationAnnouncementRetryPending;
2462
2555
  _replicationAnnouncementRetryGeneration;
2463
2556
  _replicationAnnouncementRetryController;
2557
+ replicationAnnouncementRepairDebounced;
2558
+ _replicationAnnouncementRepairPending;
2559
+ _replicationAnnouncementRepairGeneration;
2560
+ _replicationAnnouncementRepairGenerationController;
2561
+ _replicationAnnouncementRepairTargets;
2562
+ _replicationAnnouncementRepairCohortSelected;
2563
+ _replicationAnnouncementRepairFairCursorHash;
2564
+ _replicationAnnouncementRepairMaxAttempts;
2565
+ _replicationAnnouncementRepairController;
2464
2566
  // A fn for debouncing the calls for pruning
2465
2567
  pruneDebouncedFn;
2466
2568
  responseToPruneDebouncedFn;
@@ -2479,10 +2581,16 @@ let SharedLog = (() => {
2479
2581
  _repairSweepRunning;
2480
2582
  _repairSweepPendingModes;
2481
2583
  _repairSweepPendingPeersByMode;
2584
+ _repairSweepJoinWarmupGenerationByTarget;
2482
2585
  _repairFrontierByMode;
2483
2586
  _repairFrontierActiveTargetsByMode;
2484
2587
  _repairFrontierBypassKnownPeersByMode;
2588
+ _joinWarmupGenerationByTarget;
2589
+ _joinWarmupSendStateByTarget;
2590
+ _joinWarmupRetryTimersByTarget;
2591
+ _joinWarmupScheduledRetriesByTarget;
2485
2592
  _repairSweepOptimisticGidPeersPending;
2593
+ _repairSweepOptimisticGidsByPeer;
2486
2594
  _entryKnownPeers;
2487
2595
  _entryKnownPeerObservedAt;
2488
2596
  _joinAuthoritativeRepairTimersByDelay;
@@ -2527,19 +2635,36 @@ let SharedLog = (() => {
2527
2635
  this.rpc = new RPC();
2528
2636
  this._checkedPrune = new CheckedPruneCoordinator();
2529
2637
  this._pendingIHave = new Map();
2638
+ this._pendingIHaveCallbacks = new Set();
2530
2639
  this.latestReplicationInfoMessage = new Map();
2531
2640
  this._replicationInfoBlockedPeers = new Set();
2532
2641
  this._replicationInfoRequestByPeer = new Map();
2533
2642
  this._replicationInfoApplyQueueByPeer = new Map();
2643
+ this._replicationInfoReceiveEpochByPeer = new Map();
2644
+ this._subscriptionEpochByPeer = new Map();
2645
+ this._pendingReplicatorLeaveByPeer = new Set();
2646
+ this._activeReceiveHandlersByPeer = new Map();
2647
+ this._receiveHandlerDrainByPeer = new Map();
2648
+ this._receiveCleanupGateByPeer = new Map();
2649
+ this._subscriptionOpeningEpochByPeer = new Map();
2650
+ this._openingSyncCapabilitiesByPeer = new Map();
2534
2651
  this._gidPeersHistory = new Map();
2535
2652
  this._repairRetryTimers = new Set();
2536
2653
  this._recentRepairDispatch = new Map();
2537
2654
  this._repairSweepRunning = false;
2538
2655
  this._repairSweepPendingModes = new Set();
2539
2656
  this._repairSweepPendingPeersByMode = createRepairPendingPeersByMode();
2657
+ this._repairSweepJoinWarmupGenerationByTarget = new Map();
2540
2658
  this._repairFrontierByMode = createRepairFrontierByMode();
2541
2659
  this._repairFrontierActiveTargetsByMode = createRepairActiveTargetsByMode();
2660
+ this._repairFrontierBypassKnownPeersByMode =
2661
+ createRepairFrontierBypassKnownPeersByMode();
2662
+ this._joinWarmupGenerationByTarget = new Map();
2663
+ this._joinWarmupSendStateByTarget = new Map();
2664
+ this._joinWarmupRetryTimersByTarget = new Map();
2665
+ this._joinWarmupScheduledRetriesByTarget = new Map();
2542
2666
  this._repairSweepOptimisticGidPeersPending = new Map();
2667
+ this._repairSweepOptimisticGidsByPeer = new Map();
2543
2668
  this._entryKnownPeers = new Map();
2544
2669
  this._joinAuthoritativeRepairTimersByDelay = new Map();
2545
2670
  this._joinAuthoritativeRepairPeersByDelay = new Map();
@@ -2562,6 +2687,16 @@ let SharedLog = (() => {
2562
2687
  this._replicationAnnouncementRetryPending = false;
2563
2688
  this._replicationAnnouncementRetryGeneration = 0;
2564
2689
  this._replicationAnnouncementRetryController = new AbortController();
2690
+ this._replicationAnnouncementRepairPending = false;
2691
+ this._replicationAnnouncementRepairGeneration = 0;
2692
+ this._replicationAnnouncementRepairGenerationController =
2693
+ new AbortController();
2694
+ this._replicationAnnouncementRepairTargets = new Map();
2695
+ this._replicationAnnouncementRepairCohortSelected = false;
2696
+ this._replicationAnnouncementRepairFairCursorHash = undefined;
2697
+ this._replicationAnnouncementRepairMaxAttempts =
2698
+ REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS;
2699
+ this._replicationAnnouncementRepairController = new AbortController();
2565
2700
  this.pendingMaturity = new Map();
2566
2701
  this._closeController = new AbortController();
2567
2702
  }
@@ -2848,7 +2983,7 @@ let SharedLog = (() => {
2848
2983
  });
2849
2984
  }
2850
2985
  async _sendAckWithUnifiedHints(properties) {
2851
- const { peer, message, payload, fanoutUnicastOptions } = properties;
2986
+ const { peer, message, payload, priority, fanoutUnicastOptions } = properties;
2852
2987
  const hints = await this._getSortedRouteHints(peer);
2853
2988
  const hasDirectHint = hints.some((hint) => hint.kind === "directstream-ack");
2854
2989
  const fanoutHint = hints.find((hint) => hint.kind === "fanout-token");
@@ -2859,6 +2994,7 @@ let SharedLog = (() => {
2859
2994
  redundancy: 1,
2860
2995
  to: [peer],
2861
2996
  }),
2997
+ priority,
2862
2998
  });
2863
2999
  return;
2864
3000
  }
@@ -2889,6 +3025,7 @@ let SharedLog = (() => {
2889
3025
  redundancy: 1,
2890
3026
  to: [peer],
2891
3027
  }),
3028
+ priority,
2892
3029
  });
2893
3030
  }
2894
3031
  /** Live append gossip that stayed on the plain TS path (countable in tests). */
@@ -3202,6 +3339,7 @@ let SharedLog = (() => {
3202
3339
  peer,
3203
3340
  message,
3204
3341
  payload,
3342
+ priority: delivery.priority,
3205
3343
  fanoutUnicastOptions,
3206
3344
  });
3207
3345
  })());
@@ -3214,6 +3352,7 @@ let SharedLog = (() => {
3214
3352
  redundancy: 1,
3215
3353
  to: nativeDeliveryPlan.silentTo,
3216
3354
  }),
3355
+ priority: delivery.priority,
3217
3356
  })
3218
3357
  .catch((error) => logger.error(error));
3219
3358
  }
@@ -3337,6 +3476,7 @@ let SharedLog = (() => {
3337
3476
  peer,
3338
3477
  message,
3339
3478
  payload,
3479
+ priority: delivery.priority,
3340
3480
  fanoutUnicastOptions,
3341
3481
  });
3342
3482
  })());
@@ -3346,6 +3486,7 @@ let SharedLog = (() => {
3346
3486
  this.rpc
3347
3487
  .send(message, {
3348
3488
  mode: new SilentDelivery({ redundancy: 1, to: silentTo }),
3489
+ priority: delivery.priority,
3349
3490
  })
3350
3491
  .catch((error) => logger.error(error));
3351
3492
  }
@@ -3611,6 +3752,134 @@ let SharedLog = (() => {
3611
3752
  await Promise.all([...callbacks]);
3612
3753
  }
3613
3754
  }
3755
+ isPeerReceiveAdmissionOpen(peerHash, replicationLifecycleController, subscriptionEpoch, options) {
3756
+ return (this.isReplicationLifecycleActive(replicationLifecycleController) &&
3757
+ this.isCurrentSubscriptionEpoch(peerHash, subscriptionEpoch) &&
3758
+ (options?.allowReplicationInfoBlocked === true ||
3759
+ !this._replicationInfoBlockedPeers.has(peerHash)) &&
3760
+ (options?.allowCleanupGate === true ||
3761
+ (this._receiveCleanupGateByPeer.get(peerHash) ?? 0) === 0));
3762
+ }
3763
+ acquirePeerReceiveLease(peerHash, replicationLifecycleController, subscriptionEpoch, options) {
3764
+ if (!this.isPeerReceiveAdmissionOpen(peerHash, replicationLifecycleController, subscriptionEpoch, options)) {
3765
+ return;
3766
+ }
3767
+ let state = this._activeReceiveHandlersByPeer.get(peerHash);
3768
+ if (!state) {
3769
+ const current = { active: 0 };
3770
+ state = { current, activeBuckets: new Set() };
3771
+ this._activeReceiveHandlersByPeer.set(peerHash, state);
3772
+ }
3773
+ const bucket = state.current;
3774
+ bucket.active += 1;
3775
+ state.activeBuckets.add(bucket);
3776
+ let released = false;
3777
+ return () => {
3778
+ if (released) {
3779
+ return;
3780
+ }
3781
+ released = true;
3782
+ bucket.active -= 1;
3783
+ if (bucket.active > 0) {
3784
+ return;
3785
+ }
3786
+ state.activeBuckets.delete(bucket);
3787
+ bucket.drain?.resolve();
3788
+ if (state.activeBuckets.size === 0 &&
3789
+ state.current.active === 0 &&
3790
+ this._activeReceiveHandlersByPeer.get(peerHash) === state) {
3791
+ this._activeReceiveHandlersByPeer.delete(peerHash);
3792
+ }
3793
+ };
3794
+ }
3795
+ async drainPeerReceiveHandlers(peerHash) {
3796
+ const state = this._activeReceiveHandlersByPeer.get(peerHash);
3797
+ if (!state || state.activeBuckets.size === 0) {
3798
+ return;
3799
+ }
3800
+ // Rotate before awaiting so a reconnect/opening generation can keep receiving
3801
+ // sync traffic without joining the drain for the previous subscription. Cleanup
3802
+ // callers gate admission first; terminal callers also repeat until empty.
3803
+ const buckets = [...state.activeBuckets];
3804
+ state.current = { active: 0 };
3805
+ const drain = Promise.all(buckets.map((bucket) => {
3806
+ bucket.drain ??= pDefer();
3807
+ return bucket.drain.promise;
3808
+ })).then(() => undefined);
3809
+ let drains = this._receiveHandlerDrainByPeer.get(peerHash);
3810
+ if (!drains) {
3811
+ drains = new Set();
3812
+ this._receiveHandlerDrainByPeer.set(peerHash, drains);
3813
+ }
3814
+ drains.add(drain);
3815
+ try {
3816
+ await drain;
3817
+ }
3818
+ finally {
3819
+ drains.delete(drain);
3820
+ if (drains.size === 0) {
3821
+ this._receiveHandlerDrainByPeer.delete(peerHash);
3822
+ }
3823
+ }
3824
+ }
3825
+ async drainReceiveHandlers() {
3826
+ for (;;) {
3827
+ const peers = [...this._activeReceiveHandlersByPeer.keys()];
3828
+ if (peers.length === 0) {
3829
+ return;
3830
+ }
3831
+ await Promise.all(peers.map((peerHash) => this.drainPeerReceiveHandlers(peerHash)));
3832
+ }
3833
+ }
3834
+ runPendingIHaveCallback(pending, entry) {
3835
+ const replicationLifecycleController = this._replicationLifecycleController;
3836
+ if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
3837
+ if (this._pendingIHave.get(entry.hash) === pending) {
3838
+ pending.clear();
3839
+ this._pendingIHave.delete(entry.hash);
3840
+ }
3841
+ return;
3842
+ }
3843
+ // Register before invoking the callback so a synchronous terminal reentry
3844
+ // cannot make close/drop miss work that has already been admitted.
3845
+ const completion = pDefer();
3846
+ const observed = completion.promise.catch((error) => {
3847
+ if (!(this.isTerminating() && isNotStartedError(error))) {
3848
+ logger.error(error?.toString?.() ?? String(error));
3849
+ }
3850
+ });
3851
+ this._pendingIHaveCallbacks.add(observed);
3852
+ void observed.finally(() => {
3853
+ this._pendingIHaveCallbacks.delete(observed);
3854
+ if (this._pendingIHave.get(entry.hash) === pending) {
3855
+ pending.clear();
3856
+ this._pendingIHave.delete(entry.hash);
3857
+ }
3858
+ });
3859
+ try {
3860
+ Promise.resolve(pending.callback(entry)).then(() => completion.resolve(), (error) => completion.reject(error));
3861
+ }
3862
+ catch (error) {
3863
+ completion.reject(error);
3864
+ }
3865
+ }
3866
+ async drainPendingIHaveCallbacks() {
3867
+ while (this._pendingIHaveCallbacks.size > 0) {
3868
+ await Promise.all([...this._pendingIHaveCallbacks]);
3869
+ }
3870
+ }
3871
+ blockPeerReceiveAdmission(peerHash) {
3872
+ this._receiveCleanupGateByPeer.set(peerHash, (this._receiveCleanupGateByPeer.get(peerHash) ?? 0) + 1);
3873
+ }
3874
+ unblockPeerReceiveAdmission(peerHash) {
3875
+ const remaining = (this._receiveCleanupGateByPeer.get(peerHash) ?? 1) - 1;
3876
+ if (remaining > 0) {
3877
+ this._receiveCleanupGateByPeer.set(peerHash, remaining);
3878
+ }
3879
+ else {
3880
+ this._receiveCleanupGateByPeer.delete(peerHash);
3881
+ }
3882
+ }
3614
3883
  handleReplicationLifecycleSendError(error, controller = this._replicationLifecycleController) {
3615
3884
  if ((controller?.signal.aborted ||
3616
3885
  !this.isReplicationLifecycleActive(controller)) &&
@@ -3689,11 +3958,232 @@ let SharedLog = (() => {
3689
3958
  },
3690
3959
  });
3691
3960
  }
3961
+ setupReplicationAnnouncementRepairFunction(interval = REPLICATION_ANNOUNCEMENT_REPAIR_INTERVAL, maxAttempts = REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS) {
3962
+ if (!Number.isSafeInteger(maxAttempts) || maxAttempts <= 0) {
3963
+ throw new RangeError("Replication announcement repair attempts must be positive");
3964
+ }
3965
+ this.replicationAnnouncementRepairDebounced?.close();
3966
+ this._replicationAnnouncementRepairController?.abort();
3967
+ this._replicationAnnouncementRepairGenerationController?.abort();
3968
+ this._replicationAnnouncementRepairController = new AbortController();
3969
+ this._replicationAnnouncementRepairGenerationController =
3970
+ new AbortController();
3971
+ this._replicationAnnouncementRepairPending = false;
3972
+ this._replicationAnnouncementRepairGeneration =
3973
+ this._replicationAnnouncementRetryGeneration;
3974
+ this._replicationAnnouncementRepairTargets = new Map();
3975
+ this._replicationAnnouncementRepairCohortSelected = false;
3976
+ this._replicationAnnouncementRepairFairCursorHash = undefined;
3977
+ this._replicationAnnouncementRepairMaxAttempts = maxAttempts;
3978
+ this.replicationAnnouncementRepairDebounced = debounceFixedInterval(() => this.runCurrentReplicationStateAnnouncementRepair(), interval, {
3979
+ leading: false,
3980
+ // The wrapper catches worker failures while it still owns the generation
3981
+ // context. Keep this boundary visibility-only: it must never mutate a
3982
+ // possibly newer generation's pending state.
3983
+ onError: (error) => logger.error(error),
3984
+ });
3985
+ }
3986
+ cancelCurrentReplicationStateAnnouncementRepair() {
3987
+ this._replicationAnnouncementRepairPending = false;
3988
+ this._replicationAnnouncementRepairController?.abort();
3989
+ this._replicationAnnouncementRepairGenerationController?.abort();
3990
+ this.replicationAnnouncementRepairDebounced?.close();
3991
+ this._replicationAnnouncementRepairTargets?.clear();
3992
+ }
3993
+ advanceCurrentReplicationStateAnnouncementRepairGeneration() {
3994
+ const generation = this._replicationAnnouncementRetryGeneration;
3995
+ if (generation === this._replicationAnnouncementRepairGeneration) {
3996
+ return;
3997
+ }
3998
+ // Abort acknowledged sends carrying the old full-state snapshot before the
3999
+ // primary announcement for the new mutation waits on transport. Otherwise a
4000
+ // stale batch can hold the current state behind DirectStream's seek timeout.
4001
+ this._replicationAnnouncementRepairGenerationController?.abort();
4002
+ this._replicationAnnouncementRepairGenerationController =
4003
+ new AbortController();
4004
+ this._replicationAnnouncementRepairGeneration = generation;
4005
+ this._replicationAnnouncementRepairPending = false;
4006
+ this._replicationAnnouncementRepairTargets.clear();
4007
+ this._replicationAnnouncementRepairCohortSelected = false;
4008
+ }
4009
+ queueCurrentReplicationStateAnnouncementRepair() {
4010
+ if (this.closed ||
4011
+ this._closeController.signal.aborted ||
4012
+ this._replicationAnnouncementRepairController.signal.aborted ||
4013
+ !this.replicationAnnouncementRepairDebounced) {
4014
+ return;
4015
+ }
4016
+ this.advanceCurrentReplicationStateAnnouncementRepairGeneration();
4017
+ this._replicationAnnouncementRepairPending = true;
4018
+ void this.replicationAnnouncementRepairDebounced.call();
4019
+ }
4020
+ async runCurrentReplicationStateAnnouncementRepair() {
4021
+ const generation = this._replicationAnnouncementRetryGeneration;
4022
+ const lifecycleController = this._replicationAnnouncementRepairController;
4023
+ const generationController = this._replicationAnnouncementRepairGenerationController;
4024
+ try {
4025
+ await this.repairCurrentReplicationStateAnnouncement({
4026
+ generation,
4027
+ lifecycleController,
4028
+ generationController,
4029
+ });
4030
+ }
4031
+ catch (error) {
4032
+ if (this.closed ||
4033
+ this._closeController.signal.aborted ||
4034
+ lifecycleController.signal.aborted ||
4035
+ generationController.signal.aborted ||
4036
+ generation !== this._replicationAnnouncementRetryGeneration ||
4037
+ generationController !==
4038
+ this._replicationAnnouncementRepairGenerationController) {
4039
+ return;
4040
+ }
4041
+ if (isNotStartedError(error)) {
4042
+ return;
4043
+ }
4044
+ // Only the worker that still owns the current generation may conclude
4045
+ // that its repair failed. A stale worker must not clear a newer call's
4046
+ // pending flag or attribute its error to the new generation.
4047
+ this._replicationAnnouncementRepairPending = false;
4048
+ logger.error(error);
4049
+ }
4050
+ }
4051
+ async repairCurrentReplicationStateAnnouncement(context) {
4052
+ if (!this._replicationAnnouncementRepairPending) {
4053
+ return;
4054
+ }
4055
+ const generation = context?.generation ?? this._replicationAnnouncementRetryGeneration;
4056
+ const lifecycleController = context?.lifecycleController ??
4057
+ this._replicationAnnouncementRepairController;
4058
+ const generationController = context?.generationController ??
4059
+ this._replicationAnnouncementRepairGenerationController;
4060
+ const segments = (await this.getMyReplicationSegments()).map((range) => range.toReplicationRange());
4061
+ if (this.closed ||
4062
+ this._closeController.signal.aborted ||
4063
+ lifecycleController.signal.aborted ||
4064
+ generationController.signal.aborted) {
4065
+ return;
4066
+ }
4067
+ if (generation !== this._replicationAnnouncementRetryGeneration) {
4068
+ this.queueCurrentReplicationStateAnnouncementRepair();
4069
+ return;
4070
+ }
4071
+ const subscribers = (await this.node.services.pubsub.getSubscribers(this.topic)) ?? [];
4072
+ if (this.closed ||
4073
+ this._closeController.signal.aborted ||
4074
+ lifecycleController.signal.aborted ||
4075
+ generationController.signal.aborted) {
4076
+ return;
4077
+ }
4078
+ if (generation !== this._replicationAnnouncementRetryGeneration) {
4079
+ this.queueCurrentReplicationStateAnnouncementRepair();
4080
+ return;
4081
+ }
4082
+ const selfHash = this.node.identity.publicKey.hashcode();
4083
+ const currentTargets = new Map();
4084
+ for (const key of subscribers) {
4085
+ const hash = key.hashcode();
4086
+ if (hash !== selfHash &&
4087
+ !this._replicationInfoBlockedPeers.has(hash) &&
4088
+ !currentTargets.has(hash)) {
4089
+ currentTargets.set(hash, key);
4090
+ }
4091
+ }
4092
+ for (const [hash, target] of this._replicationAnnouncementRepairTargets) {
4093
+ if (target.generation !== generation || !currentTargets.has(hash)) {
4094
+ this._replicationAnnouncementRepairTargets.delete(hash);
4095
+ }
4096
+ else {
4097
+ target.key = currentTargets.get(hash);
4098
+ }
4099
+ }
4100
+ if (!this._replicationAnnouncementRepairCohortSelected) {
4101
+ const candidates = [...currentTargets.entries()].sort(([left], [right]) => left.localeCompare(right));
4102
+ const cursorIndex = this._replicationAnnouncementRepairFairCursorHash
4103
+ ? candidates.findIndex(([hash]) => hash.localeCompare(this._replicationAnnouncementRepairFairCursorHash) > 0)
4104
+ : 0;
4105
+ const fairStart = cursorIndex < 0 ? 0 : cursorIndex;
4106
+ const fairOrder = [
4107
+ ...candidates.slice(fairStart),
4108
+ ...candidates.slice(0, fairStart),
4109
+ ];
4110
+ const cohort = fairOrder.slice(0, REPLICATION_ANNOUNCEMENT_REPAIR_TARGETS_PER_GENERATION);
4111
+ for (const [hash, key] of cohort) {
4112
+ this._replicationAnnouncementRepairTargets.set(hash, {
4113
+ key,
4114
+ generation,
4115
+ attempts: 0,
4116
+ done: false,
4117
+ });
4118
+ }
4119
+ if (cohort.length > 0) {
4120
+ this._replicationAnnouncementRepairFairCursorHash =
4121
+ cohort[cohort.length - 1][0];
4122
+ }
4123
+ this._replicationAnnouncementRepairCohortSelected = true;
4124
+ }
4125
+ const batch = [
4126
+ ...this._replicationAnnouncementRepairTargets.entries(),
4127
+ ].filter(([, target]) => !target.done);
4128
+ const snapshot = new AllReplicatingSegmentsMessage({ segments });
4129
+ const results = await Promise.allSettled(batch.map(([, target]) => this.rpc.send(snapshot, {
4130
+ mode: new AcknowledgeDelivery({
4131
+ to: [target.key],
4132
+ redundancy: 1,
4133
+ }),
4134
+ priority: CONVERGENCE_MESSAGE_PRIORITY,
4135
+ signal: generationController.signal,
4136
+ })));
4137
+ if (this.closed ||
4138
+ this._closeController.signal.aborted ||
4139
+ lifecycleController.signal.aborted ||
4140
+ generationController.signal.aborted) {
4141
+ return;
4142
+ }
4143
+ if (generation !== this._replicationAnnouncementRetryGeneration) {
4144
+ this.queueCurrentReplicationStateAnnouncementRepair();
4145
+ return;
4146
+ }
4147
+ for (const [index, result] of results.entries()) {
4148
+ const [hash, attemptedTarget] = batch[index];
4149
+ const target = this._replicationAnnouncementRepairTargets.get(hash);
4150
+ if (target !== attemptedTarget || target.generation !== generation) {
4151
+ continue;
4152
+ }
4153
+ if (result.status === "fulfilled") {
4154
+ // DirectStream ACKs confirm that the signed transport envelope reached
4155
+ // the target. Applying the contained replication state remains a
4156
+ // receiver-local, best-effort operation.
4157
+ target.done = true;
4158
+ continue;
4159
+ }
4160
+ target.attempts += 1;
4161
+ if (!isTransientReplicationAnnouncementRepairError(result.reason)) {
4162
+ target.done = true;
4163
+ logger.error(result.reason);
4164
+ }
4165
+ else if (target.attempts >= this._replicationAnnouncementRepairMaxAttempts) {
4166
+ target.done = true;
4167
+ logger.trace("Acknowledged replication announcement repair exhausted for %s", hash);
4168
+ }
4169
+ }
4170
+ if (generation !== this._replicationAnnouncementRetryGeneration) {
4171
+ this.queueCurrentReplicationStateAnnouncementRepair();
4172
+ return;
4173
+ }
4174
+ if ([...this._replicationAnnouncementRepairTargets.values()].some((target) => !target.done)) {
4175
+ void this.replicationAnnouncementRepairDebounced?.call();
4176
+ return;
4177
+ }
4178
+ this._replicationAnnouncementRepairPending = false;
4179
+ this._replicationAnnouncementRepairTargets.clear();
4180
+ }
3692
4181
  cancelCurrentReplicationStateAnnouncementRetry() {
3693
4182
  this._replicationAnnouncementRetryGeneration += 1;
3694
4183
  this._replicationAnnouncementRetryPending = false;
3695
4184
  this._replicationAnnouncementRetryController?.abort();
3696
4185
  this.replicationAnnouncementRetryDebounced?.close();
4186
+ this.cancelCurrentReplicationStateAnnouncementRepair();
3697
4187
  }
3698
4188
  async sendReplicationAnnouncement(message) {
3699
4189
  // Advance before every post-mutation send, including successful ones. An
@@ -3701,10 +4191,12 @@ let SharedLog = (() => {
3701
4191
  // local state; the generation mismatch forces one more current snapshot
3702
4192
  // after that stale send settles.
3703
4193
  this._replicationAnnouncementRetryGeneration += 1;
4194
+ this.advanceCurrentReplicationStateAnnouncementRepairGeneration();
3704
4195
  try {
3705
4196
  await this.rpc.send(message, {
3706
4197
  priority: CONVERGENCE_MESSAGE_PRIORITY,
3707
4198
  });
4199
+ this.queueCurrentReplicationStateAnnouncementRepair();
3708
4200
  }
3709
4201
  catch (error) {
3710
4202
  // The local replication-index mutation precedes all calls to this
@@ -3733,6 +4225,7 @@ let SharedLog = (() => {
3733
4225
  priority: CONVERGENCE_MESSAGE_PRIORITY,
3734
4226
  signal: controller.signal,
3735
4227
  });
4228
+ this.queueCurrentReplicationStateAnnouncementRepair();
3736
4229
  }
3737
4230
  catch (error) {
3738
4231
  if (this.closed ||
@@ -4139,61 +4632,135 @@ let SharedLog = (() => {
4139
4632
  }
4140
4633
  async removeReplicator(key, options) {
4141
4634
  const keyHash = typeof key === "string" ? key : key.hashcode();
4142
- const deleted = await this.replicationIndex
4143
- .iterate({
4144
- query: { hash: keyHash },
4145
- })
4146
- .all();
4147
- this.uniqueReplicators.delete(keyHash);
4148
- this._replicatorJoinEmitted.delete(keyHash);
4149
- await this.replicationIndex.del({ query: { hash: keyHash } });
4150
- for (const result of deleted) {
4151
- this.deleteNativeReplicationRange(result.value);
4152
- }
4153
- await this.updateOldestTimestampFromIndex();
4154
- const isMe = this.node.identity.publicKey.hashcode() === keyHash;
4155
- if (isMe) {
4156
- // announce that we are no longer replicating
4157
- await this.sendReplicationAnnouncement(new AllReplicatingSegmentsMessage({ segments: [] }));
4158
- }
4159
- if (options?.noEvent !== true) {
4160
- const publicKey = toLocalPublicSignKey(key);
4161
- if (publicKey) {
4162
- this.events.dispatchEvent(new CustomEvent("replication:change", {
4163
- detail: { publicKey },
4164
- }));
4635
+ const expectedJoinWarmupGeneration = options?.expectedJoinWarmupGeneration !== undefined
4636
+ ? options.expectedJoinWarmupGeneration
4637
+ : (this._joinWarmupGenerationByTarget.get(keyHash) ?? null);
4638
+ const ownsSubscriptionEpoch = () => options?.subscriptionEpoch === undefined ||
4639
+ this.isCurrentSubscriptionEpoch(keyHash, options.subscriptionEpoch);
4640
+ const ownsReplicationLifecycle = () => options?.replicationLifecycleController === undefined ||
4641
+ this.isReplicationLifecycleActive(options.replicationLifecycleController);
4642
+ const cancelExpectedJoinWarmupTarget = () => {
4643
+ if (expectedJoinWarmupGeneration !== null &&
4644
+ this._joinWarmupGenerationByTarget.get(keyHash) ===
4645
+ expectedJoinWarmupGeneration) {
4646
+ this.cancelJoinWarmupTarget(keyHash);
4165
4647
  }
4166
- else {
4167
- throw new Error("Key was not a PublicSignKey");
4648
+ };
4649
+ const isMe = this.node.identity.publicKey.hashcode() === keyHash;
4650
+ let receiveAdmissionBlocked = false;
4651
+ const blockAndDrainPeerReceives = async () => {
4652
+ if (!receiveAdmissionBlocked) {
4653
+ this.blockPeerReceiveAdmission(keyHash);
4654
+ receiveAdmissionBlocked = true;
4168
4655
  }
4169
- }
4170
- const timestamp = BigInt(+new Date());
4171
- for (const x of deleted) {
4172
- this.replicationChangeDebounceFn.add({
4173
- range: x.value,
4174
- type: "removed",
4175
- timestamp,
4656
+ await this.drainPeerReceiveHandlers(keyHash);
4657
+ };
4658
+ const cleanupDisconnectedPeer = async () => {
4659
+ await blockAndDrainPeerReceives();
4660
+ this.removePeerFromGidPeerHistory(keyHash);
4661
+ this.cleanupPeerDisconnectTracking(keyHash);
4662
+ this.removeRepairFrontierTarget(keyHash, {
4663
+ expectedJoinWarmupGeneration,
4176
4664
  });
4177
- }
4178
- const pendingMaturity = this.pendingMaturity.get(keyHash);
4179
- if (pendingMaturity) {
4180
- for (const [_k, v] of pendingMaturity) {
4181
- clearTimeout(v.timeout);
4665
+ this._recentRepairDispatch.delete(keyHash);
4666
+ if (!isMe) {
4667
+ await this.syncronizer.onPeerDisconnected(keyHash);
4182
4668
  }
4183
- this.pendingMaturity.delete(keyHash);
4184
- }
4185
- // Keep local sync/prune state consistent even when a peer disappears
4186
- // through replication-info updates without a topic unsubscribe event.
4187
- this.removePeerFromGidPeerHistory(keyHash);
4188
- this._checkedPrune.cleanupPeer(keyHash);
4189
- this.removeRepairFrontierTarget(keyHash);
4190
- this._recentRepairDispatch.delete(keyHash);
4191
- if (!isMe) {
4192
- this.syncronizer.onPeerDisconnected(keyHash);
4193
- }
4194
- if (!isMe) {
4195
- this.rebalanceParticipationDebounced?.call();
4196
- }
4669
+ };
4670
+ let removed = false;
4671
+ // Replication-info updates already serialize per peer. Put the hash-wide
4672
+ // removal on the same queue so a newer reset cannot be deleted underneath
4673
+ // itself by an older unsubscribe callback.
4674
+ await this.withReplicationInfoApplyQueue(keyHash, async () => {
4675
+ if (!ownsReplicationLifecycle()) {
4676
+ return;
4677
+ }
4678
+ if (!ownsSubscriptionEpoch()) {
4679
+ // A reconnect may supersede an unsubscribe before its destructive
4680
+ // removal starts. Still retire the old connection's sync/request state
4681
+ // in lane order so the reconnect barrier cannot inherit stale caches.
4682
+ if (options?.cleanupIfSubscriptionSuperseded) {
4683
+ await cleanupDisconnectedPeer();
4684
+ }
4685
+ return;
4686
+ }
4687
+ if (options?.shouldRemove && !options.shouldRemove()) {
4688
+ return;
4689
+ }
4690
+ const wasReplicator = this.uniqueReplicators.has(keyHash);
4691
+ const deleted = await this.replicationIndex
4692
+ .iterate({
4693
+ query: { hash: keyHash },
4694
+ })
4695
+ .all();
4696
+ // Liveness evidence can arrive without a subscription transition while
4697
+ // this scan is pending. Stop new receives, drain those already admitted,
4698
+ // then revalidate immediately before the first destructive mutation.
4699
+ await blockAndDrainPeerReceives();
4700
+ if (options?.shouldRemove && !options.shouldRemove()) {
4701
+ return;
4702
+ }
4703
+ // Liveness removal must not cancel a still-current warmup until fresh
4704
+ // activity has been rechecked at the destructive boundary.
4705
+ cancelExpectedJoinWarmupTarget();
4706
+ await this.replicationIndex.del({ query: { hash: keyHash } });
4707
+ for (const result of deleted) {
4708
+ this.deleteNativeReplicationRange(result.value);
4709
+ }
4710
+ // Once the range indexes have removed the peer, make membership agree
4711
+ // before any later awaited bookkeeping can fail.
4712
+ this.uniqueReplicators.delete(keyHash);
4713
+ this._replicatorJoinEmitted.delete(keyHash);
4714
+ await this.updateOldestTimestampFromIndex();
4715
+ if (isMe) {
4716
+ // announce that we are no longer replicating
4717
+ await this.sendReplicationAnnouncement(new AllReplicatingSegmentsMessage({ segments: [] }));
4718
+ }
4719
+ if (options?.noEvent !== true) {
4720
+ const publicKey = toLocalPublicSignKey(key);
4721
+ if (publicKey) {
4722
+ this.events.dispatchEvent(new CustomEvent("replication:change", {
4723
+ detail: { publicKey },
4724
+ }));
4725
+ }
4726
+ else {
4727
+ throw new Error("Key was not a PublicSignKey");
4728
+ }
4729
+ }
4730
+ const timestamp = BigInt(+new Date());
4731
+ for (const x of deleted) {
4732
+ this.replicationChangeDebounceFn.add({
4733
+ range: x.value,
4734
+ type: "removed",
4735
+ timestamp,
4736
+ });
4737
+ }
4738
+ const pendingMaturity = this.pendingMaturity.get(keyHash);
4739
+ if (pendingMaturity) {
4740
+ for (const [_k, v] of pendingMaturity) {
4741
+ clearTimeout(v.timeout);
4742
+ }
4743
+ this.pendingMaturity.delete(keyHash);
4744
+ }
4745
+ // Keep local sync/prune state consistent even when a peer disappears
4746
+ // through replication-info updates without a topic unsubscribe event.
4747
+ await cleanupDisconnectedPeer();
4748
+ if (!isMe) {
4749
+ // Replication-info handlers release their receive lease before joining
4750
+ // this lane. Fence every handler queued behind this successful removal,
4751
+ // regardless of whether it came from liveness, startup pruning, or an
4752
+ // unsubscribe transition.
4753
+ this.advanceReplicationInfoRecoveryEpoch(keyHash);
4754
+ this.rebalanceParticipationDebounced?.call();
4755
+ }
4756
+ removed = true;
4757
+ options?.onRemoved?.({ wasReplicator });
4758
+ }).finally(() => {
4759
+ if (receiveAdmissionBlocked) {
4760
+ this.unblockPeerReceiveAdmission(keyHash);
4761
+ }
4762
+ });
4763
+ return removed;
4197
4764
  }
4198
4765
  async updateOldestTimestampFromIndex() {
4199
4766
  const iterator = await this.replicationIndex.iterate({
@@ -4257,6 +4824,10 @@ let SharedLog = (() => {
4257
4824
  if (this._isTrustedReplicator && !(await this._isTrustedReplicator(from))) {
4258
4825
  return undefined;
4259
4826
  }
4827
+ // Preserve what the peer announced before duplicate filtering can empty the
4828
+ // working array. A repeated authoritative/non-empty announcement is still
4829
+ // proof of live membership after this process reopens.
4830
+ const announcedReplication = ranges.length > 0;
4260
4831
  let isNewReplicator = false;
4261
4832
  let timestamp = BigInt(ts ?? +new Date());
4262
4833
  rebalance = rebalance == null ? true : rebalance;
@@ -4376,14 +4947,14 @@ let SharedLog = (() => {
4376
4947
  // means "nothing new", not "stopped". Removing the peer here would let
4377
4948
  // uniqueReplicators diverge from replicationIndex while the peer's
4378
4949
  // ranges stay indexed.
4379
- const announcedStopped = reset === true && ranges.length === 0;
4950
+ const announcedStopped = reset === true && !announcedReplication;
4380
4951
  const stoppedTransition = announcedStopped
4381
4952
  ? this.uniqueReplicators.delete(fromHash)
4382
4953
  : false;
4383
4954
  if (announcedStopped) {
4384
4955
  this._replicatorJoinEmitted.delete(fromHash);
4385
4956
  }
4386
- else if (ranges.length > 0) {
4957
+ else if (announcedReplication) {
4387
4958
  this.uniqueReplicators.add(fromHash);
4388
4959
  }
4389
4960
  let now = +new Date();
@@ -4488,6 +5059,14 @@ let SharedLog = (() => {
4488
5059
  this.rebalanceParticipationDebounced?.call();
4489
5060
  }
4490
5061
  }
5062
+ if (announcedReplication &&
5063
+ !from.equals(this.node.identity.publicKey) &&
5064
+ !this._replicatorJoinEmitted.has(fromHash)) {
5065
+ this._replicatorJoinEmitted.add(fromHash);
5066
+ this.events.dispatchEvent(new CustomEvent("replicator:join", {
5067
+ detail: { publicKey: from },
5068
+ }));
5069
+ }
4491
5070
  return diffs;
4492
5071
  }
4493
5072
  async startAnnounceReplicating(range, options = {}) {
@@ -4679,16 +5258,40 @@ let SharedLog = (() => {
4679
5258
  const observedAt = this._entryKnownPeerObservedAt.get(hash)?.get(peer);
4680
5259
  return observedAt != null && Date.now() - observedAt <= maxAgeMs;
4681
5260
  }
4682
- markRepairSweepOptimisticPeer(gid, peer) {
5261
+ markRepairSweepOptimisticPeer(gid, peer, generation) {
4683
5262
  let peers = this._repairSweepOptimisticGidPeersPending.get(gid);
4684
5263
  if (!peers) {
4685
5264
  peers = new Map();
4686
5265
  this._repairSweepOptimisticGidPeersPending.set(gid, peers);
4687
5266
  }
4688
- peers.set(peer, (peers.get(peer) || 0) + 1);
5267
+ const current = peers.get(peer);
5268
+ peers.set(peer, {
5269
+ count: current?.generation === generation ? current.count + 1 : 1,
5270
+ generation,
5271
+ });
5272
+ let gids = this._repairSweepOptimisticGidsByPeer.get(peer);
5273
+ if (!gids) {
5274
+ gids = new Set();
5275
+ this._repairSweepOptimisticGidsByPeer.set(peer, gids);
5276
+ }
5277
+ gids.add(gid);
4689
5278
  }
4690
5279
  hasPendingRepairSweepOptimisticPeer(gid, peer) {
4691
- return ((this._repairSweepOptimisticGidPeersPending.get(gid)?.get(peer) || 0) > 0);
5280
+ return ((this._repairSweepOptimisticGidPeersPending.get(gid)?.get(peer)?.count ||
5281
+ 0) > 0);
5282
+ }
5283
+ clearRepairSweepOptimisticPeer(peer) {
5284
+ for (const gid of this._repairSweepOptimisticGidsByPeer.get(peer) ?? []) {
5285
+ const peers = this._repairSweepOptimisticGidPeersPending.get(gid);
5286
+ if (!peers) {
5287
+ continue;
5288
+ }
5289
+ peers.delete(peer);
5290
+ if (peers.size === 0) {
5291
+ this._repairSweepOptimisticGidPeersPending.delete(gid);
5292
+ }
5293
+ }
5294
+ this._repairSweepOptimisticGidsByPeer.delete(peer);
4692
5295
  }
4693
5296
  createEntryReplicatedForRepair(properties) {
4694
5297
  const assignedToRangeBoundary = shouldAssignToRangeBoundary(properties.leaders, properties.replicas);
@@ -4757,14 +5360,221 @@ let SharedLog = (() => {
4757
5360
  if (!pending) {
4758
5361
  continue;
4759
5362
  }
4760
- for (const hash of hashList) {
4761
- pending.delete(hash);
5363
+ for (const hash of hashList) {
5364
+ pending.delete(hash);
5365
+ }
5366
+ if (pending.size === 0) {
5367
+ this._repairFrontierByMode.get(mode)?.delete(target);
5368
+ this._repairFrontierBypassKnownPeersByMode.get(mode)?.delete(target);
5369
+ }
5370
+ }
5371
+ }
5372
+ getJoinWarmupGeneration(target) {
5373
+ let generation = this._joinWarmupGenerationByTarget.get(target);
5374
+ if (!generation) {
5375
+ generation = {};
5376
+ this._joinWarmupGenerationByTarget.set(target, generation);
5377
+ }
5378
+ return generation;
5379
+ }
5380
+ trackJoinWarmupTimer(target, timer) {
5381
+ let timers = this._joinWarmupRetryTimersByTarget.get(target);
5382
+ if (!timers) {
5383
+ timers = new Set();
5384
+ this._joinWarmupRetryTimersByTarget.set(target, timers);
5385
+ }
5386
+ timers.add(timer);
5387
+ this._repairRetryTimers.add(timer.handle);
5388
+ }
5389
+ untrackJoinWarmupTimer(target, timer) {
5390
+ this._repairRetryTimers.delete(timer.handle);
5391
+ const timers = this._joinWarmupRetryTimersByTarget.get(target);
5392
+ if (!timers) {
5393
+ return;
5394
+ }
5395
+ timers.delete(timer);
5396
+ if (timers.size === 0) {
5397
+ this._joinWarmupRetryTimersByTarget.delete(target);
5398
+ }
5399
+ }
5400
+ cancelJoinWarmupTimers(target) {
5401
+ const timers = this._joinWarmupRetryTimersByTarget.get(target);
5402
+ if (!timers) {
5403
+ return;
5404
+ }
5405
+ for (const timer of [...timers]) {
5406
+ clearTimeout(timer.handle);
5407
+ timer.resolve?.();
5408
+ this.untrackJoinWarmupTimer(target, timer);
5409
+ }
5410
+ }
5411
+ cancelJoinWarmupTarget(target) {
5412
+ this._joinWarmupGenerationByTarget.delete(target);
5413
+ const pendingWarmupPeers = this._repairSweepPendingPeersByMode.get("join-warmup");
5414
+ pendingWarmupPeers?.delete(target);
5415
+ if (pendingWarmupPeers?.size === 0) {
5416
+ this._repairSweepPendingModes.delete("join-warmup");
5417
+ }
5418
+ this._repairSweepJoinWarmupGenerationByTarget.delete(target);
5419
+ this.clearRepairSweepOptimisticPeer(target);
5420
+ this.cancelJoinWarmupTimers(target);
5421
+ this._joinWarmupScheduledRetriesByTarget.delete(target);
5422
+ const state = this._joinWarmupSendStateByTarget.get(target);
5423
+ if (!state) {
5424
+ return;
5425
+ }
5426
+ state.bypassKnownPeerHints = false;
5427
+ state.entries.clear();
5428
+ state.pending = false;
5429
+ if (!state.running) {
5430
+ this._joinWarmupSendStateByTarget.delete(target);
5431
+ }
5432
+ }
5433
+ cancelAllJoinWarmupTargets() {
5434
+ const targets = new Set([
5435
+ ...this._joinWarmupGenerationByTarget.keys(),
5436
+ ...this._joinWarmupRetryTimersByTarget.keys(),
5437
+ ...this._joinWarmupScheduledRetriesByTarget.keys(),
5438
+ ...this._joinWarmupSendStateByTarget.keys(),
5439
+ ]);
5440
+ for (const target of targets) {
5441
+ this.cancelJoinWarmupTarget(target);
5442
+ }
5443
+ }
5444
+ async sleepJoinWarmupTracked(target, delayMs) {
5445
+ if (delayMs <= 0) {
5446
+ return;
5447
+ }
5448
+ await new Promise((resolve) => {
5449
+ let settled = false;
5450
+ let trackedTimer;
5451
+ const settle = () => {
5452
+ if (settled) {
5453
+ return;
5454
+ }
5455
+ settled = true;
5456
+ this.untrackJoinWarmupTimer(target, trackedTimer);
5457
+ resolve();
5458
+ };
5459
+ const handle = setTimeout(settle, delayMs);
5460
+ handle.unref?.();
5461
+ trackedTimer = { handle, resolve: settle };
5462
+ this.trackJoinWarmupTimer(target, trackedTimer);
5463
+ });
5464
+ }
5465
+ scheduleJoinWarmupRetries(target, generation, delaysMs, entries, bypassKnownPeerHints) {
5466
+ if (this.closed ||
5467
+ this._joinWarmupGenerationByTarget.get(target) !== generation) {
5468
+ return;
5469
+ }
5470
+ let scheduled = this._joinWarmupScheduledRetriesByTarget.get(target);
5471
+ if (scheduled?.generation !== generation) {
5472
+ this.cancelJoinWarmupTimers(target);
5473
+ this._joinWarmupScheduledRetriesByTarget.delete(target);
5474
+ scheduled = undefined;
5475
+ }
5476
+ if (!scheduled) {
5477
+ scheduled = {
5478
+ generation,
5479
+ slotsByDelay: new Map(),
5480
+ };
5481
+ this._joinWarmupScheduledRetriesByTarget.set(target, scheduled);
5482
+ }
5483
+ const delays = [...new Set(delaysMs)];
5484
+ const batch = {
5485
+ bypassKnownPeerHints,
5486
+ entries: new Map(entries),
5487
+ remainingAttempts: delays.length,
5488
+ };
5489
+ const now = Date.now();
5490
+ for (const delayMs of delays) {
5491
+ let slot = scheduled.slotsByDelay.get(delayMs);
5492
+ if (!slot) {
5493
+ slot = { cohorts: [], head: 0 };
5494
+ scheduled.slotsByDelay.set(delayMs, slot);
5495
+ }
5496
+ const tail = slot.cohorts.at(-1);
5497
+ const dueAt = Math.max(tail?.dueAt ?? 0, now + delayMs);
5498
+ if (tail?.dueAt === dueAt) {
5499
+ tail.batches.push(batch);
5500
+ }
5501
+ else {
5502
+ slot.cohorts.push({
5503
+ batches: [batch],
5504
+ dueAt,
5505
+ });
5506
+ }
5507
+ this.armJoinWarmupRetrySlot(target, scheduled, delayMs, slot);
5508
+ }
5509
+ }
5510
+ armJoinWarmupRetrySlot(target, scheduled, delayMs, slot) {
5511
+ const nextDueAt = slot.cohorts[slot.head]?.dueAt;
5512
+ if (nextDueAt == null) {
5513
+ return;
5514
+ }
5515
+ if (slot.timer && slot.timerDueAt === nextDueAt) {
5516
+ return;
5517
+ }
5518
+ if (slot.timer) {
5519
+ clearTimeout(slot.timer.handle);
5520
+ this.untrackJoinWarmupTimer(target, slot.timer);
5521
+ }
5522
+ let trackedTimer;
5523
+ const handle = setTimeout(() => {
5524
+ this.untrackJoinWarmupTimer(target, trackedTimer);
5525
+ if (slot.timer !== trackedTimer) {
5526
+ return;
5527
+ }
5528
+ slot.timer = undefined;
5529
+ slot.timerDueAt = undefined;
5530
+ const current = this._joinWarmupScheduledRetriesByTarget.get(target);
5531
+ if (current !== scheduled ||
5532
+ current.slotsByDelay.get(delayMs) !== slot) {
5533
+ return;
5534
+ }
5535
+ const dueEntries = new Map();
5536
+ let bypassKnownPeerHints = false;
5537
+ const now = Date.now();
5538
+ while (slot.head < slot.cohorts.length &&
5539
+ slot.cohorts[slot.head].dueAt <= now) {
5540
+ const cohort = slot.cohorts[slot.head++];
5541
+ for (const batch of cohort.batches) {
5542
+ for (const [hash, entry] of batch.entries) {
5543
+ dueEntries.set(hash, entry);
5544
+ }
5545
+ bypassKnownPeerHints ||=
5546
+ batch.bypassKnownPeerHints;
5547
+ batch.remainingAttempts -= 1;
5548
+ if (batch.remainingAttempts === 0) {
5549
+ batch.entries.clear();
5550
+ }
5551
+ }
5552
+ cohort.batches.length = 0;
5553
+ }
5554
+ if (dueEntries.size > 0 &&
5555
+ !this.closed &&
5556
+ this._joinWarmupGenerationByTarget.get(target) ===
5557
+ scheduled.generation) {
5558
+ this.queueJoinWarmupSend(target, scheduled.generation, dueEntries, bypassKnownPeerHints);
4762
5559
  }
4763
- if (pending.size === 0) {
4764
- this._repairFrontierByMode.get(mode)?.delete(target);
4765
- this._repairFrontierBypassKnownPeersByMode.get(mode)?.delete(target);
5560
+ if (slot.head === slot.cohorts.length) {
5561
+ current.slotsByDelay.delete(delayMs);
5562
+ if (current.slotsByDelay.size === 0) {
5563
+ this._joinWarmupScheduledRetriesByTarget.delete(target);
5564
+ }
5565
+ return;
4766
5566
  }
4767
- }
5567
+ if (slot.head >= 1_024 && slot.head * 2 >= slot.cohorts.length) {
5568
+ slot.cohorts = slot.cohorts.slice(slot.head);
5569
+ slot.head = 0;
5570
+ }
5571
+ this.armJoinWarmupRetrySlot(target, current, delayMs, slot);
5572
+ }, Math.max(0, nextDueAt - Date.now()));
5573
+ handle.unref?.();
5574
+ trackedTimer = { handle };
5575
+ slot.timer = trackedTimer;
5576
+ slot.timerDueAt = nextDueAt;
5577
+ this.trackJoinWarmupTimer(target, trackedTimer);
4768
5578
  }
4769
5579
  async getFullReplicaRepairCandidates(extraPeers, options) {
4770
5580
  const candidates = new Set([
@@ -4795,7 +5605,13 @@ let SharedLog = (() => {
4795
5605
  }
4796
5606
  return candidates;
4797
5607
  }
4798
- removeRepairFrontierTarget(target) {
5608
+ removeRepairFrontierTarget(target, options) {
5609
+ if (options?.expectedJoinWarmupGeneration === undefined ||
5610
+ (options.expectedJoinWarmupGeneration !== null &&
5611
+ this._joinWarmupGenerationByTarget.get(target) ===
5612
+ options.expectedJoinWarmupGeneration)) {
5613
+ this.cancelJoinWarmupTarget(target);
5614
+ }
4799
5615
  for (const mode of REPAIR_DISPATCH_MODES) {
4800
5616
  this._repairFrontierByMode.get(mode)?.delete(target);
4801
5617
  this._repairFrontierActiveTargetsByMode.get(mode)?.delete(target);
@@ -4870,6 +5686,89 @@ let SharedLog = (() => {
4870
5686
  targets: [target],
4871
5687
  });
4872
5688
  }
5689
+ queueJoinWarmupSend(target, generation, entries, bypassKnownPeerHints) {
5690
+ if (this.closed ||
5691
+ this._joinWarmupGenerationByTarget.get(target) !== generation) {
5692
+ return;
5693
+ }
5694
+ let state = this._joinWarmupSendStateByTarget.get(target);
5695
+ if (!state) {
5696
+ state = {
5697
+ bypassKnownPeerHints: false,
5698
+ entries: new Map(),
5699
+ generation,
5700
+ lastCompletedAt: Number.NEGATIVE_INFINITY,
5701
+ pending: false,
5702
+ running: false,
5703
+ };
5704
+ this._joinWarmupSendStateByTarget.set(target, state);
5705
+ }
5706
+ else if (state.generation !== generation) {
5707
+ state.bypassKnownPeerHints = false;
5708
+ state.entries.clear();
5709
+ state.pending = false;
5710
+ }
5711
+ for (const [hash, entry] of entries) {
5712
+ state.entries.set(hash, entry);
5713
+ }
5714
+ state.bypassKnownPeerHints ||= bypassKnownPeerHints;
5715
+ state.generation = generation;
5716
+ state.pending = true;
5717
+ if (state.running) {
5718
+ return;
5719
+ }
5720
+ void this.drainJoinWarmupSends(target, state).catch((error) => logger.error(error));
5721
+ }
5722
+ async drainJoinWarmupSends(target, state) {
5723
+ if (state.running) {
5724
+ return;
5725
+ }
5726
+ state.running = true;
5727
+ try {
5728
+ while (state.pending) {
5729
+ state.pending = false;
5730
+ const generation = state.generation;
5731
+ const entries = new Map(state.entries);
5732
+ state.entries.clear();
5733
+ const bypassKnownPeerHints = state.bypassKnownPeerHints;
5734
+ state.bypassKnownPeerHints = false;
5735
+ const spacingMs = Math.max(0, state.lastCompletedAt + JOIN_WARMUP_SEND_SPACING_MS - Date.now());
5736
+ await this.sleepJoinWarmupTracked(target, spacingMs);
5737
+ if (this.closed ||
5738
+ state.generation !== generation ||
5739
+ this._joinWarmupGenerationByTarget.get(target) !== generation) {
5740
+ continue;
5741
+ }
5742
+ if (entries.size === 0) {
5743
+ continue;
5744
+ }
5745
+ this._repairMetrics["join-warmup"].simpleFallbackPasses += 1;
5746
+ try {
5747
+ await this.sendRepairEntriesWithTransport(target, entries, "simple", {
5748
+ bypassKnownPeers: bypassKnownPeerHints,
5749
+ bypassRecentKnownPeers: bypassKnownPeerHints,
5750
+ });
5751
+ }
5752
+ catch (error) {
5753
+ logger.error(error);
5754
+ }
5755
+ finally {
5756
+ state.lastCompletedAt = Date.now();
5757
+ }
5758
+ }
5759
+ }
5760
+ finally {
5761
+ state.running = false;
5762
+ if (this._joinWarmupSendStateByTarget.get(target) === state) {
5763
+ if (state.pending) {
5764
+ void this.drainJoinWarmupSends(target, state).catch((error) => logger.error(error));
5765
+ }
5766
+ else if (!this._joinWarmupGenerationByTarget.has(target)) {
5767
+ this._joinWarmupSendStateByTarget.delete(target);
5768
+ }
5769
+ }
5770
+ }
5771
+ }
4873
5772
  async sendMaybeMissingEntriesNow(target, entries, options) {
4874
5773
  if (entries.size === 0) {
4875
5774
  return;
@@ -5074,25 +5973,41 @@ let SharedLog = (() => {
5074
5973
  const bucket = this._repairMetrics[options.mode];
5075
5974
  bucket.dispatches += 1;
5076
5975
  bucket.entries += filteredEntries.size;
5976
+ const joinWarmupGeneration = options.mode === "join-warmup"
5977
+ ? this.getJoinWarmupGeneration(target)
5978
+ : undefined;
5979
+ const bypassKnownPeerHints = this.shouldBypassKnownPeerHints(options.mode, options.bypassKnownPeerHints);
5077
5980
  const run = (transport) => {
5078
- if (transport === "simple") {
5079
- bucket.simpleFallbackPasses += 1;
5981
+ if (transport === "simple" &&
5982
+ options.mode === "join-warmup" &&
5983
+ joinWarmupGeneration) {
5984
+ this.queueJoinWarmupSend(target, joinWarmupGeneration, filteredEntries, bypassKnownPeerHints);
5985
+ return;
5080
5986
  }
5081
- else {
5987
+ if (transport === "rateless") {
5082
5988
  bucket.ratelessFirstPasses += 1;
5083
5989
  }
5084
- const bypassKnownPeerHints = this.shouldBypassKnownPeerHints(options.mode, options.bypassKnownPeerHints);
5990
+ else {
5991
+ bucket.simpleFallbackPasses += 1;
5992
+ }
5085
5993
  return Promise.resolve(this.sendRepairEntriesWithTransport(target, filteredEntries, transport, {
5086
5994
  bypassKnownPeers: bypassKnownPeerHints,
5087
5995
  bypassRecentKnownPeers: bypassKnownPeerHints,
5088
5996
  })).catch((error) => logger.error(error));
5089
5997
  };
5998
+ const delayedJoinWarmupRetries = [];
5090
5999
  retrySchedule.forEach((delayMs, index) => {
5091
6000
  const transport = getRepairTransportForAttempt(options.mode, index);
5092
6001
  if (delayMs === 0) {
5093
6002
  void run(transport);
5094
6003
  return;
5095
6004
  }
6005
+ if (options.mode === "join-warmup" &&
6006
+ joinWarmupGeneration &&
6007
+ transport === "simple") {
6008
+ delayedJoinWarmupRetries.push(delayMs);
6009
+ return;
6010
+ }
5096
6011
  const timer = setTimeout(() => {
5097
6012
  this._repairRetryTimers.delete(timer);
5098
6013
  if (this.closed) {
@@ -5103,15 +6018,29 @@ let SharedLog = (() => {
5103
6018
  timer.unref?.();
5104
6019
  this._repairRetryTimers.add(timer);
5105
6020
  });
6021
+ if (joinWarmupGeneration && delayedJoinWarmupRetries.length > 0) {
6022
+ this.scheduleJoinWarmupRetries(target, joinWarmupGeneration, delayedJoinWarmupRetries, filteredEntries, bypassKnownPeerHints);
6023
+ }
5106
6024
  }
5107
6025
  scheduleRepairSweep(options) {
5108
- this._repairSweepPendingModes.add(options.mode);
5109
6026
  const pendingPeers = this._repairSweepPendingPeersByMode.get(options.mode);
5110
6027
  if (pendingPeers) {
5111
6028
  for (const peer of options.peers ?? []) {
6029
+ if (options.mode === "join-warmup") {
6030
+ const generation = options.joinWarmupGenerations?.get(peer) ??
6031
+ this.getJoinWarmupGeneration(peer);
6032
+ if (this._joinWarmupGenerationByTarget.get(peer) !== generation) {
6033
+ continue;
6034
+ }
6035
+ this._repairSweepJoinWarmupGenerationByTarget.set(peer, generation);
6036
+ }
5112
6037
  pendingPeers.add(peer);
5113
6038
  }
5114
6039
  }
6040
+ if (!pendingPeers || pendingPeers.size === 0) {
6041
+ return;
6042
+ }
6043
+ this._repairSweepPendingModes.add(options.mode);
5115
6044
  if (!this._repairSweepRunning && !this.closed) {
5116
6045
  this._repairSweepRunning = true;
5117
6046
  void this.runRepairSweep();
@@ -5162,10 +6091,26 @@ let SharedLog = (() => {
5162
6091
  while (!this.closed) {
5163
6092
  const pendingModes = new Set(this._repairSweepPendingModes);
5164
6093
  const pendingPeersByMode = cloneRepairPendingPeersByMode(this._repairSweepPendingPeersByMode);
6094
+ const pendingJoinWarmupGenerations = new Map(this._repairSweepJoinWarmupGenerationByTarget);
5165
6095
  this._repairSweepPendingModes.clear();
5166
6096
  for (const peers of this._repairSweepPendingPeersByMode.values()) {
5167
6097
  peers.clear();
5168
6098
  }
6099
+ this._repairSweepJoinWarmupGenerationByTarget.clear();
6100
+ const pendingJoinWarmupPeers = pendingPeersByMode.get("join-warmup");
6101
+ const pruneStaleJoinWarmupPeers = () => {
6102
+ for (const peer of [...(pendingJoinWarmupPeers ?? [])]) {
6103
+ if (this._joinWarmupGenerationByTarget.get(peer) !==
6104
+ pendingJoinWarmupGenerations.get(peer)) {
6105
+ pendingJoinWarmupPeers?.delete(peer);
6106
+ }
6107
+ }
6108
+ if (pendingJoinWarmupPeers?.size === 0) {
6109
+ pendingModes.delete("join-warmup");
6110
+ }
6111
+ return pendingModes.size > 0;
6112
+ };
6113
+ pruneStaleJoinWarmupPeers();
5169
6114
  if (pendingModes.size === 0) {
5170
6115
  return;
5171
6116
  }
@@ -5182,14 +6127,14 @@ let SharedLog = (() => {
5182
6127
  ._repairSweepOptimisticGidPeersPending) {
5183
6128
  let matchedPeers;
5184
6129
  let matchedCounts;
5185
- for (const [peer, count] of peerCounts) {
6130
+ for (const [peer, state] of peerCounts) {
5186
6131
  if (!modePeers.has(peer)) {
5187
6132
  continue;
5188
6133
  }
5189
6134
  matchedPeers ||= new Set();
5190
6135
  matchedCounts ||= new Map();
5191
6136
  matchedPeers.add(peer);
5192
- matchedCounts.set(peer, count);
6137
+ matchedCounts.set(peer, { ...state });
5193
6138
  }
5194
6139
  if (matchedPeers && matchedCounts) {
5195
6140
  optimisticGidPeers.set(gid, matchedPeers);
@@ -5211,6 +6156,7 @@ let SharedLog = (() => {
5211
6156
  const fullReplicaRepairCandidates = await this.getFullReplicaRepairCandidates(pendingRepairPeers, {
5212
6157
  includeSubscribers: false,
5213
6158
  });
6159
+ pruneStaleJoinWarmupPeers();
5214
6160
  const fullReplicaRepairCandidateCount = Math.max(1, fullReplicaRepairCandidates.size);
5215
6161
  const nextFrontierByMode = new Map([
5216
6162
  ["join-authoritative", new Map()],
@@ -5222,6 +6168,16 @@ let SharedLog = (() => {
5222
6168
  if (!entries || entries.size === 0) {
5223
6169
  return;
5224
6170
  }
6171
+ if (mode === "join-warmup" &&
6172
+ this._joinWarmupGenerationByTarget.get(target) !==
6173
+ pendingJoinWarmupGenerations.get(target)) {
6174
+ targets?.delete(target);
6175
+ pendingJoinWarmupPeers?.delete(target);
6176
+ if (pendingJoinWarmupPeers?.size === 0) {
6177
+ pendingModes.delete("join-warmup");
6178
+ }
6179
+ return;
6180
+ }
5225
6181
  this.dispatchMaybeMissingEntries(target, entries, {
5226
6182
  bypassRecentDedupe: true,
5227
6183
  bypassKnownPeerHints: mode === "churn" ||
@@ -5233,6 +6189,15 @@ let SharedLog = (() => {
5233
6189
  targets?.delete(target);
5234
6190
  };
5235
6191
  const queueEntryForTarget = (mode, target, entry) => {
6192
+ if (mode === "join-warmup" &&
6193
+ this._joinWarmupGenerationByTarget.get(target) !==
6194
+ pendingJoinWarmupGenerations.get(target)) {
6195
+ pendingJoinWarmupPeers?.delete(target);
6196
+ if (pendingJoinWarmupPeers?.size === 0) {
6197
+ pendingModes.delete("join-warmup");
6198
+ }
6199
+ return;
6200
+ }
5236
6201
  const sweepTargets = nextFrontierByMode.get(mode);
5237
6202
  if (sweepTargets) {
5238
6203
  let sweepSet = sweepTargets.get(target);
@@ -5260,14 +6225,17 @@ let SharedLog = (() => {
5260
6225
  if ((this._nativeBackbone ?? this._nativeSharedLogState) &&
5261
6226
  residentEntriesByHash &&
5262
6227
  !this.hasCustomFindLeaders()) {
5263
- const repairDispatchPlan = await this.planResidentRepairDispatchBatch({
5264
- pendingModes,
5265
- pendingPeersByMode,
5266
- optimisticGidPeersByMode,
5267
- fullReplicaRepairCandidates,
5268
- fullReplicaRepairCandidateCount,
5269
- selfHash: this.node.identity.publicKey.hashcode(),
5270
- });
6228
+ const repairDispatchPlan = pruneStaleJoinWarmupPeers()
6229
+ ? await this.planResidentRepairDispatchBatch({
6230
+ pendingModes,
6231
+ pendingPeersByMode,
6232
+ optimisticGidPeersByMode,
6233
+ fullReplicaRepairCandidates,
6234
+ fullReplicaRepairCandidateCount,
6235
+ selfHash: this.node.identity.publicKey.hashcode(),
6236
+ })
6237
+ : new Map();
6238
+ pruneStaleJoinWarmupPeers();
5271
6239
  for (const [mode, targets] of repairDispatchPlan) {
5272
6240
  for (const [target, hashes] of targets) {
5273
6241
  for (const hash of hashes) {
@@ -5279,11 +6247,16 @@ let SharedLog = (() => {
5279
6247
  }
5280
6248
  }
5281
6249
  }
5282
- else {
6250
+ else if (pruneStaleJoinWarmupPeers()) {
5283
6251
  const iterator = this.entryCoordinatesIndex.iterate({});
5284
6252
  try {
5285
- while (!this.closed && !iterator.done()) {
6253
+ while (!this.closed &&
6254
+ !iterator.done() &&
6255
+ pruneStaleJoinWarmupPeers()) {
5286
6256
  const entries = await iterator.next(REPAIR_SWEEP_ENTRY_BATCH_SIZE);
6257
+ if (!pruneStaleJoinWarmupPeers()) {
6258
+ break;
6259
+ }
5287
6260
  const entryReplicatedBatch = entries.map((entry) => entry.value);
5288
6261
  const requestedReplicasBatch = entryReplicatedBatch.map((entry) => decodeReplicas(entry).getValue(this));
5289
6262
  const repairDispatchPlan = await this.planRepairDispatchBatch({
@@ -5296,6 +6269,9 @@ let SharedLog = (() => {
5296
6269
  fullReplicaRepairCandidateCount,
5297
6270
  selfHash: this.node.identity.publicKey.hashcode(),
5298
6271
  });
6272
+ if (!pruneStaleJoinWarmupPeers()) {
6273
+ break;
6274
+ }
5299
6275
  const entriesByHash = new Map(entryReplicatedBatch.map((entry) => [entry.hash, entry]));
5300
6276
  for (const [mode, targets] of repairDispatchPlan) {
5301
6277
  for (const [target, hashes] of targets) {
@@ -5319,14 +6295,26 @@ let SharedLog = (() => {
5319
6295
  if (!pendingPeerCounts) {
5320
6296
  continue;
5321
6297
  }
5322
- for (const [peer, count] of peerCounts) {
5323
- const current = pendingPeerCounts.get(peer) || 0;
5324
- const next = current - count;
6298
+ for (const [peer, consumed] of peerCounts) {
6299
+ const current = pendingPeerCounts.get(peer);
6300
+ if (!current ||
6301
+ current.generation !== consumed.generation) {
6302
+ continue;
6303
+ }
6304
+ const next = current.count - consumed.count;
5325
6305
  if (next > 0) {
5326
- pendingPeerCounts.set(peer, next);
6306
+ pendingPeerCounts.set(peer, {
6307
+ count: next,
6308
+ generation: current.generation,
6309
+ });
5327
6310
  }
5328
6311
  else {
5329
6312
  pendingPeerCounts.delete(peer);
6313
+ const gids = this._repairSweepOptimisticGidsByPeer.get(peer);
6314
+ gids?.delete(gid);
6315
+ if (gids?.size === 0) {
6316
+ this._repairSweepOptimisticGidsByPeer.delete(peer);
6317
+ }
5330
6318
  }
5331
6319
  }
5332
6320
  if (pendingPeerCounts.size === 0) {
@@ -5772,10 +6760,20 @@ let SharedLog = (() => {
5772
6760
  if (toPrune.size === 0) {
5773
6761
  return;
5774
6762
  }
5775
- void Promise.allSettled(this.prune(toPrune));
6763
+ const pruneTasks = this.prune(toPrune);
6764
+ const confirmationTasks = [];
5776
6765
  for (const hash of responseStillApplies) {
5777
- void this._checkedPrune.getPendingDelete(hash)?.resolve(publicKeyHash);
6766
+ const pendingDelete = this._checkedPrune.getPendingDelete(hash);
6767
+ if (pendingDelete) {
6768
+ confirmationTasks.push(Promise.resolve(pendingDelete.resolve(publicKeyHash)));
6769
+ }
5778
6770
  }
6771
+ // The restarted prune session owns its own bounded timeout and revalidates
6772
+ // before deletion. Only the peer-derived confirmations must remain inside
6773
+ // this receive lease; waiting for the whole prune session could stall close
6774
+ // or disconnect until its background timeout.
6775
+ void Promise.allSettled(pruneTasks);
6776
+ await Promise.allSettled(confirmationTasks);
5779
6777
  }
5780
6778
  async append(data, options) {
5781
6779
  this.throwIfNativeDurableCommitFailed();
@@ -8193,20 +9191,37 @@ let SharedLog = (() => {
8193
9191
  this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 2e4;
8194
9192
  this._checkedPrune = new CheckedPruneCoordinator();
8195
9193
  this._pendingIHave = new Map();
9194
+ this._pendingIHaveCallbacks = new Set();
8196
9195
  this.latestReplicationInfoMessage = new Map();
8197
9196
  this._replicationInfoBlockedPeers = new Set();
8198
9197
  this._replicationInfoRequestByPeer = new Map();
9198
+ // Terminal close/drop drains the previous lifecycle before another open can
9199
+ // install fresh lanes and opaque per-subscription ownership tokens.
8199
9200
  this._replicationInfoApplyQueueByPeer = new Map();
9201
+ this._replicationInfoReceiveEpochByPeer = new Map();
9202
+ this._subscriptionEpochByPeer = new Map();
9203
+ this._pendingReplicatorLeaveByPeer = new Set();
9204
+ this._activeReceiveHandlersByPeer = new Map();
9205
+ this._receiveHandlerDrainByPeer = new Map();
9206
+ this._receiveCleanupGateByPeer = new Map();
9207
+ this._subscriptionOpeningEpochByPeer = new Map();
9208
+ this._openingSyncCapabilitiesByPeer = new Map();
8200
9209
  this._repairRetryTimers = new Set();
8201
9210
  this._recentRepairDispatch = new Map();
8202
9211
  this._repairSweepRunning = false;
8203
9212
  this._repairSweepPendingModes = new Set();
8204
9213
  this._repairSweepPendingPeersByMode = createRepairPendingPeersByMode();
9214
+ this._repairSweepJoinWarmupGenerationByTarget = new Map();
8205
9215
  this._repairFrontierByMode = createRepairFrontierByMode();
8206
9216
  this._repairFrontierActiveTargetsByMode = createRepairActiveTargetsByMode();
8207
9217
  this._repairFrontierBypassKnownPeersByMode =
8208
9218
  createRepairFrontierBypassKnownPeersByMode();
9219
+ this._joinWarmupGenerationByTarget = new Map();
9220
+ this._joinWarmupSendStateByTarget ??= new Map();
9221
+ this._joinWarmupRetryTimersByTarget = new Map();
9222
+ this._joinWarmupScheduledRetriesByTarget = new Map();
8209
9223
  this._repairSweepOptimisticGidPeersPending = new Map();
9224
+ this._repairSweepOptimisticGidsByPeer = new Map();
8210
9225
  this._entryKnownPeers = new Map();
8211
9226
  this._entryKnownPeerObservedAt = new Map();
8212
9227
  this._joinAuthoritativeRepairTimersByDelay = new Map();
@@ -8273,6 +9288,7 @@ let SharedLog = (() => {
8273
9288
  }
8274
9289
  this._closeController = new AbortController();
8275
9290
  this.setupReplicationAnnouncementRetryFunction();
9291
+ this.setupReplicationAnnouncementRepairFunction();
8276
9292
  this._closeController.signal.addEventListener("abort", () => {
8277
9293
  for (const [_peer, state] of this._replicationInfoRequestByPeer) {
8278
9294
  if (state.timer)
@@ -8353,7 +9369,9 @@ let SharedLog = (() => {
8353
9369
  publish: (message, options) => this.rpc.send(new BlocksMessage(message), options),
8354
9370
  waitFor: this.rpc.waitFor.bind(this.rpc),
8355
9371
  publicKey: this.node.identity.publicKey,
8356
- eagerBlocks: options?.eagerBlocks ?? true,
9372
+ // Unsolicited block retention is opt-in. Explicit `true` retains the
9373
+ // compatible eager path with bounded validation and storage budgets.
9374
+ eagerBlocks: options?.eagerBlocks ?? false,
8357
9375
  resolveProviders: async (cid, opts) => {
8358
9376
  // 1) tracker-backed provider directory (best-effort, bounded)
8359
9377
  try {
@@ -9096,10 +10114,13 @@ let SharedLog = (() => {
9096
10114
  async afterOpen() {
9097
10115
  await super.afterOpen();
9098
10116
  const existingSubscribersPromise = this._getTopicSubscribers(this.topic);
10117
+ const replicationLifecycleController = this._replicationLifecycleController;
9099
10118
  // We do this here, because these calls requires this.closed == false
9100
10119
  void this.pruneOfflineReplicators()
9101
10120
  .then(() => {
9102
- this._replicatorsReconciled = true;
10121
+ if (this.isReplicationLifecycleActive(replicationLifecycleController)) {
10122
+ this._replicatorsReconciled = true;
10123
+ }
9103
10124
  })
9104
10125
  .catch((error) => {
9105
10126
  if (isNotStartedError(error)) {
@@ -9126,46 +10147,81 @@ let SharedLog = (() => {
9126
10147
  async pruneOfflineReplicators() {
9127
10148
  // Go through all segments and wait for replicators to become reachable;
9128
10149
  // otherwise prune them away from the local membership view.
10150
+ const replicationLifecycleController = this._replicationLifecycleController;
9129
10151
  try {
10152
+ if (!replicationLifecycleController ||
10153
+ !this.isReplicationLifecycleActive(replicationLifecycleController)) {
10154
+ return;
10155
+ }
9130
10156
  const promises = [];
9131
10157
  const iterator = this.replicationIndex.iterate();
9132
10158
  const checkedIsAlive = new Set();
9133
10159
  while (!iterator.done()) {
9134
- for (const segment of await iterator.next(1000)) {
10160
+ const segments = await iterator.next(1000);
10161
+ if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
10162
+ return;
10163
+ }
10164
+ for (const segment of segments) {
9135
10165
  if (checkedIsAlive.has(segment.value.hash) ||
9136
10166
  this.node.identity.publicKey.hashcode() === segment.value.hash) {
9137
10167
  this.uniqueReplicators.add(this.node.identity.publicKey.hashcode());
9138
10168
  continue;
9139
10169
  }
9140
10170
  checkedIsAlive.add(segment.value.hash);
9141
- promises.push(waitForSubscribers(this.node, segment.value.hash, this.rpc.topic, {
10171
+ const peerHash = segment.value.hash;
10172
+ const subscriptionEpoch = this.getSubscriptionEpoch(peerHash);
10173
+ promises.push(waitForSubscribers(this.node, peerHash, this.rpc.topic, {
9142
10174
  timeout: this.waitForReplicatorTimeout,
9143
10175
  signal: this._closeController.signal,
9144
10176
  })
9145
10177
  .then(async () => {
9146
- const key = await this._resolvePublicKeyFromHash(segment.value.hash);
10178
+ if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
10179
+ return;
10180
+ }
10181
+ const key = await this._resolvePublicKeyFromHash(peerHash);
9147
10182
  if (!key) {
9148
10183
  throw new Error("Failed to resolve public key from hash: " +
9149
- segment.value.hash);
10184
+ peerHash);
9150
10185
  }
9151
10186
  const keyHash = key.hashcode();
9152
- this.uniqueReplicators.add(keyHash);
9153
- if (!this._replicatorJoinEmitted.has(keyHash)) {
9154
- this._replicatorJoinEmitted.add(keyHash);
9155
- this.events.dispatchEvent(new CustomEvent("replicator:join", {
9156
- detail: { publicKey: key },
9157
- }));
9158
- this.events.dispatchEvent(new CustomEvent("replication:change", {
9159
- detail: { publicKey: key },
9160
- }));
10187
+ if (keyHash !== peerHash) {
10188
+ return;
9161
10189
  }
10190
+ return this.withReplicationInfoApplyQueue(keyHash, async () => {
10191
+ // A successful reachability check may legitimately span a
10192
+ // subscribe event during startup. The current lane's blocked
10193
+ // state plus an extant index row are the authoritative guard;
10194
+ // only the destructive catch path remains tied to the old token.
10195
+ if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
10196
+ this.closed ||
10197
+ this._replicationInfoBlockedPeers.has(keyHash)) {
10198
+ return;
10199
+ }
10200
+ const hasReplicationRange = (await this.replicationIndex.count({
10201
+ query: { hash: keyHash },
10202
+ })) > 0;
10203
+ if (!hasReplicationRange ||
10204
+ !this.isReplicationLifecycleActive(replicationLifecycleController) ||
10205
+ this._replicationInfoBlockedPeers.has(keyHash)) {
10206
+ return;
10207
+ }
10208
+ this.uniqueReplicators.add(keyHash);
10209
+ if (!this._replicatorJoinEmitted.has(keyHash)) {
10210
+ this._replicatorJoinEmitted.add(keyHash);
10211
+ this.events.dispatchEvent(new CustomEvent("replicator:join", { detail: { publicKey: key } }));
10212
+ this.events.dispatchEvent(new CustomEvent("replication:change", { detail: { publicKey: key } }));
10213
+ }
10214
+ });
9162
10215
  })
9163
10216
  .catch(async (error) => {
9164
- if (isNotStartedError(error)) {
10217
+ if (isNotStartedError(error) ||
10218
+ !this.isReplicationLifecycleActive(replicationLifecycleController)) {
9165
10219
  return;
9166
10220
  }
9167
- return this.removeReplicator(segment.value.hash, {
10221
+ return this.removeReplicator(peerHash, {
9168
10222
  noEvent: true,
10223
+ replicationLifecycleController,
10224
+ subscriptionEpoch,
9169
10225
  });
9170
10226
  }));
9171
10227
  }
@@ -9173,7 +10229,8 @@ let SharedLog = (() => {
9173
10229
  return Promise.all(promises);
9174
10230
  }
9175
10231
  catch (error) {
9176
- if (isNotStartedError(error)) {
10232
+ if (isNotStartedError(error) ||
10233
+ !this.isReplicationLifecycleActive(replicationLifecycleController)) {
9177
10234
  return;
9178
10235
  }
9179
10236
  throw error;
@@ -9228,10 +10285,26 @@ let SharedLog = (() => {
9228
10285
  this._replicatorLivenessFailures.delete(peerHash);
9229
10286
  this._replicatorLastActivityAt.delete(peerHash);
9230
10287
  this._peerSyncCapabilities.delete(peerHash);
10288
+ this.cleanupPendingIHavePeer(peerHash);
9231
10289
  this._checkedPrune.cleanupPeer(peerHash);
9232
10290
  }
10291
+ cleanupPendingIHavePeer(peerHash) {
10292
+ for (const [hash, pending] of this._pendingIHave) {
10293
+ pending.requesting.delete(peerHash);
10294
+ if (pending.requesting.size === 0) {
10295
+ pending.clear();
10296
+ this._pendingIHave.delete(hash);
10297
+ }
10298
+ }
10299
+ }
9233
10300
  markReplicatorActivity(peerHash, now = Date.now()) {
9234
10301
  this._replicatorLastActivityAt.set(peerHash, now);
10302
+ // Any recent authenticated activity is positive liveness evidence. Reset the
10303
+ // consecutive miss streak immediately, including while an eviction is
10304
+ // waiting in the per-peer mutation lane.
10305
+ if (Date.now() - now < REPLICATOR_LIVENESS_IDLE_THRESHOLD_MS) {
10306
+ this._replicatorLivenessFailures.delete(peerHash);
10307
+ }
9235
10308
  }
9236
10309
  hasRecentReplicatorActivity(peerHash, now = Date.now()) {
9237
10310
  const lastActivityAt = this._replicatorLastActivityAt.get(peerHash);
@@ -9242,31 +10315,48 @@ let SharedLog = (() => {
9242
10315
  }
9243
10316
  return false;
9244
10317
  }
9245
- async evictReplicatorFromLiveness(peerHash, publicKey) {
9246
- const wasReplicator = this.uniqueReplicators.has(peerHash);
9247
- const watermark = BigInt(+new Date());
9248
- const previousWatermark = this.latestReplicationInfoMessage.get(peerHash);
9249
- if (!previousWatermark || previousWatermark < watermark) {
9250
- this.latestReplicationInfoMessage.set(peerHash, watermark);
9251
- }
10318
+ advanceReplicationInfoRecoveryEpoch(peerHash) {
10319
+ // Handlers admitted before a successful peer removal must not restore state
10320
+ // when they eventually reach the apply lane. Reset the sender's
10321
+ // ordering watermark with the local epoch so a later arrival can be
10322
+ // accepted without comparing its clock to this receiver's clock.
10323
+ this.advanceReplicationInfoReceiveEpoch(peerHash);
10324
+ this.latestReplicationInfoMessage.delete(peerHash);
10325
+ }
10326
+ async evictReplicatorFromLiveness(peerHash, publicKey, replicationLifecycleController, subscriptionEpoch, observedActivityAt) {
9252
10327
  try {
9253
- await this.removeReplicator(publicKey, { noEvent: true });
10328
+ await this.removeReplicator(publicKey, {
10329
+ noEvent: true,
10330
+ replicationLifecycleController,
10331
+ shouldRemove: () => this._replicatorLastActivityAt.get(peerHash) === observedActivityAt,
10332
+ subscriptionEpoch,
10333
+ onRemoved: ({ wasReplicator }) => {
10334
+ if (wasReplicator) {
10335
+ this._pendingReplicatorLeaveByPeer.add(peerHash);
10336
+ }
10337
+ // A newer subscription/lifecycle may have started while the admitted
10338
+ // removal was completing. Its reconnect barrier owns all later effects.
10339
+ if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
10340
+ !this.isCurrentSubscriptionEpoch(peerHash, subscriptionEpoch)) {
10341
+ return;
10342
+ }
10343
+ if (this._pendingReplicatorLeaveByPeer.delete(peerHash)) {
10344
+ this.events.dispatchEvent(new CustomEvent("replicator:leave", {
10345
+ detail: { publicKey },
10346
+ }));
10347
+ }
10348
+ if (!this._replicationInfoBlockedPeers.has(peerHash)) {
10349
+ this.scheduleReplicationInfoRequests(publicKey, replicationLifecycleController);
10350
+ }
10351
+ this._replicatorLivenessTargetsSize = -1;
10352
+ },
10353
+ });
9254
10354
  }
9255
10355
  catch (error) {
9256
10356
  if (!isNotStartedError(error)) {
9257
10357
  throw error;
9258
10358
  }
9259
10359
  }
9260
- this.cleanupPeerDisconnectTracking(peerHash);
9261
- if (wasReplicator) {
9262
- this.events.dispatchEvent(new CustomEvent("replicator:leave", {
9263
- detail: { publicKey },
9264
- }));
9265
- }
9266
- if (!this._replicationInfoBlockedPeers.has(peerHash)) {
9267
- this.scheduleReplicationInfoRequests(publicKey);
9268
- }
9269
- this._replicatorLivenessTargetsSize = -1;
9270
10360
  }
9271
10361
  async resolveCandidatePeersForHash(hash, options) {
9272
10362
  if (options?.signal?.aborted)
@@ -9338,7 +10428,10 @@ let SharedLog = (() => {
9338
10428
  return pickDeterministicSubset(peers, seed, maxPeers);
9339
10429
  }
9340
10430
  async runReplicatorLivenessSweep() {
9341
- if (this.closed || this._closeController.signal.aborted) {
10431
+ const replicationLifecycleController = this._replicationLifecycleController;
10432
+ if (this.closed ||
10433
+ this._closeController.signal.aborted ||
10434
+ !this.isReplicationLifecycleActive(replicationLifecycleController)) {
9342
10435
  return;
9343
10436
  }
9344
10437
  if (this._replicatorLivenessSweepRunning) {
@@ -9364,13 +10457,23 @@ let SharedLog = (() => {
9364
10457
  }
9365
10458
  }
9366
10459
  finally {
9367
- this._replicatorLivenessSweepRunning = false;
10460
+ if (this._replicationLifecycleController ===
10461
+ replicationLifecycleController) {
10462
+ this._replicatorLivenessSweepRunning = false;
10463
+ }
9368
10464
  }
9369
10465
  }
9370
10466
  async probeReplicatorLiveness(peerHash) {
9371
- if (this.closed || this._closeController.signal.aborted) {
10467
+ const replicationLifecycleController = this._replicationLifecycleController;
10468
+ if (this.closed ||
10469
+ this._closeController.signal.aborted ||
10470
+ !replicationLifecycleController ||
10471
+ !this.isReplicationLifecycleActive(replicationLifecycleController)) {
9372
10472
  return;
9373
10473
  }
10474
+ const subscriptionEpoch = this.getSubscriptionEpoch(peerHash);
10475
+ const ownsProbe = () => this.isReplicationLifecycleActive(replicationLifecycleController) &&
10476
+ this.isCurrentSubscriptionEpoch(peerHash, subscriptionEpoch);
9374
10477
  if (!this.uniqueReplicators.has(peerHash)) {
9375
10478
  this._replicatorLivenessFailures.delete(peerHash);
9376
10479
  return;
@@ -9378,18 +10481,35 @@ let SharedLog = (() => {
9378
10481
  if (this.hasRecentReplicatorActivity(peerHash)) {
9379
10482
  return;
9380
10483
  }
10484
+ const observedActivityAt = this._replicatorLastActivityAt.get(peerHash);
9381
10485
  const publicKey = await this._resolvePublicKeyFromHash(peerHash);
10486
+ if (!ownsProbe()) {
10487
+ return;
10488
+ }
10489
+ if (this.hasRecentReplicatorActivity(peerHash)) {
10490
+ return;
10491
+ }
9382
10492
  if (!publicKey) {
9383
10493
  try {
9384
- await this.removeReplicator(peerHash, { noEvent: true });
10494
+ await this.removeReplicator(peerHash, {
10495
+ noEvent: true,
10496
+ replicationLifecycleController,
10497
+ shouldRemove: () => this._replicatorLastActivityAt.get(peerHash) ===
10498
+ observedActivityAt,
10499
+ subscriptionEpoch,
10500
+ onRemoved: () => {
10501
+ if (!ownsProbe()) {
10502
+ return;
10503
+ }
10504
+ this._replicatorLivenessTargetsSize = -1;
10505
+ },
10506
+ });
9385
10507
  }
9386
10508
  catch (error) {
9387
10509
  if (!isNotStartedError(error)) {
9388
10510
  throw error;
9389
10511
  }
9390
10512
  }
9391
- this.cleanupPeerDisconnectTracking(peerHash);
9392
- this._replicatorLivenessTargetsSize = -1;
9393
10513
  return;
9394
10514
  }
9395
10515
  try {
@@ -9400,6 +10520,9 @@ let SharedLog = (() => {
9400
10520
  priority: ACK_CONTROL_PRIORITY,
9401
10521
  responsePriority: ACK_CONTROL_PRIORITY,
9402
10522
  });
10523
+ if (!ownsProbe()) {
10524
+ return;
10525
+ }
9403
10526
  this.markReplicatorActivity(peerHash);
9404
10527
  this._replicatorLivenessFailures.delete(peerHash);
9405
10528
  return;
@@ -9409,25 +10532,40 @@ let SharedLog = (() => {
9409
10532
  return;
9410
10533
  }
9411
10534
  }
10535
+ if (!ownsProbe()) {
10536
+ return;
10537
+ }
10538
+ if (this.hasRecentReplicatorActivity(peerHash)) {
10539
+ return;
10540
+ }
9412
10541
  // Relay-backed prod paths can keep a peer subscribed/reachable even if an
9413
10542
  // ACKed liveness ping gets delayed or dropped under load. Treat observed
9414
10543
  // topic presence as a positive liveness signal before evicting the peer.
9415
10544
  if (await this.confirmReplicatorSubscriberPresence(peerHash)) {
10545
+ if (!ownsProbe()) {
10546
+ return;
10547
+ }
9416
10548
  this.markReplicatorActivity(peerHash);
9417
10549
  this._replicatorLivenessFailures.delete(peerHash);
9418
10550
  return;
9419
10551
  }
10552
+ if (!ownsProbe()) {
10553
+ return;
10554
+ }
10555
+ if (this.hasRecentReplicatorActivity(peerHash)) {
10556
+ return;
10557
+ }
9420
10558
  const failures = (this._replicatorLivenessFailures.get(peerHash) ?? 0) + 1;
9421
10559
  this._replicatorLivenessFailures.set(peerHash, failures);
9422
- this.scheduleReplicationInfoRequests(publicKey);
10560
+ this.scheduleReplicationInfoRequests(publicKey, replicationLifecycleController);
9423
10561
  if (failures < REPLICATOR_LIVENESS_PROBE_FAILURES_TO_EVICT) {
9424
10562
  return;
9425
10563
  }
9426
- if (!this.uniqueReplicators.has(peerHash)) {
10564
+ if (!ownsProbe() || !this.uniqueReplicators.has(peerHash)) {
9427
10565
  this._replicatorLivenessFailures.delete(peerHash);
9428
10566
  return;
9429
10567
  }
9430
- await this.evictReplicatorFromLiveness(peerHash, publicKey);
10568
+ await this.evictReplicatorFromLiveness(peerHash, publicKey, replicationLifecycleController, subscriptionEpoch, observedActivityAt);
9431
10569
  }
9432
10570
  async confirmReplicatorSubscriberPresence(peerHash) {
9433
10571
  try {
@@ -9459,6 +10597,24 @@ let SharedLog = (() => {
9459
10597
  return this.log.blocks.size();
9460
10598
  /* ((await this.log.entryIndex?.getMemoryUsage()) || 0) */ // + (await this.log.blocks.size())
9461
10599
  }
10600
+ /** Return a detached snapshot of effective shared-log runtime settings. */
10601
+ getRuntimeSnapshot() {
10602
+ const nativeGraph = this.log.entryIndex.properties.nativeGraph;
10603
+ const active = nativeGraph?.graph != null;
10604
+ return Object.freeze({
10605
+ nativeGraph: Object.freeze({
10606
+ active,
10607
+ useHeads: active && nativeGraph?.useHeads === true,
10608
+ }),
10609
+ });
10610
+ }
10611
+ /**
10612
+ * Return a detached snapshot of the optional eager-response cache.
10613
+ * Undefined means eager response retention is disabled for this log.
10614
+ */
10615
+ getEagerBlockCacheTelemetry() {
10616
+ return this.remoteBlocks?.getEagerBlockCacheTelemetry();
10617
+ }
9462
10618
  clampReplicas(value) {
9463
10619
  const lower = this.replicas.min?.getValue(this) || 1;
9464
10620
  const higher = this.replicas.max?.getValue(this) ?? Number.MAX_SAFE_INTEGER;
@@ -10073,6 +11229,7 @@ let SharedLog = (() => {
10073
11229
  captureSync(() => this.node.services.pubsub.removeEventListener("subscribe", this._onSubscriptionFn));
10074
11230
  captureSync(() => this.node.services.pubsub.removeEventListener("unsubscribe", this._onUnsubscriptionFn));
10075
11231
  captureSync(() => {
11232
+ this.cancelAllJoinWarmupTargets();
10076
11233
  for (const timer of this._repairRetryTimers ?? [])
10077
11234
  clearTimeout(timer);
10078
11235
  this._repairRetryTimers?.clear();
@@ -10081,7 +11238,9 @@ let SharedLog = (() => {
10081
11238
  this._repairSweepPendingModes?.clear();
10082
11239
  for (const peers of this._repairSweepPendingPeersByMode?.values() ?? [])
10083
11240
  peers.clear();
11241
+ this._repairSweepJoinWarmupGenerationByTarget?.clear();
10084
11242
  this._repairSweepOptimisticGidPeersPending?.clear();
11243
+ this._repairSweepOptimisticGidsByPeer?.clear();
10085
11244
  this._entryKnownPeers?.clear();
10086
11245
  this._entryKnownPeerObservedAt?.clear();
10087
11246
  this._nativeSharedLogState?.clearEntryKnownPeers();
@@ -10118,7 +11277,14 @@ let SharedLog = (() => {
10118
11277
  }
10119
11278
  captureSync(() => {
10120
11279
  this._pendingIHave?.clear();
11280
+ this._pendingIHaveCallbacks?.clear();
10121
11281
  this.latestReplicationInfoMessage?.clear();
11282
+ this._replicationInfoReceiveEpochByPeer?.clear();
11283
+ this._activeReceiveHandlersByPeer?.clear();
11284
+ this._receiveHandlerDrainByPeer?.clear();
11285
+ this._receiveCleanupGateByPeer?.clear();
11286
+ this._subscriptionOpeningEpochByPeer?.clear();
11287
+ this._openingSyncCapabilitiesByPeer?.clear();
10122
11288
  this._gidPeersHistory?.clear();
10123
11289
  this._peerSyncCapabilities?.clear();
10124
11290
  this._liveRawGossipBatches?.clear();
@@ -10181,7 +11347,14 @@ let SharedLog = (() => {
10181
11347
  // SharedLog-specific await or observable teardown can admit a new owner.
10182
11348
  this.preventParentAttachments();
10183
11349
  this.stopSubscriptionChangeCallbackAdmission();
11350
+ this.cancelAllJoinWarmupTargets();
10184
11351
  await this.drainSubscriptionChangeCallbacks();
11352
+ // An already-admitted subscription callback can create a fresh warmup
11353
+ // generation while the first cancellation is draining.
11354
+ this.cancelAllJoinWarmupTargets();
11355
+ await this.drainReceiveHandlers();
11356
+ await this.drainReplicationInfoApplyQueues();
11357
+ await this.drainPendingIHaveCallbacks();
10185
11358
  this.ensureNativeDurabilityRuntimeState();
10186
11359
  this.cancelCurrentReplicationStateAnnouncementRetry();
10187
11360
  // Best-effort: announce that we are going offline before tearing down
@@ -10301,7 +11474,14 @@ let SharedLog = (() => {
10301
11474
  // the terminal fence only after that precondition succeeds.
10302
11475
  this.preventParentAttachments();
10303
11476
  this.stopSubscriptionChangeCallbackAdmission();
11477
+ this.cancelAllJoinWarmupTargets();
10304
11478
  await this.drainSubscriptionChangeCallbacks();
11479
+ // An already-admitted subscription callback can create a fresh warmup
11480
+ // generation while the first cancellation is draining.
11481
+ this.cancelAllJoinWarmupTargets();
11482
+ await this.drainReceiveHandlers();
11483
+ await this.drainReplicationInfoApplyQueues();
11484
+ await this.drainPendingIHaveCallbacks();
10305
11485
  this.cancelCurrentReplicationStateAnnouncementRetry();
10306
11486
  // Best-effort: announce that we are going offline before tearing down
10307
11487
  // RPC/subscription state (same reasoning as in `close()`).
@@ -10512,13 +11692,35 @@ let SharedLog = (() => {
10512
11692
  const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
10513
11693
  ? msg
10514
11694
  : undefined;
11695
+ let releasePeerReceiveLease;
10515
11696
  try {
10516
11697
  this.throwIfNativeDurableCommitFailed();
10517
11698
  if (!context.from) {
10518
11699
  throw new Error("Missing from in update role message");
10519
11700
  }
11701
+ // Snapshot receive ownership before any async handler gets a chance to
11702
+ // yield. Replication-info messages reach their branch only after the
11703
+ // synchronizer declines them, and a U/S transition can happen meanwhile.
11704
+ const receiveFromHash = context.from.hashcode();
11705
+ const receiveReplicationLifecycleController = this._replicationLifecycleController;
11706
+ const receiveSubscriptionEpoch = this.getSubscriptionEpoch(receiveFromHash);
11707
+ const isOpeningSubscriptionReceive = this._subscriptionOpeningEpochByPeer.get(receiveFromHash) ===
11708
+ receiveSubscriptionEpoch;
11709
+ const isOpeningCapabilityAdvertisement = msg instanceof SyncCapabilitiesMessage && isOpeningSubscriptionReceive;
11710
+ releasePeerReceiveLease = this.acquirePeerReceiveLease(receiveFromHash, receiveReplicationLifecycleController, receiveSubscriptionEpoch, {
11711
+ // The replication-info fence existed before receive leases and is
11712
+ // intentionally narrower than the subscription itself. Keep admitting
11713
+ // sync negotiation/data while the new subscription drains the previous
11714
+ // apply generation; replication-info branches recheck the fence below.
11715
+ allowReplicationInfoBlocked: isOpeningSubscriptionReceive,
11716
+ allowCleanupGate: isOpeningCapabilityAdvertisement,
11717
+ });
11718
+ if (!releasePeerReceiveLease) {
11719
+ return;
11720
+ }
11721
+ const receiveReplicationInfoReceiveEpoch = this.getReplicationInfoReceiveEpoch(receiveFromHash);
10520
11722
  if (!context.from.equals(this.node.identity.publicKey)) {
10521
- this.markReplicatorActivity(context.from.hashcode());
11723
+ this.markReplicatorActivity(receiveFromHash);
10522
11724
  }
10523
11725
  if (msg instanceof ResponseRoleMessage) {
10524
11726
  msg = msg.toReplicationInfoMessage(); // migration
@@ -11601,6 +12803,7 @@ let SharedLog = (() => {
11601
12803
  allToMergeMaterializedEntries ??= allToMerge.map((entry) => entry.entry);
11602
12804
  return allToMergeMaterializedEntries;
11603
12805
  };
12806
+ let admittedMergeHashes = new Set();
11604
12807
  let nativePreparedCommittedHashes;
11605
12808
  if (allToMerge.length > 0) {
11606
12809
  const validateStartedAt = syncProfileStart(syncProfile);
@@ -11696,8 +12899,8 @@ let SharedLog = (() => {
11696
12899
  const entriesToPersist = usedNativeAllKeptJoinPlan
11697
12900
  ? allToMergeShallowEntries
11698
12901
  : joinPlans.flatMap((plan) => plan.toPersist);
11699
- const coordinatePersistFallbackEntries = [];
11700
- const reusableCoordinatePersistItems = [];
12902
+ let coordinatePersistFallbackEntries = [];
12903
+ let reusableCoordinatePersistItems = [];
11701
12904
  for (const entry of entriesToPersist) {
11702
12905
  const reusablePlan = reusableCoordinatePlans.get(entry.hash);
11703
12906
  if (!reusablePlan) {
@@ -11713,7 +12916,6 @@ let SharedLog = (() => {
11713
12916
  prepared: reusablePlan.prepared,
11714
12917
  });
11715
12918
  }
11716
- const reusableCoordinatePersistItemCount = reusableCoordinatePersistItems.length;
11717
12919
  let nativePreparedCoordinateBatch;
11718
12920
  let nativePreparedCoordinatesFinished = false;
11719
12921
  let nativeBackboneOnlyPersistedHashes;
@@ -11789,6 +12991,26 @@ let SharedLog = (() => {
11789
12991
  __peerbitProfile: syncProfile,
11790
12992
  });
11791
12993
  }
12994
+ // A recursive lower-log join can resolve successfully while declining
12995
+ // an individual top-level entry (for example, when one of its parents
12996
+ // is temporarily unavailable). The public Log.join() API intentionally
12997
+ // does not expose that per-entry result, so make local index presence the
12998
+ // authority before publishing any SharedLog-side effects. A successful
12999
+ // prepared-facts batch is atomic and already proves every input hash.
13000
+ const admittedHashes = joinedPreparedFacts
13001
+ ? new Set(allToMergeHashes)
13002
+ : await this.log.hasMany(allToMergeHashes);
13003
+ admittedMergeHashes = admittedHashes;
13004
+ const admittedShallowEntries = admittedHashes.size === allToMergeShallowEntries.length
13005
+ ? allToMergeShallowEntries
13006
+ : allToMergeShallowEntries.filter((entry) => admittedHashes.has(entry.hash));
13007
+ if (!joinedPreparedFacts) {
13008
+ reusableCoordinatePersistItems =
13009
+ reusableCoordinatePersistItems.filter((item) => admittedHashes.has(item.entry.hash));
13010
+ coordinatePersistFallbackEntries =
13011
+ coordinatePersistFallbackEntries.filter((entry) => admittedHashes.has(entry.hash));
13012
+ }
13013
+ const reusableCoordinatePersistItemCount = reusableCoordinatePersistItems.length;
11792
13014
  if (syncProfile) {
11793
13015
  emitSyncProfileDuration(syncProfile, lowerLogJoinStartedAt, {
11794
13016
  name: "sharedLog.receive.lowerLogJoin",
@@ -11800,6 +13022,7 @@ let SharedLog = (() => {
11800
13022
  batchHashOnlyEntryAdded,
11801
13023
  programOnChange,
11802
13024
  joinedPreparedFacts,
13025
+ admittedEntries: admittedHashes.size,
11803
13026
  nativePreparedCoordinatesFinished,
11804
13027
  },
11805
13028
  });
@@ -11854,11 +13077,11 @@ let SharedLog = (() => {
11854
13077
  },
11855
13078
  });
11856
13079
  }
11857
- for (const hash of allToMergeHashes) {
13080
+ for (const hash of admittedHashes) {
11858
13081
  confirmedHashes.add(hash);
11859
13082
  }
11860
13083
  const checkedPruneStartedAt = syncProfileStart(syncProfile);
11861
- await this.pruneJoinedEntriesNoLongerLed(allToMergeShallowEntries, {
13084
+ await this.pruneJoinedEntriesNoLongerLed(admittedShallowEntries, {
11862
13085
  decodedReplicaCounts: receiveReplicaCounts,
11863
13086
  reusableLeaderPlans: reusableCoordinatePlans,
11864
13087
  profile: syncProfile,
@@ -11872,10 +13095,12 @@ let SharedLog = (() => {
11872
13095
  });
11873
13096
  }
11874
13097
  for (const plan of joinPlans) {
11875
- plan.toDelete?.map((x) => this.pruneDebouncedFnAddIfNotKeeping({
11876
- key: x.hash,
13098
+ plan.toDelete
13099
+ ?.filter((entry) => admittedMergeHashes.has(entry.hash))
13100
+ .map((entry) => this.pruneDebouncedFnAddIfNotKeeping({
13101
+ key: entry.hash,
11877
13102
  value: {
11878
- entry: x,
13103
+ entry,
11879
13104
  leaders: plan.leaders,
11880
13105
  },
11881
13106
  }));
@@ -11887,14 +13112,18 @@ let SharedLog = (() => {
11887
13112
  continue;
11888
13113
  }
11889
13114
  for (const entries of plan.maybeDelete) {
11890
- const minReplicas = await this.getMaxReplicasFromHeads(this.getEntryGid(entries[0].entry));
13115
+ const admittedEntries = entries.filter((entry) => admittedMergeHashes.has(getExchangeHeadHash(entry)));
13116
+ if (admittedEntries.length === 0) {
13117
+ continue;
13118
+ }
13119
+ const minReplicas = await this.getMaxReplicasFromHeads(this.getEntryGid(admittedEntries[0].entry));
11891
13120
  if (minReplicas != null) {
11892
13121
  const isLeader = await this.isLeader({
11893
- entry: entries[0].entry,
13122
+ entry: admittedEntries[0].entry,
11894
13123
  replicas: minReplicas,
11895
13124
  });
11896
13125
  if (!isLeader) {
11897
- for (const x of entries) {
13126
+ for (const x of admittedEntries) {
11898
13127
  this.pruneDebouncedFnAddIfNotKeeping({
11899
13128
  key: x.entry.hash,
11900
13129
  value: {
@@ -12101,6 +13330,9 @@ let SharedLog = (() => {
12101
13330
  let pendingIHaveExtended = 0;
12102
13331
  const leaderResponseHashes = [];
12103
13332
  for (let i = 0; i < msg.hashes.length; i++) {
13333
+ if (!this.isReplicationLifecycleActive(receiveReplicationLifecycleController)) {
13334
+ return;
13335
+ }
12104
13336
  const hash = msg.hashes[i];
12105
13337
  const nativeEntryGid = nativeLeaderHints.nativeEntryGids?.[i];
12106
13338
  const nativeEntryData = nativeLeaderHints.nativeEntryDataByIndex?.[i];
@@ -12213,8 +13445,14 @@ let SharedLog = (() => {
12213
13445
  resetTimeout: () => this.resetPendingIHaveTimeout(pendingIHave),
12214
13446
  clear: () => this.clearPendingIHaveTimeout(pendingIHave),
12215
13447
  callback: async (entry) => {
12216
- this.removePeerFromGidPeerHistory(from, entry.meta.gid);
12217
- this.removePruneRequestSent(entry.hash, from);
13448
+ if (!this.isReplicationLifecycleActive(receiveReplicationLifecycleController) ||
13449
+ requesting.size === 0) {
13450
+ return;
13451
+ }
13452
+ for (const requester of requesting) {
13453
+ this.removePeerFromGidPeerHistory(requester, entry.meta.gid);
13454
+ this.removePruneRequestSent(entry.hash, requester);
13455
+ }
12218
13456
  let isLeader = false;
12219
13457
  await this._waitForEntryReplicators(entry, decodeReplicas(entry).getValue(this), [
12220
13458
  {
@@ -12228,12 +13466,15 @@ let SharedLog = (() => {
12228
13466
  key === this.node.identity.publicKey.hashcode();
12229
13467
  },
12230
13468
  });
13469
+ if (!this.isReplicationLifecycleActive(receiveReplicationLifecycleController) ||
13470
+ requesting.size === 0) {
13471
+ return;
13472
+ }
12231
13473
  if (isLeader) {
12232
13474
  this.responseToPruneDebouncedFn.add({
12233
13475
  hashes: [entry.hash],
12234
- peers: requesting,
13476
+ peers: new Set(requesting),
12235
13477
  });
12236
- this._pendingIHave.delete(hash);
12237
13478
  }
12238
13479
  },
12239
13480
  };
@@ -12276,17 +13517,24 @@ let SharedLog = (() => {
12276
13517
  }
12277
13518
  else if (msg instanceof ResponseIPrune) {
12278
13519
  const lateResponses = [];
13520
+ const responseTasks = [];
12279
13521
  for (const hash of msg.hashes) {
12280
13522
  const pendingDelete = this._checkedPrune.getPendingDelete(hash);
12281
13523
  if (pendingDelete) {
12282
- void pendingDelete.resolve(context.from.hashcode());
13524
+ responseTasks.push(Promise.resolve(pendingDelete.resolve(context.from.hashcode())));
12283
13525
  }
12284
13526
  else {
12285
13527
  lateResponses.push(hash);
12286
13528
  }
12287
13529
  }
12288
13530
  if (lateResponses.length > 0) {
12289
- void this.recoverCheckedPruneFromLateResponses(lateResponses, context.from.hashcode()).catch((error) => logger.error(error.toString()));
13531
+ responseTasks.push(this.recoverCheckedPruneFromLateResponses(lateResponses, context.from.hashcode()));
13532
+ }
13533
+ const results = await Promise.allSettled(responseTasks);
13534
+ for (const result of results) {
13535
+ if (result.status === "rejected") {
13536
+ logger.error(result.reason?.toString?.() ?? String(result.reason));
13537
+ }
12290
13538
  }
12291
13539
  }
12292
13540
  else if (msg instanceof ConfirmEntriesMessage) {
@@ -12296,7 +13544,20 @@ let SharedLog = (() => {
12296
13544
  }
12297
13545
  else if (msg instanceof SyncCapabilitiesMessage) {
12298
13546
  if (!context.from.equals(this.node.identity.publicKey)) {
12299
- this._peerSyncCapabilities.set(context.from.hashcode(), msg.capabilities);
13547
+ const openingEpoch = this._subscriptionOpeningEpochByPeer.get(receiveFromHash);
13548
+ if (this._replicationInfoBlockedPeers.has(receiveFromHash) &&
13549
+ openingEpoch === receiveSubscriptionEpoch) {
13550
+ // A prior unsubscribe cleanup may still be ahead of this reconnect
13551
+ // barrier. Stage the new generation's one-shot advertisement so that
13552
+ // cleanup cannot erase it before the opening transition commits.
13553
+ this._openingSyncCapabilitiesByPeer.set(receiveFromHash, {
13554
+ epoch: openingEpoch,
13555
+ capabilities: msg.capabilities,
13556
+ });
13557
+ }
13558
+ else {
13559
+ this._peerSyncCapabilities.set(receiveFromHash, msg.capabilities);
13560
+ }
12300
13561
  }
12301
13562
  return;
12302
13563
  }
@@ -12316,9 +13577,9 @@ let SharedLog = (() => {
12316
13577
  if (context.from.equals(this.node.identity.publicKey)) {
12317
13578
  return;
12318
13579
  }
12319
- const replicationLifecycleController = this._replicationLifecycleController;
13580
+ const replicationLifecycleController = receiveReplicationLifecycleController;
12320
13581
  if (!replicationLifecycleController ||
12321
- !this.isReplicationLifecycleActive(replicationLifecycleController)) {
13582
+ !this.isPeerReceiveAdmissionOpen(receiveFromHash, replicationLifecycleController, receiveSubscriptionEpoch)) {
12322
13583
  return;
12323
13584
  }
12324
13585
  let replicationSegments;
@@ -12326,13 +13587,13 @@ let SharedLog = (() => {
12326
13587
  replicationSegments = await this.getMyReplicationSegments();
12327
13588
  }
12328
13589
  catch (error) {
12329
- if (!this.isReplicationLifecycleActive(replicationLifecycleController) &&
13590
+ if (!this.isPeerReceiveAdmissionOpen(receiveFromHash, replicationLifecycleController, receiveSubscriptionEpoch) &&
12330
13591
  isNotStartedError(error)) {
12331
13592
  return;
12332
13593
  }
12333
13594
  throw error;
12334
13595
  }
12335
- if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
13596
+ if (!this.isPeerReceiveAdmissionOpen(receiveFromHash, replicationLifecycleController, receiveSubscriptionEpoch)) {
12336
13597
  return;
12337
13598
  }
12338
13599
  const segments = replicationSegments.map((x) => x.toReplicationRange());
@@ -12345,7 +13606,7 @@ let SharedLog = (() => {
12345
13606
  signal: replicationLifecycleController.signal,
12346
13607
  })
12347
13608
  .catch((error) => this.handleReplicationLifecycleSendError(error, replicationLifecycleController));
12348
- if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
13609
+ if (!this.isPeerReceiveAdmissionOpen(receiveFromHash, replicationLifecycleController, receiveSubscriptionEpoch)) {
12349
13610
  return;
12350
13611
  }
12351
13612
  // for backwards compatibility (v8) remove this when we are sure that all nodes are v9+
@@ -12380,14 +13641,21 @@ let SharedLog = (() => {
12380
13641
  // (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
12381
13642
  const from = context.from;
12382
13643
  const fromHash = from.hashcode();
12383
- if (this._replicationInfoBlockedPeers.has(fromHash)) {
13644
+ if (!this.isReplicationLifecycleActive(receiveReplicationLifecycleController) ||
13645
+ !this.isCurrentReplicationInfoReceiveEpoch(receiveFromHash, receiveReplicationInfoReceiveEpoch) ||
13646
+ this._replicationInfoBlockedPeers.has(fromHash)) {
12384
13647
  return;
12385
13648
  }
12386
13649
  const messageTimestamp = context.message.header.timestamp;
13650
+ releasePeerReceiveLease?.();
13651
+ releasePeerReceiveLease = undefined;
12387
13652
  await this.withReplicationInfoApplyQueue(fromHash, async () => {
12388
13653
  try {
12389
13654
  // The peer may have unsubscribed after this message was queued.
12390
- if (this._replicationInfoBlockedPeers.has(fromHash)) {
13655
+ if (!this.isReplicationLifecycleActive(receiveReplicationLifecycleController) ||
13656
+ !this.isCurrentSubscriptionEpoch(fromHash, receiveSubscriptionEpoch) ||
13657
+ !this.isCurrentReplicationInfoReceiveEpoch(fromHash, receiveReplicationInfoReceiveEpoch) ||
13658
+ this._replicationInfoBlockedPeers.has(fromHash)) {
12391
13659
  return;
12392
13660
  }
12393
13661
  // Process in-order to avoid races where repeated reset messages arrive
@@ -12422,24 +13690,47 @@ let SharedLog = (() => {
12422
13690
  });
12423
13691
  }
12424
13692
  else if (msg instanceof StoppedReplicating) {
12425
- if (context.from.equals(this.node.identity.publicKey)) {
13693
+ const from = context.from;
13694
+ const segmentIds = msg.segmentIds;
13695
+ if (from.equals(this.node.identity.publicKey)) {
12426
13696
  return;
12427
13697
  }
12428
- const fromHash = context.from.hashcode();
12429
- if (this._replicationInfoBlockedPeers.has(fromHash)) {
13698
+ const fromHash = from.hashcode();
13699
+ if (!this.isReplicationLifecycleActive(receiveReplicationLifecycleController) ||
13700
+ !this.isCurrentReplicationInfoReceiveEpoch(receiveFromHash, receiveReplicationInfoReceiveEpoch) ||
13701
+ this._replicationInfoBlockedPeers.has(fromHash)) {
12430
13702
  return;
12431
13703
  }
12432
- this._replicatorLivenessFailures.delete(fromHash);
12433
- const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(msg.segmentIds, context.from);
12434
- await this.removeReplicationRanges(rangesToRemove, context.from);
12435
- const timestamp = BigInt(+new Date());
12436
- for (const range of rangesToRemove) {
12437
- this.replicationChangeDebounceFn.add({
12438
- range,
12439
- type: "removed",
12440
- timestamp,
12441
- });
12442
- }
13704
+ const messageTimestamp = context.message.header.timestamp;
13705
+ releasePeerReceiveLease?.();
13706
+ releasePeerReceiveLease = undefined;
13707
+ await this.withReplicationInfoApplyQueue(fromHash, async () => {
13708
+ if (!this.isReplicationLifecycleActive(receiveReplicationLifecycleController) ||
13709
+ !this.isCurrentSubscriptionEpoch(fromHash, receiveSubscriptionEpoch) ||
13710
+ !this.isCurrentReplicationInfoReceiveEpoch(fromHash, receiveReplicationInfoReceiveEpoch) ||
13711
+ this._replicationInfoBlockedPeers.has(fromHash)) {
13712
+ return;
13713
+ }
13714
+ const previousTimestamp = this.latestReplicationInfoMessage.get(fromHash);
13715
+ if (previousTimestamp && previousTimestamp > messageTimestamp) {
13716
+ return;
13717
+ }
13718
+ this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
13719
+ this._replicatorLivenessFailures.delete(fromHash);
13720
+ if (this.closed) {
13721
+ return;
13722
+ }
13723
+ const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(segmentIds, from);
13724
+ await this.removeReplicationRanges(rangesToRemove, from);
13725
+ const timestamp = BigInt(+new Date());
13726
+ for (const range of rangesToRemove) {
13727
+ this.replicationChangeDebounceFn.add({
13728
+ range,
13729
+ type: "removed",
13730
+ timestamp,
13731
+ });
13732
+ }
13733
+ });
12443
13734
  }
12444
13735
  else {
12445
13736
  throw new Error("Unexpected message");
@@ -12485,6 +13776,8 @@ let SharedLog = (() => {
12485
13776
  // Every return and every locally swallowed receive error passes this
12486
13777
  // boundary. Release a native wire stash exactly once first, then surface
12487
13778
  // any durable mutation poison that arose while handling the message.
13779
+ releasePeerReceiveLease?.();
13780
+ releasePeerReceiveLease = undefined;
12488
13781
  this.throwIfNativeDurableCommitFailed();
12489
13782
  }
12490
13783
  }
@@ -12960,12 +14253,16 @@ let SharedLog = (() => {
12960
14253
  }
12961
14254
  async waitForLeaderSelection(waitFor, options, checkLeaders) {
12962
14255
  const timeout = options.timeout ?? this.waitForReplicatorTimeout;
14256
+ const closeSignal = this._closeController.signal;
14257
+ const replicationLifecycleSignal = this._replicationLifecycleController?.signal;
12963
14258
  return new Promise((resolve, reject) => {
12964
14259
  let settled = false;
14260
+ const checks = new Set();
12965
14261
  const removeListeners = () => {
12966
14262
  this.events.removeEventListener("replication:change", roleListener);
12967
14263
  this.events.removeEventListener("replicator:mature", roleListener);
12968
- this._closeController.signal.removeEventListener("abort", abortListener);
14264
+ closeSignal.removeEventListener("abort", abortListener);
14265
+ replicationLifecycleSignal?.removeEventListener("abort", abortListener);
12969
14266
  };
12970
14267
  const settleResolve = (value) => {
12971
14268
  if (settled)
@@ -12973,7 +14270,10 @@ let SharedLog = (() => {
12973
14270
  settled = true;
12974
14271
  removeListeners();
12975
14272
  clearTimeout(timer);
12976
- resolve(value);
14273
+ // Leader planning may persist coordinates. Keep the caller (and any
14274
+ // receive lease it owns) alive until checks admitted before this
14275
+ // timeout/abort have finished their local side effects.
14276
+ void Promise.allSettled([...checks]).then(() => resolve(value));
12977
14277
  };
12978
14278
  const settleReject = (error) => {
12979
14279
  if (settled)
@@ -12981,7 +14281,7 @@ let SharedLog = (() => {
12981
14281
  settled = true;
12982
14282
  removeListeners();
12983
14283
  clearTimeout(timer);
12984
- reject(error);
14284
+ void Promise.allSettled([...checks]).then(() => reject(error));
12985
14285
  };
12986
14286
  const abortListener = () => {
12987
14287
  settleResolve(false);
@@ -13010,16 +14310,30 @@ let SharedLog = (() => {
13010
14310
  settleResolve(leaders);
13011
14311
  };
13012
14312
  const runCheck = () => {
13013
- void check().catch((error) => {
14313
+ if (settled)
14314
+ return;
14315
+ let running;
14316
+ running = check()
14317
+ .catch((error) => {
13014
14318
  settleReject(error);
13015
- });
14319
+ })
14320
+ .finally(() => checks.delete(running));
14321
+ checks.add(running);
13016
14322
  };
13017
14323
  const roleListener = () => {
13018
14324
  runCheck();
13019
14325
  };
13020
14326
  this.events.addEventListener("replication:change", roleListener);
13021
14327
  this.events.addEventListener("replicator:mature", roleListener);
13022
- this._closeController.signal.addEventListener("abort", abortListener);
14328
+ closeSignal.addEventListener("abort", abortListener);
14329
+ replicationLifecycleSignal?.addEventListener("abort", abortListener);
14330
+ // AbortSignal does not replay an abort event to listeners added after it
14331
+ // fired. Recheck after registration so work started concurrently with the
14332
+ // terminal fence cannot wait for the full leader-selection timeout.
14333
+ if (closeSignal.aborted || replicationLifecycleSignal?.aborted) {
14334
+ abortListener();
14335
+ return;
14336
+ }
13023
14337
  runCheck();
13024
14338
  });
13025
14339
  }
@@ -15054,6 +16368,40 @@ let SharedLog = (() => {
15054
16368
  }
15055
16369
  });
15056
16370
  }
16371
+ async drainReplicationInfoApplyQueues() {
16372
+ for (;;) {
16373
+ const tails = [...(this._replicationInfoApplyQueueByPeer?.values() ?? [])];
16374
+ if (tails.length === 0) {
16375
+ return;
16376
+ }
16377
+ await Promise.allSettled(tails);
16378
+ // Queue cleanup runs in `finally`; give it a microtask before checking for
16379
+ // tails admitted while the previous snapshot was settling.
16380
+ await Promise.resolve();
16381
+ }
16382
+ }
16383
+ advanceReplicationInfoReceiveEpoch(peerHash) {
16384
+ const next = {};
16385
+ this._replicationInfoReceiveEpochByPeer.set(peerHash, next);
16386
+ return next;
16387
+ }
16388
+ getReplicationInfoReceiveEpoch(peerHash) {
16389
+ return this._replicationInfoReceiveEpochByPeer.get(peerHash) ?? null;
16390
+ }
16391
+ isCurrentReplicationInfoReceiveEpoch(peerHash, epoch) {
16392
+ return this.getReplicationInfoReceiveEpoch(peerHash) === epoch;
16393
+ }
16394
+ advanceSubscriptionEpoch(peerHash) {
16395
+ const next = {};
16396
+ this._subscriptionEpochByPeer.set(peerHash, next);
16397
+ return next;
16398
+ }
16399
+ getSubscriptionEpoch(peerHash) {
16400
+ return this._subscriptionEpochByPeer.get(peerHash) ?? null;
16401
+ }
16402
+ isCurrentSubscriptionEpoch(peerHash, epoch) {
16403
+ return this.getSubscriptionEpoch(peerHash) === epoch;
16404
+ }
15057
16405
  cancelReplicationInfoRequests(peerHash) {
15058
16406
  const state = this._replicationInfoRequestByPeer.get(peerHash);
15059
16407
  if (!state)
@@ -15118,7 +16466,7 @@ let SharedLog = (() => {
15118
16466
  };
15119
16467
  tick();
15120
16468
  }
15121
- async handleSubscriptionChange(publicKey, topics, subscribed) {
16469
+ async handleSubscriptionChange(publicKey, topics, subscribed, subscriptionEpoch) {
15122
16470
  if (!topics.includes(this.topic)) {
15123
16471
  return;
15124
16472
  }
@@ -15128,30 +16476,91 @@ let SharedLog = (() => {
15128
16476
  return;
15129
16477
  }
15130
16478
  const peerHash = publicKey.hashcode();
16479
+ const expectedSubscriptionEpoch = subscriptionEpoch ?? this.advanceSubscriptionEpoch(peerHash);
16480
+ const ownsSubscriptionEpoch = () => this.isCurrentSubscriptionEpoch(peerHash, expectedSubscriptionEpoch);
16481
+ if (!ownsSubscriptionEpoch()) {
16482
+ return;
16483
+ }
16484
+ if (subscribed) {
16485
+ const pendingOpeningCapabilities = this._openingSyncCapabilitiesByPeer.get(peerHash);
16486
+ if (pendingOpeningCapabilities &&
16487
+ pendingOpeningCapabilities.epoch !== expectedSubscriptionEpoch) {
16488
+ this._openingSyncCapabilitiesByPeer.delete(peerHash);
16489
+ }
16490
+ this._subscriptionOpeningEpochByPeer.set(peerHash, expectedSubscriptionEpoch);
16491
+ // Fence new messages immediately, drain handlers admitted by the previous
16492
+ // subscription, then wait behind every queued replication mutation. A
16493
+ // reconnect must not inherit metadata or ranges from the old connection.
16494
+ try {
16495
+ this._replicationInfoBlockedPeers.add(peerHash);
16496
+ await this.drainPeerReceiveHandlers(peerHash);
16497
+ await this.withReplicationInfoApplyQueue(peerHash, async () => { });
16498
+ if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
16499
+ !ownsSubscriptionEpoch()) {
16500
+ return;
16501
+ }
16502
+ // The timestamp watermark belongs to the previous subscription epoch.
16503
+ // Sender clocks are not synchronized, so carrying a local unsubscribe
16504
+ // timestamp forward could reject every valid announcement after reconnect.
16505
+ this.latestReplicationInfoMessage.delete(peerHash);
16506
+ this._pendingReplicatorLeaveByPeer.delete(peerHash);
16507
+ const openingCapabilities = this._openingSyncCapabilitiesByPeer.get(peerHash);
16508
+ if (openingCapabilities?.epoch === expectedSubscriptionEpoch) {
16509
+ this._peerSyncCapabilities.set(peerHash, openingCapabilities.capabilities);
16510
+ this._openingSyncCapabilitiesByPeer.delete(peerHash);
16511
+ }
16512
+ this._replicationInfoBlockedPeers.delete(peerHash);
16513
+ }
16514
+ finally {
16515
+ if (this._openingSyncCapabilitiesByPeer.get(peerHash)?.epoch ===
16516
+ expectedSubscriptionEpoch) {
16517
+ this._openingSyncCapabilitiesByPeer.delete(peerHash);
16518
+ }
16519
+ if (this._subscriptionOpeningEpochByPeer.get(peerHash) ===
16520
+ expectedSubscriptionEpoch) {
16521
+ this._subscriptionOpeningEpochByPeer.delete(peerHash);
16522
+ }
16523
+ }
16524
+ }
15131
16525
  if (!subscribed) {
16526
+ this._subscriptionOpeningEpochByPeer.delete(peerHash);
16527
+ this._openingSyncCapabilitiesByPeer.delete(peerHash);
15132
16528
  this._replicationInfoBlockedPeers.add(peerHash);
16529
+ const disconnectedJoinWarmupGeneration = this._joinWarmupGenerationByTarget.get(peerHash) ?? null;
16530
+ this.cancelJoinWarmupTarget(peerHash);
15133
16531
  const now = BigInt(+new Date());
15134
16532
  const previous = this.latestReplicationInfoMessage.get(peerHash);
15135
16533
  if (!previous || previous < now) {
15136
16534
  this.latestReplicationInfoMessage.set(peerHash, now);
15137
16535
  }
15138
- const wasReplicator = this.uniqueReplicators.has(peerHash);
16536
+ let removed = false;
15139
16537
  try {
15140
16538
  // Unsubscribe can race with the peer's final replication reset message.
15141
16539
  // Proactively evict its ranges so leader selection doesn't keep stale owners.
15142
- await this.removeReplicator(publicKey, { noEvent: true });
16540
+ removed = await this.removeReplicator(publicKey, {
16541
+ cleanupIfSubscriptionSuperseded: true,
16542
+ expectedJoinWarmupGeneration: disconnectedJoinWarmupGeneration,
16543
+ noEvent: true,
16544
+ onRemoved: ({ wasReplicator }) => {
16545
+ if (wasReplicator) {
16546
+ this._pendingReplicatorLeaveByPeer.add(peerHash);
16547
+ }
16548
+ },
16549
+ replicationLifecycleController,
16550
+ subscriptionEpoch: expectedSubscriptionEpoch,
16551
+ });
15143
16552
  }
15144
16553
  catch (error) {
15145
16554
  if (!isNotStartedError(error)) {
15146
16555
  throw error;
15147
16556
  }
15148
16557
  }
15149
- if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
16558
+ if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
16559
+ !ownsSubscriptionEpoch() ||
16560
+ !removed) {
15150
16561
  return;
15151
16562
  }
15152
- this._replicatorJoinEmitted.delete(peerHash);
15153
- this.cleanupPeerDisconnectTracking(peerHash);
15154
- if (wasReplicator) {
16563
+ if (this._pendingReplicatorLeaveByPeer.delete(peerHash)) {
15155
16564
  this.events.dispatchEvent(new CustomEvent("replicator:leave", {
15156
16565
  detail: { publicKey },
15157
16566
  }));
@@ -15185,7 +16594,8 @@ let SharedLog = (() => {
15185
16594
  }
15186
16595
  throw error;
15187
16596
  }
15188
- if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
16597
+ if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
16598
+ !ownsSubscriptionEpoch()) {
15189
16599
  return;
15190
16600
  }
15191
16601
  if (replicationSegments.length > 0) {
@@ -15197,7 +16607,8 @@ let SharedLog = (() => {
15197
16607
  signal: replicationLifecycleController.signal,
15198
16608
  })
15199
16609
  .catch((error) => this.handleReplicationLifecycleSendError(error, replicationLifecycleController));
15200
- if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
16610
+ if (!this.isReplicationLifecycleActive(replicationLifecycleController) ||
16611
+ !ownsSubscriptionEpoch()) {
15201
16612
  return;
15202
16613
  }
15203
16614
  if (this.v8Behaviour) {
@@ -15218,7 +16629,8 @@ let SharedLog = (() => {
15218
16629
  // Request the remote peer's replication info. This makes joins resilient to
15219
16630
  // timing-sensitive delivery/order issues where we may miss their initial
15220
16631
  // replication announcement.
15221
- if (this.isReplicationLifecycleActive(replicationLifecycleController)) {
16632
+ if (this.isReplicationLifecycleActive(replicationLifecycleController) &&
16633
+ ownsSubscriptionEpoch()) {
15222
16634
  this.scheduleReplicationInfoRequests(publicKey, replicationLifecycleController);
15223
16635
  }
15224
16636
  }
@@ -15532,12 +16944,18 @@ let SharedLog = (() => {
15532
16944
  if (this.closed) {
15533
16945
  return;
15534
16946
  }
15535
- await this.log.trim();
15536
16947
  const batchedChanges = Array.isArray(changeOrChanges[0])
15537
16948
  ? changeOrChanges
15538
16949
  : [changeOrChanges];
15539
16950
  const changes = batchedChanges.flat();
15540
16951
  const selfHash = this.node.identity.publicKey.hashcode();
16952
+ const joinWarmupGenerations = new Map();
16953
+ for (const change of changes) {
16954
+ if (change.type === "added" && change.range.hash !== selfHash) {
16955
+ joinWarmupGenerations.set(change.range.hash, this.getJoinWarmupGeneration(change.range.hash));
16956
+ }
16957
+ }
16958
+ await this.log.trim();
15541
16959
  // On removed ranges (peer leaves / shrink), gid-level history can hide
15542
16960
  // per-entry gaps. Force a fresh delivery pass for reassigned entries.
15543
16961
  const forceFreshDelivery = changes.some((change) => change.type === "removed");
@@ -15597,6 +17015,10 @@ let SharedLog = (() => {
15597
17015
  ? changes.filter((change) => !(change.range.hash === selfHash &&
15598
17016
  (change.type === "added" || change.type === "replaced")))
15599
17017
  : changes;
17018
+ const isCurrentJoinWarmupTarget = (target) => warmupPeers.has(target) &&
17019
+ this._joinWarmupGenerationByTarget.get(target) ===
17020
+ joinWarmupGenerations.get(target);
17021
+ const areJoinWarmupGenerationsCurrent = () => [...warmupPeers].every(isCurrentJoinWarmupTarget);
15600
17022
  try {
15601
17023
  const uncheckedDeliver = new Map();
15602
17024
  const flushUncheckedDeliverTarget = (target) => {
@@ -15605,6 +17027,10 @@ let SharedLog = (() => {
15605
17027
  return;
15606
17028
  }
15607
17029
  const isWarmupTarget = warmupPeers.has(target);
17030
+ if (isWarmupTarget && !isCurrentJoinWarmupTarget(target)) {
17031
+ uncheckedDeliver.delete(target);
17032
+ return;
17033
+ }
15608
17034
  const mode = forceFreshDelivery
15609
17035
  ? "churn"
15610
17036
  : isWarmupTarget
@@ -15624,6 +17050,9 @@ let SharedLog = (() => {
15624
17050
  uncheckedDeliver.delete(target);
15625
17051
  };
15626
17052
  const queueUncheckedDeliver = (target, entry) => {
17053
+ if (warmupPeers.has(target) && !isCurrentJoinWarmupTarget(target)) {
17054
+ return;
17055
+ }
15627
17056
  churnRepairPeers.add(target);
15628
17057
  let set = uncheckedDeliver.get(target);
15629
17058
  if (!set) {
@@ -15642,7 +17071,9 @@ let SharedLog = (() => {
15642
17071
  for await (const entryReplicated of toRebalance(immediateRebalanceChanges, this.entryCoordinatesIndex, this.recentlyRebalanced, {
15643
17072
  forceFresh: forceFreshDelivery || useJoinWarmupFastPath,
15644
17073
  })) {
15645
- if (this.closed) {
17074
+ if (this.closed ||
17075
+ (useJoinWarmupFastPath &&
17076
+ !areJoinWarmupGenerationsCurrent())) {
15646
17077
  break;
15647
17078
  }
15648
17079
  if (useJoinWarmupFastPath) {
@@ -15659,7 +17090,9 @@ let SharedLog = (() => {
15659
17090
  }
15660
17091
  const candidatePeers = new Set([selfHash]);
15661
17092
  for (const target of warmupPeers) {
15662
- candidatePeers.add(target);
17093
+ if (isCurrentJoinWarmupTarget(target)) {
17094
+ candidatePeers.add(target);
17095
+ }
15663
17096
  }
15664
17097
  if (oldPeersSet) {
15665
17098
  for (const oldPeer of oldPeersSet) {
@@ -15671,6 +17104,9 @@ let SharedLog = (() => {
15671
17104
  candidates: candidatePeers,
15672
17105
  persist: false,
15673
17106
  });
17107
+ if (!areJoinWarmupGenerationsCurrent()) {
17108
+ continue;
17109
+ }
15674
17110
  if (oldPeersSet) {
15675
17111
  for (const oldPeer of oldPeersSet) {
15676
17112
  if (!currentPeers.has(oldPeer)) {
@@ -15679,11 +17115,11 @@ let SharedLog = (() => {
15679
17115
  }
15680
17116
  }
15681
17117
  for (const [peer] of currentPeers) {
15682
- if (warmupPeers.has(peer)) {
15683
- this.markRepairSweepOptimisticPeer(entryReplicated.gid, peer);
17118
+ if (isCurrentJoinWarmupTarget(peer)) {
17119
+ this.markRepairSweepOptimisticPeer(entryReplicated.gid, peer, joinWarmupGenerations.get(peer));
15684
17120
  }
15685
17121
  }
15686
- const authoritativePeers = [...currentPeers.keys()].filter((peer) => !warmupPeers.has(peer) &&
17122
+ const authoritativePeers = [...currentPeers.keys()].filter((peer) => !isCurrentJoinWarmupTarget(peer) &&
15687
17123
  !this.hasPendingRepairSweepOptimisticPeer(entryReplicated.gid, peer));
15688
17124
  this.addPeersToGidPeerHistory(entryReplicated.gid, authoritativePeers, true);
15689
17125
  if (!currentPeers.has(selfHash)) {
@@ -15730,8 +17166,8 @@ let SharedLog = (() => {
15730
17166
  }
15731
17167
  }
15732
17168
  for (const [peer] of currentPeers) {
15733
- if (addedPeers.has(peer)) {
15734
- this.markRepairSweepOptimisticPeer(entryReplicated.gid, peer);
17169
+ if (isCurrentJoinWarmupTarget(peer)) {
17170
+ this.markRepairSweepOptimisticPeer(entryReplicated.gid, peer, joinWarmupGenerations.get(peer));
15735
17171
  }
15736
17172
  }
15737
17173
  const authoritativePeers = [...currentPeers.keys()].filter((peer) => !addedPeers.has(peer) &&
@@ -15771,6 +17207,7 @@ let SharedLog = (() => {
15771
17207
  this.scheduleRepairSweep({
15772
17208
  mode: "join-warmup",
15773
17209
  peers,
17210
+ joinWarmupGenerations,
15774
17211
  });
15775
17212
  }, 250);
15776
17213
  timer.unref?.();
@@ -15827,6 +17264,7 @@ let SharedLog = (() => {
15827
17264
  return;
15828
17265
  }
15829
17266
  const fromHash = evt.detail.from.hashcode();
17267
+ const subscriptionEpoch = this.advanceSubscriptionEpoch(fromHash);
15830
17268
  this._replicationInfoBlockedPeers.add(fromHash);
15831
17269
  this._recentRepairDispatch.delete(fromHash);
15832
17270
  // Keep a per-peer timestamp watermark when we observe an unsubscribe. This
@@ -15838,17 +17276,19 @@ let SharedLog = (() => {
15838
17276
  this.latestReplicationInfoMessage.set(fromHash, now);
15839
17277
  }
15840
17278
  this.invalidateSharedLogTopicSubscribersCache();
15841
- return this.handleSubscriptionChange(evt.detail.from, evt.detail.topics, false);
17279
+ return this.handleSubscriptionChange(evt.detail.from, evt.detail.topics, false, subscriptionEpoch);
15842
17280
  }
15843
17281
  async _onSubscription(evt) {
15844
17282
  logger.trace(`New peer '${evt.detail.from.hashcode()}' connected to '${JSON.stringify(evt.detail.topics.map((x) => x))}'`);
15845
17283
  if (!evt.detail.topics.includes(this.topic)) {
15846
17284
  return;
15847
17285
  }
17286
+ const fromHash = evt.detail.from.hashcode();
17287
+ const subscriptionEpoch = this.advanceSubscriptionEpoch(fromHash);
15848
17288
  this.remoteBlocks.onReachable(evt.detail.from);
15849
- this._replicationInfoBlockedPeers.delete(evt.detail.from.hashcode());
17289
+ this._replicationInfoBlockedPeers.add(fromHash);
15850
17290
  this.invalidateSharedLogTopicSubscribersCache();
15851
- await this.handleSubscriptionChange(evt.detail.from, evt.detail.topics, true);
17291
+ await this.handleSubscriptionChange(evt.detail.from, evt.detail.topics, true, subscriptionEpoch);
15852
17292
  }
15853
17293
  async rebalanceParticipation() {
15854
17294
  // update more participation rate to converge to the average expected rate or bounded by
@@ -15998,7 +17438,7 @@ let SharedLog = (() => {
15998
17438
  const ih = this._pendingIHave.get(entry.hash);
15999
17439
  if (ih) {
16000
17440
  ih.clear();
16001
- ih.callback(entry);
17441
+ this.runPendingIHaveCallback(ih, entry);
16002
17442
  }
16003
17443
  this.syncronizer.onEntryAdded(entry);
16004
17444
  }
@@ -16010,7 +17450,7 @@ let SharedLog = (() => {
16010
17450
  }
16011
17451
  const entry = materializeEntry();
16012
17452
  ih.clear();
16013
- ih.callback(entry);
17453
+ this.runPendingIHaveCallback(ih, entry);
16014
17454
  this.syncronizer.onEntryAdded(entry);
16015
17455
  return;
16016
17456
  }