@peerbit/shared-log 13.2.16 → 13.2.18

package/dist/src/index.js

@@ -1768,6 +1768,13 @@ let SharedLog = (() => {
         // before a liveness eviction but reach the per-peer apply lane after it. Unlike
         // message timestamps, these tokens never compare clocks from different peers.
         _replicationInfoReceiveEpochByPeer;
+        // Receive-side ownership plans may span lower-log joins that invoke user code.
+        // Increment synchronously with leader-cache invalidation so the handler can
+        // detect whether its pre-join plan needs one fresh post-persist audit.
+        _receiveOwnershipRevision = 0;
+        // Count ownership-changing range mutations from queue admission through
+        // settlement, including mutations already pending when a receive starts.
+        _receiveOwnershipMutationAdmissions = 0;
         // Subscription callbacks can overlap because removing a replicator mutates the
         // replication index asynchronously. Keep that lifecycle separate from message
         // timestamps so a reconnect can synchronously revoke an older unsubscribe.
@@ -3778,8 +3785,13 @@ let SharedLog = (() => {
             this.invalidateTopicSubscribersCache(this.topic, this.rpc.topic);
         }
         invalidateLeaderSelectionContextCache() {
+            this._receiveOwnershipRevision++;
             this._leaderSelectionContextCache = undefined;
         }
+        isReceiveOwnershipSnapshotStable(revision) {
+            return (this._receiveOwnershipMutationAdmissions === 0 &&
+                this._receiveOwnershipRevision === revision);
+        }
         canCacheLeaderSelectionContext(options) {
             return options?.roleAge == null && options?.candidates == null;
         }
@@ -5033,7 +5045,7 @@ let SharedLog = (() => {
                     let deleted = [];
                     let ownerHasRanges = false;
                     let mutationError;
-                    const mutationCommitted = await this.withReplicationRangeMutationQueue(async () => {
+                    const mutationCommitted = await this.withReceiveOwnershipMutationQueue(async () => {
                         if (!ownsReplicationOwnershipLifecycle() ||
                             !ownsReplicationLifecycle() ||
                             !ownsSubscriptionEpoch() ||
@@ -5201,7 +5213,7 @@ let SharedLog = (() => {
         }
         removeReplicationRanges(ranges, from, options, replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle()) {
             this.throwIfReplicationOwnershipLifecycleInactive(replicationOwnershipLifecycleController);
-            return this.withReplicationRangeMutationQueue(() => this.removeReplicationRangesUnlocked(ranges, from, options), replicationOwnershipLifecycleController);
+            return this.withReceiveOwnershipMutationQueue(() => this.removeReplicationRangesUnlocked(ranges, from, options), replicationOwnershipLifecycleController);
         }
         async removeReplicationRangesUnlocked(ranges, from, options) {
             if (ranges.length === 0) {
@@ -5303,7 +5315,7 @@ let SharedLog = (() => {
                 return undefined;
             }
             this.throwIfReplicationOwnershipLifecycleInactive(replicationOwnershipLifecycleController);
-            return this.withReplicationRangeMutationQueue(() => this.addReplicationRangeUnlocked(ranges, from, options, replicationOwnershipLifecycleController), replicationOwnershipLifecycleController);
+            return this.withReceiveOwnershipMutationQueue(() => this.addReplicationRangeUnlocked(ranges, from, options, replicationOwnershipLifecycleController), replicationOwnershipLifecycleController);
         }
         async addReplicationRangeUnlocked(ranges, from, { reset, checkDuplicates, timestamp: ts, rebalance, allowLegacyOrderedReplacementPairs, onConfirmedDurableStateChanged, shouldApply, } = {}, replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle()) {
             this.validateReplicationRangeAnnouncement(ranges);
@@ -7435,6 +7447,10 @@ let SharedLog = (() => {
                 return;
             }
             const selfHash = this.node.identity.publicKey.hashcode();
+            const freshReceiveRoleAge = options?.freshReceiveOwnerAudit
+                ? await this.getDefaultMinRoleAge()
+                : undefined;
+            this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
             const plans = new Array(entries.length);
             const leaderItems = [];
             const leaderItemIndexes = [];
@@ -7443,7 +7459,9 @@ let SharedLog = (() => {
                 const entry = entries[i];
                 const replicas = options?.decodedReplicaCounts?.get(entry.hash) ??
                     decodeReplicas(entry).getValue(this);
-                const reusablePlan = options?.reusableLeaderPlans?.get(entry.hash);
+                const reusablePlan = options?.freshReceiveOwnerAudit
+                    ? undefined
+                    : options?.reusableLeaderPlans?.get(entry.hash);
                 if (reusablePlan && reusablePlan.replicas === replicas) {
                     plans[i] = reusablePlan.plan;
                     reusableLeaderPlanHits++;
@@ -7452,7 +7470,9 @@ let SharedLog = (() => {
                 leaderItems.push({
                     entry,
                     replicas,
-                    options: { roleAge: 0, persist: false },
+                    options: options?.freshReceiveOwnerAudit
+                        ? { roleAge: freshReceiveRoleAge, persist: false }
+                        : { roleAge: 0, persist: false },
                 });
                 leaderItemIndexes.push(i);
             }
@@ -7502,6 +7522,7 @@ let SharedLog = (() => {
             const loopStartedAt = syncProfileStart(options?.profile);
             let enqueuedPrune = 0;
             let cancelledLocalLeader = 0;
+            let localLeaderResults = 0;
             for (let i = 0; i < entries.length; i++) {
                 if (!this.isRepairLifecycleActive(ownershipLifecycleController)) {
                     continue;
@@ -7509,9 +7530,12 @@ let SharedLog = (() => {
                 const entry = entries[i];
                 const leaders = plans[i]?.leaders ?? new Map();
                 if (leaders.has(selfHash)) {
-                    await this.cancelCheckedPruneForLocalLeader(entry.hash);
-                    this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
-                    cancelledLocalLeader++;
+                    if (!options?.preserveExistingPruneOnLocalResult) {
+                        await this.cancelCheckedPruneForLocalLeader(entry.hash);
+                        this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
+                        cancelledLocalLeader++;
+                    }
+                    localLeaderResults++;
                     continue;
                 }
                 if (this._checkedPrune.hasPendingDelete(entry.hash)) {
@@ -7534,7 +7558,7 @@ let SharedLog = (() => {
                 entries: entries.length,
                 count: enqueuedPrune,
                 messages: 1,
-                details: { cancelledLocalLeader },
+                details: { cancelledLocalLeader, localLeaderResults },
             });
         }
         async pruneIndexedEntriesNoLongerLed(options, ownershipLifecycleController = this.captureReplicationOwnershipLifecycle()) {
@@ -7736,9 +7760,11 @@ let SharedLog = (() => {
             const state = checkedPruneCoordinator.getRetry(hash) ??
                 {
                     attempts: 0,
+                    generation: 0,
                     entry: args.entry,
                     leaders: args.leaders,
                 };
+            state.generation++;
             state.entry = args.entry;
             state.leaders = args.leaders;
             if (state.timer)
@@ -7756,6 +7782,7 @@ let SharedLog = (() => {
             const delayMs = Math.min(CHECKED_PRUNE_RETRY_MAX_DELAY_MS, 1_000 * 2 ** (attempt - 1) + jitterMs);
             state.attempts = attempt;
             const timer = setTimeout(() => {
+                const generation = state.generation;
                 const run = async () => {
                     const st = checkedPruneCoordinator.getRetry(hash);
                     if (st !== state || state.timer !== timer || !isCurrent()) {
@@ -7764,6 +7791,7 @@ let SharedLog = (() => {
                     state.timer = undefined;
                     const isExactAttempt = () => isCurrent() &&
                         checkedPruneCoordinator.getRetry(hash) === state &&
+                        state.generation === generation &&
                         state.timer === undefined;
                     if (checkedPruneCoordinator.hasPendingDelete(hash))
                         return;
@@ -7800,21 +7828,27 @@ let SharedLog = (() => {
                         !checkedPruneCoordinator.hasPendingDelete(hash)) {
                         // A keep policy is terminal for this background prune. Do not
                         // retain a retry record with neither a timer nor pending work.
-                        checkedPruneCoordinator.clearRetry(hash);
+                        checkedPruneCoordinator.clearRetry(hash, {
+                            state,
+                            generation,
+                        });
                     }
                 };
                 void run().catch((error) => {
                     if (isCurrent() && !isNotStartedError(error)) {
                         logger.error(error);
-                        const retry = checkedPruneCoordinator.getRetry(hash);
-                        if (retry === state &&
-                            !retry.timer &&
+                        if (checkedPruneCoordinator.getRetry(hash) === state &&
+                            state.generation === generation &&
+                            !state.timer &&
                             !checkedPruneCoordinator.hasPendingDelete(hash)) {
                             try {
-                                this.scheduleCheckedPruneRetry({ entry: retry.entry, leaders: retry.leaders }, ownershipLifecycleController);
+                                this.scheduleCheckedPruneRetry({ entry: state.entry, leaders: state.leaders }, ownershipLifecycleController);
                             }
                             catch {
-                                checkedPruneCoordinator.clearRetry(hash);
+                                checkedPruneCoordinator.clearRetry(hash, {
+                                    state,
+                                    generation,
+                                });
                             }
                         }
                     }
@@ -10380,6 +10414,8 @@ let SharedLog = (() => {
             this._checkedPruneRemoveBlocksLocalRangeMutationAdmission = 0;
             this._checkedPruneRemovalCallbackInvocationDepth = 0;
             this._localReplicationRoleGeneration = 0;
+            this._receiveOwnershipRevision = 0;
+            this._receiveOwnershipMutationAdmissions = 0;
             this._pruneRemovesClosing = false;
             this._replicationRangeMutationFailure = undefined;
             this.startRepairLifecycle();
@@ -10709,7 +10745,8 @@ let SharedLog = (() => {
                             continue;
                         }
                         if (checkedPruneLeaders.localLeader) {
-                            const retry = checkedPruneCoordinator.getRetry(hash);
+                            const retryIdentity = checkedPruneCoordinator.getRetryIdentity(hash);
+                            const retry = retryIdentity?.state;
                             await this.cancelCheckedPruneForLocalLeader(hash, {
                                 preserveRetry: retry != null,
                             });
@@ -10719,7 +10756,7 @@ let SharedLog = (() => {
                             if (retry) {
                                 // A delayed confirmation of local ownership is terminal.
                                 // Future ownership mutations will enqueue fresh work.
-                                checkedPruneCoordinator.clearRetry(hash);
+                                checkedPruneCoordinator.clearRetry(hash, retryIdentity);
                             }
                             else {
                                 // A first positive view can be a transient membership
@@ -10760,6 +10797,10 @@ let SharedLog = (() => {
                                 this.scheduleCheckedPruneRetry(value, pruneOwnershipLifecycleController);
                             }
                             catch {
+                                if (value.workToken != null &&
+                                    checkedPruneCoordinator.isCandidateTokenCurrent(hash, value.workToken)) {
+                                    checkedPruneCoordinator.invalidateCandidateToken(hash);
+                                }
                                 checkedPruneCoordinator.clearRetry(hash);
                             }
                         }
@@ -11385,7 +11426,7 @@ let SharedLog = (() => {
         }
         async updateTimestampOfOwnedReplicationRanges(timestamp = +new Date()) {
             const ownershipLifecycleController = this.captureReplicationOwnershipLifecycle();
-            return this.withReplicationRangeMutationQueue(async () => {
+            return this.withReceiveOwnershipMutationQueue(async () => {
                 this.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
                 const all = await this.replicationIndex
                     .iterate({
@@ -13170,6 +13211,8 @@ let SharedLog = (() => {
                 if (!releasePeerReceiveLease) {
                     return;
                 }
+                const receiveOwnershipRevision = this._receiveOwnershipRevision;
+                const receiveOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle();
                 const receiveReplicationInfoReceiveEpoch = this.getReplicationInfoReceiveEpoch(receiveFromHash);
                 if (msg instanceof ResponseRoleMessage) {
                     msg = msg.toReplicationInfoMessage(); // migration
@@ -13377,6 +13420,7 @@ let SharedLog = (() => {
                                 fromIsSelf,
                                 syncProfile,
                                 selection: await getRawPreparedReceiveSelection(heads, hashes),
+                                receiveOwnershipRevision,
                             }),
                         selectPreparedRawReceiveHashes: rawIsRepairHint
                             ? undefined
@@ -13387,6 +13431,7 @@ let SharedLog = (() => {
                                 fromIsSelf,
                                 syncProfile,
                                 selection: await getRawPreparedReceiveSelection(heads, hashes),
+                                receiveOwnershipRevision,
                             }),
                     });
                     if (materializedRawMessage === undefined) {
@@ -13972,6 +14017,7 @@ let SharedLog = (() => {
                             !isReplicating &&
                             !this.keep &&
                             !isRepairHint &&
+                            this.isReceiveOwnershipSnapshotStable(receiveOwnershipRevision) &&
                             receiveGroups.length > 0 &&
                             receiveGroups.every((group) => group.isLeader === false &&
                                 group.fromIsLeader === false &&
@@ -14578,11 +14624,42 @@ let SharedLog = (() => {
                                 confirmedHashes.add(hash);
                             }
                             const checkedPruneStartedAt = syncProfileStart(syncProfile);
-                            await this.pruneJoinedEntriesNoLongerLed(admittedShallowEntries, {
-                                decodedReplicaCounts: receiveReplicaCounts,
-                                reusableLeaderPlans: reusableCoordinatePlans,
-                                profile: syncProfile,
-                            });
+                            const ownershipChangedDuringReceive = !this.isReceiveOwnershipSnapshotStable(receiveOwnershipRevision);
+                            if (ownershipChangedDuringReceive) {
+                                const freshAuditRevision = this._receiveOwnershipRevision;
+                                const armFreshAuditRetry = () => {
+                                    for (const entry of admittedShallowEntries) {
+                                        this.scheduleCheckedPruneRetry({ entry, leaders: new Map() }, receiveOwnershipLifecycleController);
+                                    }
+                                };
+                                try {
+                                    await this.pruneJoinedEntriesNoLongerLed(admittedShallowEntries, {
+                                        decodedReplicaCounts: receiveReplicaCounts,
+                                        freshReceiveOwnerAudit: true,
+                                        preserveExistingPruneOnLocalResult: true,
+                                        profile: syncProfile,
+                                    }, receiveOwnershipLifecycleController);
+                                    this.throwIfReplicationOwnershipLifecycleInactive(receiveOwnershipLifecycleController);
+                                    if (!this.isReceiveOwnershipSnapshotStable(freshAuditRevision)) {
+                                        armFreshAuditRetry();
+                                    }
+                                }
+                                catch {
+                                    // The lower-log and coordinate commits are already durable. A
+                                    // sender retry will filter these hashes as present, so retain a
+                                    // bounded local obligation instead of failing the admitted receive.
+                                    this.throwIfReplicationOwnershipLifecycleInactive(receiveOwnershipLifecycleController);
+                                    armFreshAuditRetry();
+                                }
+                            }
+                            else {
+                                await this.pruneJoinedEntriesNoLongerLed(admittedShallowEntries, {
+                                    decodedReplicaCounts: receiveReplicaCounts,
+                                    preserveExistingPruneOnLocalResult: true,
+                                    reusableLeaderPlans: reusableCoordinatePlans,
+                                    profile: syncProfile,
+                                }, receiveOwnershipLifecycleController);
+                            }
                             if (syncProfile) {
                                 emitSyncProfileDuration(syncProfile, checkedPruneStartedAt, {
                                     name: "sharedLog.receive.checkedPrune",
@@ -17059,13 +17136,16 @@ let SharedLog = (() => {
         }
         async tryFastDropPreparedRawReceive(properties) {
             const backbone = this._nativeBackbone;
+            const ownershipRevision = properties.receiveOwnershipRevision;
             if (!backbone || !this.syncronizer.onReceivedEntryHashes) {
                 return false;
             }
             const receivePlanStartedAt = syncProfileStart(properties.syncProfile);
             const selection = properties.selection ??
                 (await this.planNativePreparedRawReceiveSelection(properties));
-            if (!selection || selection.retainedHashes.length > 0) {
+            if (!selection ||
+                selection.retainedHashes.length > 0 ||
+                !this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
                 return false;
             }
             if (properties.syncProfile) {
@@ -17094,6 +17174,9 @@ let SharedLog = (() => {
                 hashes: selection.droppedHashes,
                 from: properties.from,
             });
+            if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
+                return false;
+            }
             if (properties.syncProfile) {
                 emitSyncProfileDuration(properties.syncProfile, notifyStartedAt, {
                     name: "sharedLog.receive.notifySynchronizer",
@@ -17124,13 +17207,15 @@ let SharedLog = (() => {
             return true;
         }
         async selectNativePreparedRawReceiveHashes(properties) {
+            const ownershipRevision = properties.receiveOwnershipRevision;
             if (!this.syncronizer.onReceivedEntryHashes) {
                 return undefined;
             }
             const receivePlanStartedAt = syncProfileStart(properties.syncProfile);
             const selection = properties.selection ??
                 (await this.planNativePreparedRawReceiveSelection(properties));
-            if (!selection) {
+            if (!selection ||
+                !this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
                 return undefined;
             }
             if (selection.droppedHashes.length === 0) {
@@ -17144,6 +17229,9 @@ let SharedLog = (() => {
                 hashes: selection.droppedHashes,
                 from: properties.from,
             });
+            if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) {
+                return undefined;
+            }
             emitSyncProfileDuration(properties.syncProfile, notifyStartedAt, {
                 name: "sharedLog.receive.notifySynchronizer",
                 component: "shared-log",
@@ -17639,7 +17727,7 @@ let SharedLog = (() => {
             this._checkedPruneRemovalCallbackInvocationDepth--;
             return result;
         }
-        withReplicationRangeMutationQueue(fn, replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle()) {
+        withReplicationRangeMutationQueue(fn, replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle(), options) {
             try {
                 this.throwIfReplicationOwnershipLifecycleInactive(replicationOwnershipLifecycleController);
             }
@@ -17649,6 +17737,15 @@ let SharedLog = (() => {
             if (this._replicationRangeMutationsClosing) {
                 return Promise.reject(new TerminalOperationNotStartedError("Replication range mutations are closing"));
             }
+            let releaseReceiveOwnershipMutationAdmission;
+            if (options?.affectsReceiveOwnership) {
+                this._receiveOwnershipMutationAdmissions++;
+                this.invalidateLeaderSelectionContextCache();
+                releaseReceiveOwnershipMutationAdmission = () => {
+                    this._receiveOwnershipMutationAdmissions--;
+                    this.invalidateLeaderSelectionContextCache();
+                };
+            }
             const run = (this._replicationRangeMutationTail ?? Promise.resolve())
                 .catch(() => {
                 // A failed predecessor must not leave the queue permanently rejected.
@@ -17660,8 +17757,14 @@ let SharedLog = (() => {
                 this.throwIfReplicationOwnershipLifecycleInactive(replicationOwnershipLifecycleController);
                 return fn();
             });
-            this._replicationRangeMutationTail = run.then(() => undefined, () => undefined);
-            return run;
+            const fencedRun = releaseReceiveOwnershipMutationAdmission
+                ? run.finally(releaseReceiveOwnershipMutationAdmission)
+                : run;
+            this._replicationRangeMutationTail = fencedRun.then(() => undefined, () => undefined);
+            return fencedRun;
+        }
+        withReceiveOwnershipMutationQueue(fn, replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle()) {
+            return this.withReplicationRangeMutationQueue(fn, replicationOwnershipLifecycleController, { affectsReceiveOwnership: true });
         }
         acquireReplicationRangeMutationTerminalFence() {
             this._replicationRangeMutationsClosing = true;
@@ -18349,7 +18452,8 @@ let SharedLog = (() => {
                 for (const { hash } of admitted) {
                     const candidate = checkedPruneCoordinator.getRestartCandidate(hash);
                     const pending = checkedPruneCoordinator.getPendingDelete(hash);
-                    const retry = checkedPruneCoordinator.getRetry(hash);
+                    const retryIdentity = checkedPruneCoordinator.getRetryIdentity(hash);
+                    const retry = retryIdentity?.state;
                     const hadQueuedCandidate = this.pruneDebouncedFn.has(hash);
                     const hadInFlightCandidate = checkedPruneCoordinator.hasCandidate(hash);
                     const shouldRestart = candidate != null &&
@@ -18384,7 +18488,7 @@ let SharedLog = (() => {
                         restartCandidates.push({
                             candidate,
                             reservation: checkedPruneCoordinator.reserveRestart(hash),
-                            retry,
+                            retryIdentity,
                         });
                     }
                 }
@@ -18411,13 +18515,12 @@ let SharedLog = (() => {
             finally {
                 admission.barrier.resolve();
                 await admission.observedBarrier;
-                for (const { candidate, reservation, retry, } of admission.restartCandidates) {
+                for (const { candidate, reservation, retryIdentity, } of admission.restartCandidates) {
                     if (this._checkedPrune !== checkedPruneCoordinator ||
                         !this.isRepairLifecycleActive(ownershipLifecycleController)) {
                         checkedPruneCoordinator.cancelRestartReservation(candidate.hash, reservation);
-                        if (retry &&
-                            checkedPruneCoordinator.getRetry(candidate.hash) === retry) {
-                            checkedPruneCoordinator.clearRetry(candidate.hash);
+                        if (retryIdentity) {
+                            checkedPruneCoordinator.clearRetry(candidate.hash, retryIdentity);
                         }
                         continue;
                     }