@peerbit/shared-log 13.1.15 → 13.1.17

package/dist/src/index.js

@@ -44,12 +44,13 @@ import { ClosedError, Program } from "@peerbit/program";
 import { FanoutChannel, waitForSubscribers, } from "@peerbit/pubsub";
 import { SubscriptionEvent, UnsubcriptionEvent, } from "@peerbit/pubsub-interface";
 import { RPC } from "@peerbit/rpc";
-import { ACK_CONTROL_PRIORITY, AcknowledgeDelivery, AnyWhere, createRequestTransportContext, DataMessage, MessageHeader, NotStartedError, SilentDelivery, } from "@peerbit/stream-interface";
+import { ACK_CONTROL_PRIORITY, AcknowledgeDelivery, AnyWhere, BACKGROUND_MESSAGE_PRIORITY, CONVERGENCE_MESSAGE_PRIORITY, createRequestTransportContext, DataMessage, MessageHeader, NotStartedError, SilentDelivery, } from "@peerbit/stream-interface";
 import { AbortError, TimeoutError, debounceAccumulator, debounceFixedInterval, waitFor, } from "@peerbit/time";
 import pDefer, {} from "p-defer";
 import PQueue from "p-queue";
-import { concat } from "uint8arrays";
+import { concat, fromString } from "uint8arrays";
 import { BlocksMessage } from "./blocks.js";
+import { CheckedPruneCoordinator, } from "./checked-prune.js";
 import { CPUUsageIntervalLag } from "./cpu.js";
 import { debouncedAccumulatorMap, } from "./debounce.js";
 import { NoPeersError } from "./errors.js";
@@ -66,7 +67,7 @@ import {} from "./replication-domain.js";
 import { AbsoluteReplicas, AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, MinReplicas, ReplicationPingMessage, ReplicationError, RequestReplicationInfoMessage, ResponseRoleMessage, StoppedReplicating, decodeReplicas, encodeReplicas, maxReplicas, } from "./replication.js";
 import { Observer, Replicator } from "./role.js";
 import { RatelessIBLTSynchronizer } from "./sync/rateless-iblt.js";
-import { ConfirmEntriesMessage, SimpleSyncronizer } from "./sync/simple.js";
+import { ConfirmEntriesMessage, SYNC_MESSAGE_PRIORITY, SimpleSyncronizer, } from "./sync/simple.js";
 import { groupByGid } from "./utils.js";
 const toLocalPublicSignKey = (key) => {
     if (typeof key === "string") {
@@ -238,6 +239,7 @@ export const WAIT_FOR_PRUNE_DELAY = 0;
 const PRUNE_DEBOUNCE_INTERVAL = 500;
 const CHECKED_PRUNE_RESEND_INTERVAL_MIN_MS = 250;
 const CHECKED_PRUNE_RESEND_INTERVAL_MAX_MS = 5_000;
+const CHECKED_PRUNE_BACKGROUND_TIMEOUT_MIN_MS = 120_000;
 const CHECKED_PRUNE_RETRY_MAX_ATTEMPTS = 3;
 const CHECKED_PRUNE_RETRY_MAX_DELAY_MS = 30_000;
 // DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
@@ -426,7 +428,10 @@ let SharedLog = (() => {
         _logProperties;
         _closeController;
         _respondToIHaveTimeout;
-        _pendingDeletes;
+        _checkedPrune;
+        get _pendingDeletes() {
+            return this._checkedPrune.pendingDeletes;
+        }
         _pendingIHave;
         // public key hash to range id to range
         pendingMaturity; // map of peerId to timeout
@@ -453,9 +458,15 @@ let SharedLog = (() => {
         // A fn for debouncing the calls for pruning
         pruneDebouncedFn;
         responseToPruneDebouncedFn;
-        _requestIPruneSent; // tracks entry hash to peer hash for requesting I prune messages
-        _requestIPruneResponseReplicatorSet; // tracks entry hash to peer hash
-        _checkedPruneRetries;
+        get _requestIPruneSent() {
+            return this._checkedPrune.requestIPruneSent;
+        }
+        get _requestIPruneResponseReplicatorSet() {
+            return this._checkedPrune.responseReplicatorSet;
+        }
+        get _checkedPruneRetries() {
+            return this._checkedPrune.retries;
+        }
         replicationChangeDebounceFn;
         _repairRetryTimers;
         _recentRepairDispatch;
@@ -606,7 +617,7 @@ let SharedLog = (() => {
                 header: new MessageHeader({
                     session: 0,
                     mode: new AnyWhere(),
-                    priority: 0,
+                    priority: BACKGROUND_MESSAGE_PRIORITY,
                 }),
             });
             contextMessage.header.timestamp = envelope.timestamp;
@@ -634,7 +645,7 @@ let SharedLog = (() => {
                 header: new MessageHeader({
                     session: 0,
                     mode: new AnyWhere(),
-                    priority: 0,
+                    priority: BACKGROUND_MESSAGE_PRIORITY,
                 }),
             });
             contextMessage.header.timestamp = detail.timestamp;
@@ -1347,7 +1358,7 @@ let SharedLog = (() => {
                 await this.rpc.send(new StoppedReplicating({
                     segmentIds: rangesToUnreplicate.map((x) => x.id),
                 }), {
-                    priority: 1,
+                    priority: CONVERGENCE_MESSAGE_PRIORITY,
                 });
             }
             return rangesToReplicate;
@@ -1374,12 +1385,18 @@ let SharedLog = (() => {
             this.setupRebalanceDebounceFunction(options?.limits?.interval);
         }
         async replicate(rangeOrEntry, options) {
+            const entryRangeId = (entry) => sha256Sync(concat([
+                this.log.id,
+                fromString(entry.hash),
+                fromString(this.node.identity.publicKey.hashcode()),
+            ]));
             let range = undefined;
             if (rangeOrEntry instanceof ReplicationRangeMessage) {
                 range = rangeOrEntry;
             }
             else if (rangeOrEntry instanceof Entry) {
                 range = {
+                    id: entryRangeId(rangeOrEntry),
                     factor: 1,
                     offset: await this.domain.fromEntry(rangeOrEntry),
                     normalized: false,
@@ -1390,6 +1407,7 @@ let SharedLog = (() => {
                 for (const entry of rangeOrEntry) {
                     if (entry instanceof Entry) {
                         ranges.push({
+                            id: entryRangeId(entry),
                             factor: 1,
                             offset: await this.domain.fromEntry(entry),
                             normalized: false,
@@ -1452,7 +1470,7 @@ let SharedLog = (() => {
             const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(segmentIds, this.node.identity.publicKey);
             await this.removeReplicationRanges(rangesToRemove, this.node.identity.publicKey);
             await this.rpc.send(new StoppedReplicating({ segmentIds }), {
-                priority: 1,
+                priority: CONVERGENCE_MESSAGE_PRIORITY,
             });
         }
         async removeReplicator(key, options) {
@@ -1470,7 +1488,7 @@ let SharedLog = (() => {
             if (isMe) {
                 // announce that we are no longer replicating
                 await this.rpc.send(new AllReplicatingSegmentsMessage({ segments: [] }), {
-                    priority: 1,
+                    priority: CONVERGENCE_MESSAGE_PRIORITY,
                 });
             }
             if (options?.noEvent !== true) {
@@ -1833,7 +1851,7 @@ let SharedLog = (() => {
                     }
                     else {
                         await this.rpc.send(message, {
-                            priority: 1,
+                            priority: CONVERGENCE_MESSAGE_PRIORITY,
                         });
                     }
                 }
@@ -2020,7 +2038,7 @@ let SharedLog = (() => {
             for (let i = 0; i < uniqueHashes.length; i += REPAIR_CONFIRMATION_HASH_BATCH_SIZE) {
                 const chunk = uniqueHashes.slice(i, i + REPAIR_CONFIRMATION_HASH_BATCH_SIZE);
                 await this.rpc.send(new ConfirmEntriesMessage({ hashes: chunk }), {
-                    priority: 1,
+                    priority: CONVERGENCE_MESSAGE_PRIORITY,
                     mode: new SilentDelivery({ to: [target], redundancy: 1 }),
                 });
             }
@@ -2029,7 +2047,7 @@ let SharedLog = (() => {
             for await (const message of createExchangeHeadsMessages(this.log, [...entries.keys()])) {
                 message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
                 await this.rpc.send(message, {
-                    priority: 1,
+                    priority: SYNC_MESSAGE_PRIORITY,
                     mode: new SilentDelivery({ to: [target], redundancy: 1 }),
                 });
             }
@@ -2504,28 +2522,19 @@ let SharedLog = (() => {
                         const frontierTargets = this._repairFrontierByMode.get(mode);
                         for (const target of pendingPeersByMode.get(mode) ?? []) {
                             const replacement = nextTargets.get(target);
-                            if (mode === "join-authoritative") {
-                                // Authoritative join repair is receipt-driven: a later sweep can have a
-                                // narrower transient leader view, but it must not forget unconfirmed
-                                // hashes that were already queued for this joiner.
-                                if (replacement && replacement.size > 0) {
-                                    const existing = frontierTargets?.get(target);
-                                    if (existing && existing.size > 0) {
-                                        for (const [hash, entry] of replacement) {
-                                            existing.set(hash, entry);
-                                        }
-                                    }
-                                    else {
-                                        frontierTargets?.set(target, replacement);
+                            // These repairs are receipt-driven: a later sweep can have a narrower
+                            // transient leader view, but it must not forget unconfirmed hashes
+                            // that were already queued for this target.
+                            if (replacement && replacement.size > 0) {
+                                const existing = frontierTargets?.get(target);
+                                if (existing && existing.size > 0) {
+                                    for (const [hash, entry] of replacement) {
+                                        existing.set(hash, entry);
                                     }
                                 }
-                                continue;
-                            }
-                            if (replacement && replacement.size > 0) {
-                                frontierTargets?.set(target, replacement);
-                            }
-                            else {
-                                frontierTargets?.delete(target);
+                                else {
+                                    frontierTargets?.set(target, replacement);
+                                }
                             }
                         }
                     }
@@ -2553,25 +2562,22 @@ let SharedLog = (() => {
             if (this.keep && (await this.keep(args.value.entry))) {
                 return false;
             }
+            this._checkedPrune.trackCandidate(args.key, args.value.entry, args.value.leaders);
             void this.pruneDebouncedFn.add(args);
             return true;
         }
-        async cancelCheckedPruneForLocalLeader(hash) {
+        async cancelCheckedPruneForLocalLeader(hash, options) {
             this.pruneDebouncedFn.delete(hash);
-            this.clearCheckedPruneRetry(hash);
-            this.removePruneRequestSent(hash);
-            this._requestIPruneResponseReplicatorSet.delete(hash);
-            await this._pendingDeletes
-                .get(hash)
-                ?.reject(new Error("Failed to delete, is leader again"));
+            const pendingDelete = this._checkedPrune.getPendingDelete(hash);
+            this._checkedPrune.markCancelled(hash, {
+                preserveRetry: options?.preserveRetry,
+            });
+            await pendingDelete?.reject(new Error("Failed to delete, is leader again"));
         }
         hasActiveCheckedPruneWork(hash) {
-            return (this._pendingDeletes.has(hash) ||
-                this._requestIPruneSent.has(hash) ||
-                this._requestIPruneResponseReplicatorSet.has(hash) ||
-                this._checkedPruneRetries.has(hash));
+            return this._checkedPrune.hasActiveWork(hash);
         }
-        async resolveCheckedPruneLeaders(args) {
+        async revalidateCheckedPruneOwnership(args) {
             const selfHash = this.node.identity.publicKey.hashcode();
             if (args.leaders.has(selfHash)) {
                 if (args.selfReplicating === false) {
@@ -2617,7 +2623,7 @@ let SharedLog = (() => {
                     await this.cancelCheckedPruneForLocalLeader(entry.hash);
                     continue;
                 }
-                if (this._pendingDeletes.has(entry.hash)) {
+                if (this._checkedPrune.hasPendingDelete(entry.hash)) {
                     continue;
                 }
                 if (leaders.size === 0) {
@@ -2630,7 +2636,7 @@ let SharedLog = (() => {
                 this.responseToPruneDebouncedFn.delete(entry.hash);
             }
         }
-        async pruneIndexedEntriesNoLongerLed() {
+        async pruneIndexedEntriesNoLongerLed(options) {
             const selfHash = this.node.identity.publicKey.hashcode();
             const iterator = this.entryCoordinatesIndex.iterate({});
             let enqueuedPrune = false;
@@ -2642,12 +2648,12 @@ let SharedLog = (() => {
                         if (this.closed) {
                             continue;
                         }
-                        const leaders = await this.findLeaders(entryReplicated.coordinates, entryReplicated, { roleAge: 0 });
+                        const leaders = await this.findLeaders(entryReplicated.coordinates, entryReplicated, options?.useDefaultRoleAge ? undefined : { roleAge: 0 });
                         if (leaders.has(selfHash)) {
                             await this.cancelCheckedPruneForLocalLeader(entryReplicated.hash);
                             continue;
                         }
-                        if (this._pendingDeletes.has(entryReplicated.hash)) {
+                        if (this._checkedPrune.hasPendingDelete(entryReplicated.hash)) {
                             continue;
                         }
                         if (leaders.size === 0) {
@@ -2669,20 +2675,63 @@ let SharedLog = (() => {
                 await this.pruneDebouncedFn.flush();
             }
         }
-        clearCheckedPruneRetry(hash) {
-            const state = this._checkedPruneRetries.get(hash);
-            if (state?.timer) {
-                clearTimeout(state.timer);
+        async pruneCurrentHeadsNoLongerLed(options) {
+            const selfHash = this.node.identity.publicKey.hashcode();
+            const heads = await this.log.getHeads(true).all();
+            let enqueuedPrune = false;
+            for (const head of heads) {
+                if (this.closed) {
+                    break;
+                }
+                const leaders = await this.findLeadersFromEntry(head, maxReplicas(this, [head]), options?.useDefaultRoleAge ? undefined : { roleAge: 0 });
+                if (leaders.has(selfHash)) {
+                    await this.cancelCheckedPruneForLocalLeader(head.hash);
+                    continue;
+                }
+                if (this._checkedPrune.hasPendingDelete(head.hash)) {
+                    continue;
+                }
+                if (leaders.size === 0) {
+                    continue;
+                }
+                enqueuedPrune =
+                    (await this.pruneDebouncedFnAddIfNotKeeping({
+                        key: head.hash,
+                        value: { entry: head, leaders },
+                    })) || enqueuedPrune;
+                this.responseToPruneDebouncedFn.delete(head.hash);
+            }
+            if (enqueuedPrune && !this.closed) {
+                await this.pruneDebouncedFn.flush();
+            }
+        }
+        checkedPruneLeadersToMap(leaders) {
+            if (leaders instanceof Map) {
+                return new Map(leaders);
+            }
+            const leadersMap = new Map();
+            for (const leader of leaders) {
+                leadersMap.set(leader, { intersecting: true });
             }
-            this._checkedPruneRetries.delete(hash);
+            return leadersMap;
+        }
+        clearCheckedPruneRetry(hash) {
+            this._checkedPrune.clearRetry(hash);
         }
         scheduleCheckedPruneRetry(args) {
             if (this.closed)
                 return;
-            if (this._pendingDeletes.has(args.entry.hash))
+            if (this._checkedPrune.hasPendingDelete(args.entry.hash))
                 return;
             const hash = args.entry.hash;
-            const state = this._checkedPruneRetries.get(hash) ?? { attempts: 0 };
+            const state = this._checkedPrune.getRetry(hash) ??
+                {
+                    attempts: 0,
+                    entry: args.entry,
+                    leaders: args.leaders,
+                };
+            state.entry = args.entry;
+            state.leaders = args.leaders;
             if (state.timer)
                 return;
             if (state.attempts >= CHECKED_PRUNE_RETRY_MAX_ATTEMPTS) {
@@ -2695,17 +2744,19 @@ let SharedLog = (() => {
             const delayMs = Math.min(CHECKED_PRUNE_RETRY_MAX_DELAY_MS, 1_000 * 2 ** (attempt - 1) + jitterMs);
             state.attempts = attempt;
             state.timer = setTimeout(async () => {
-                const st = this._checkedPruneRetries.get(hash);
+                const st = this._checkedPrune.getRetry(hash);
                 if (st)
                     st.timer = undefined;
                 if (this.closed)
                     return;
-                if (this._pendingDeletes.has(hash))
+                if (this._checkedPrune.hasPendingDelete(hash))
                     return;
+                const retryEntry = st?.entry ?? args.entry;
+                const retryLeaders = st?.leaders ?? args.leaders;
                 let leadersMap;
                 try {
-                    const replicas = decodeReplicas(args.entry).getValue(this);
-                    leadersMap = await this.findLeadersFromEntry(args.entry, replicas, {
+                    const replicas = decodeReplicas(retryEntry).getValue(this);
+                    leadersMap = await this.findLeadersFromEntry(retryEntry, replicas, {
                         roleAge: 0,
                     });
                 }
@@ -2713,21 +2764,13 @@ let SharedLog = (() => {
                     // Best-effort only.
                 }
                 if (!leadersMap || leadersMap.size === 0) {
-                    if (args.leaders instanceof Map) {
-                        leadersMap = args.leaders;
-                    }
-                    else {
-                        leadersMap = new Map();
-                        for (const k of args.leaders) {
-                            leadersMap.set(k, { intersecting: true });
-                        }
-                    }
+                    leadersMap = this.checkedPruneLeadersToMap(retryLeaders);
                 }
                 try {
                     const leadersForRetry = leadersMap ?? new Map();
                     await this.pruneDebouncedFnAddIfNotKeeping({
                         key: hash,
-                        value: { entry: args.entry, leaders: leadersForRetry },
+                        value: { entry: retryEntry, leaders: leadersForRetry },
                     });
                 }
                 catch {
@@ -2735,7 +2778,56 @@ let SharedLog = (() => {
                 }
             }, delayMs);
             state.timer.unref?.();
-            this._checkedPruneRetries.set(hash, state);
+            this._checkedPrune.setRetry(hash, state);
+        }
+        async recoverCheckedPruneFromLateResponses(hashes, publicKeyHash) {
+            if (this.closed)
+                return;
+            const selfHash = this.node.identity.publicKey.hashcode();
+            const toPrune = new Map();
+            const responseStillApplies = [];
+            for (const hash of hashes) {
+                if (this.closed) {
+                    break;
+                }
+                if (this._checkedPrune.hasPendingDelete(hash)) {
+                    continue;
+                }
+                const retry = this._checkedPrune.clearRetryTimer(hash);
+                if (!retry) {
+                    continue;
+                }
+                const entry = retry.entry;
+                let leaders = this.checkedPruneLeadersToMap(retry.leaders);
+                try {
+                    const currentLeaders = await this.findLeadersFromEntry(entry, decodeReplicas(entry).getValue(this), { roleAge: 0 });
+                    if (currentLeaders.size > 0) {
+                        leaders = currentLeaders;
+                    }
+                }
+                catch {
+                    // Best-effort only; the stored retry leaders came from a previous
+                    // checked-prune decision for this exact entry.
+                }
+                if (leaders.has(selfHash)) {
+                    await this.cancelCheckedPruneForLocalLeader(hash);
+                    continue;
+                }
+                if (leaders.size === 0) {
+                    continue;
+                }
+                toPrune.set(hash, { entry, leaders });
+                if (leaders.has(publicKeyHash)) {
+                    responseStillApplies.push(hash);
+                }
+            }
+            if (toPrune.size === 0) {
+                return;
+            }
+            void Promise.allSettled(this.prune(toPrune));
+            for (const hash of responseStillApplies) {
+                void this._checkedPrune.getPendingDelete(hash)?.resolve(publicKeyHash);
+            }
         }
         async append(data, options) {
             if (this._isAdaptiveReplicating) {
@@ -2843,7 +2935,7 @@ let SharedLog = (() => {
                 : createReplicationDomainHash(options?.compatibility && options?.compatibility < 10 ? "u32" : "u64")(this);
             this.indexableDomain = createIndexableDomainFromResolution(this.domain.resolution);
             this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 2e4;
-            this._pendingDeletes = new Map();
+            this._checkedPrune = new CheckedPruneCoordinator();
             this._pendingIHave = new Map();
             this.latestReplicationInfoMessage = new Map();
             this._replicationInfoBlockedPeers = new Set();
@@ -3001,22 +3093,28 @@ let SharedLog = (() => {
                 ],
             })) > 0;
             this._gidPeersHistory = new Map();
-            this._requestIPruneSent = new Map();
-            this._requestIPruneResponseReplicatorSet = new Map();
-            this._checkedPruneRetries = new Map();
             this.replicationChangeDebounceFn = debounceAggregationChanges((change) => this.onReplicationChange(change).then(() => this.rebalanceParticipationDebounced?.call()), this.distributionDebounceTime);
             this.pruneDebouncedFn = debouncedAccumulatorMap(async (map) => {
                 const current = new Map();
                 const selfReplicating = await this.isReplicating();
                 for (const [hash, value] of map) {
-                    const checkedPruneLeaders = await this.resolveCheckedPruneLeaders({
+                    const checkedPruneLeaders = await this.revalidateCheckedPruneOwnership({
                         hash,
                         entry: value.entry,
                         leaders: value.leaders,
                         selfReplicating,
                     });
                     if (checkedPruneLeaders.localLeader) {
-                        await this.cancelCheckedPruneForLocalLeader(hash);
+                        const preserveRetry = this._checkedPrune.hasRetry(hash);
+                        await this.cancelCheckedPruneForLocalLeader(hash, {
+                            preserveRetry,
+                        });
+                        if (preserveRetry) {
+                            this.scheduleCheckedPruneRetry({
+                                entry: value.entry,
+                                leaders: checkedPruneLeaders.leaders,
+                            });
+                        }
                         continue;
                     }
                     current.set(hash, {
@@ -3050,7 +3148,7 @@ let SharedLog = (() => {
                             to: allRequestingPeers,
                             redundancy: 1,
                         }),
-                        priority: 1,
+                        priority: CONVERGENCE_MESSAGE_PRIORITY,
                     });
             }, () => {
                 let accumulator = new Map();
@@ -3339,18 +3437,7 @@ let SharedLog = (() => {
             this.cancelReplicationInfoRequests(peerHash);
             this._replicatorLivenessFailures.delete(peerHash);
             this._replicatorLastActivityAt.delete(peerHash);
-            for (const [hash, peers] of this._requestIPruneSent) {
-                peers.delete(peerHash);
-                if (peers.size === 0) {
-                    this._requestIPruneSent.delete(hash);
-                }
-            }
-            for (const [hash, peers] of this._requestIPruneResponseReplicatorSet) {
-                peers.delete(peerHash);
-                if (peers.size === 0) {
-                    this._requestIPruneResponseReplicatorSet.delete(hash);
-                }
-            }
+            this._checkedPrune.cleanupPeer(peerHash);
         }
         markReplicatorActivity(peerHash, now = Date.now()) {
             this._replicatorLastActivityAt.set(peerHash, now);
@@ -3396,14 +3483,14 @@ let SharedLog = (() => {
             const maxPeers = options?.maxPeers ?? 8;
             const self = this.node.identity.publicKey.hashcode();
             const seed = hashToSeed32(hash);
-            const hinted = this._requestIPruneResponseReplicatorSet.get(hash);
+            const hinted = this._checkedPrune.getConfirmedReplicators(hash);
             if (hinted && hinted.size > 0) {
                 const peers = [...hinted].filter((p) => p !== self);
                 return peers.length > 0
                     ? pickDeterministicSubset(peers, seed, maxPeers)
                     : undefined;
             }
-            const contacted = this._requestIPruneSent.get(hash);
+            const contacted = this._checkedPrune.getContactedReplicators(hash);
             if (contacted && contacted.size > 0) {
                 const peers = [...contacted].filter((p) => p !== self);
                 return peers.length > 0
@@ -3760,25 +3847,14 @@ let SharedLog = (() => {
                 this._appendBackfillTimer = undefined;
             }
             this._appendBackfillPendingByTarget.clear();
-            for (const [_k, v] of this._pendingDeletes) {
-                v.clear();
-                v.promise.resolve(); // TODO or reject?
-            }
             for (const [_k, v] of this._pendingIHave) {
                 v.clear();
             }
-            for (const [_k, v] of this._checkedPruneRetries) {
-                if (v.timer)
-                    clearTimeout(v.timer);
-            }
+            this._checkedPrune.close();
             await this.remoteBlocks.stop();
-            this._pendingDeletes.clear();
             this._pendingIHave.clear();
-            this._checkedPruneRetries.clear();
             this.latestReplicationInfoMessage.clear();
             this._gidPeersHistory.clear();
-            this._requestIPruneSent.clear();
-            this._requestIPruneResponseReplicatorSet.clear();
             // Cancel any pending debounced timers so they can't fire after we've torn down
             // indexes/RPC state.
             this.rebalanceParticipationDebounced?.close();
@@ -3829,7 +3905,7 @@ let SharedLog = (() => {
                     try {
                         await this.rpc
                             .send(new AllReplicatingSegmentsMessage({ segments: [] }), {
-                            priority: 1,
+                            priority: CONVERGENCE_MESSAGE_PRIORITY,
                             signal: abort.signal,
                         })
                             .catch(() => { });
@@ -3873,7 +3949,7 @@ let SharedLog = (() => {
                     try {
                         await this.rpc
                             .send(new AllReplicatingSegmentsMessage({ segments: [] }), {
-                            priority: 1,
+                            priority: CONVERGENCE_MESSAGE_PRIORITY,
                             signal: abort.signal,
                         })
                             .catch(() => { });
@@ -4016,7 +4092,7 @@ let SharedLog = (() => {
                                     for (const entry of entries) {
                                         this.pruneDebouncedFn.delete(entry.entry.hash);
                                         this.removePruneRequestSent(entry.entry.hash);
-                                        this._requestIPruneResponseReplicatorSet.delete(entry.entry.hash);
+                                        this._checkedPrune.clearConfirmedReplicators(entry.entry.hash);
                                         if (fromIsLeader) {
                                             this.addPeersToGidPeerHistory(gid, [
                                                 context.from.hashcode(),
@@ -4112,27 +4188,37 @@ let SharedLog = (() => {
                         this.removeEntriesKnownByPeer([hash], from);
                         // if we expect the remote to be owner of this entry because we are to prune ourselves, then we need to remove the remote
                         // this is due to that the remote has previously indicated to be a replicator to help us prune but now has changed their mind
-                        const outGoingPrunes = this._requestIPruneResponseReplicatorSet.get(hash);
-                        if (outGoingPrunes) {
-                            outGoingPrunes.delete(from);
-                        }
+                        this._checkedPrune.removeConfirmedReplicator(hash, from);
                         const indexedEntry = await this.log.entryIndex.getShallow(hash);
                         let isLeader = false;
-                        if (indexedEntry &&
-                            !this._pendingDeletes.has(hash) &&
-                            (await this.log.blocks.has(hash))) {
-                            this.removePeerFromGidPeerHistory(context.from.hashcode(), indexedEntry.value.meta.gid);
-                            await this._waitForReplicators(await this.createCoordinates(indexedEntry.value, decodeReplicas(indexedEntry.value).getValue(this)), indexedEntry.value, [
-                                {
-                                    key: this.node.identity.publicKey.hashcode(),
-                                    replicator: true,
-                                },
-                            ], {
-                                onLeader: (key) => {
-                                    isLeader =
-                                        isLeader || key === this.node.identity.publicKey.hashcode();
-                                },
-                            });
+                        if (indexedEntry && (await this.log.blocks.has(hash))) {
+                            const pendingDelete = this._checkedPrune.getPendingDelete(hash);
+                            if (pendingDelete) {
+                                const ownership = await this.revalidateCheckedPruneOwnership({
+                                    hash,
+                                    entry: indexedEntry.value,
+                                    leaders: new Map(),
+                                });
+                                if (ownership.localLeader) {
+                                    await this.cancelCheckedPruneForLocalLeader(hash);
+                                    isLeader = true;
+                                }
+                            }
+                            else {
+                                this.removePeerFromGidPeerHistory(context.from.hashcode(), indexedEntry.value.meta.gid);
+                                await this._waitForReplicators(await this.createCoordinates(indexedEntry.value, decodeReplicas(indexedEntry.value).getValue(this)), indexedEntry.value, [
+                                    {
+                                        key: this.node.identity.publicKey.hashcode(),
+                                        replicator: true,
+                                    },
+                                ], {
+                                    onLeader: (key) => {
+                                        isLeader =
+                                            isLeader ||
+                                                key === this.node.identity.publicKey.hashcode();
+                                    },
+                                });
+                            }
                         }
                         if (isLeader) {
                             hasAndIsLeader.push(hash);
@@ -4195,8 +4281,18 @@ let SharedLog = (() => {
                     }
                 }
                 else if (msg instanceof ResponseIPrune) {
+                    const lateResponses = [];
                     for (const hash of msg.hashes) {
-                        this._pendingDeletes.get(hash)?.resolve(context.from.hashcode());
+                        const pendingDelete = this._checkedPrune.getPendingDelete(hash);
+                        if (pendingDelete) {
+                            void pendingDelete.resolve(context.from.hashcode());
+                        }
+                        else {
+                            lateResponses.push(hash);
+                        }
+                    }
+                    if (lateResponses.length > 0) {
+                        void this.recoverCheckedPruneFromLateResponses(lateResponses, context.from.hashcode()).catch((error) => logger.error(error.toString()));
                     }
                 }
                 else if (msg instanceof ConfirmEntriesMessage) {
@@ -4539,7 +4635,7 @@ let SharedLog = (() => {
                 }
                 await this.replicate(entriesToReplicate, {
                     rebalance: assumeSynced ? false : true,
-                    checkDuplicates: true,
+                    checkDuplicates: assumeSynced ? false : true,
                     mergeSegments: typeof options.replicate !== "boolean" && options.replicate
                         ? options.replicate.mergeSegments
                         : false,
@@ -4566,7 +4662,7 @@ let SharedLog = (() => {
                 }
                 if (messageToSend) {
                     await this.rpc.send(messageToSend, {
-                        priority: 1,
+                        priority: CONVERGENCE_MESSAGE_PRIORITY,
                     });
                 }
             }
@@ -5125,8 +5221,8 @@ let SharedLog = (() => {
             };
             this._replicationInfoRequestByPeer.set(peerHash, state);
             const intervalMs = Math.max(50, this.waitForReplicatorRequestIntervalMs);
-            const maxAttempts = Math.min(5, this.waitForReplicatorRequestMaxAttempts ??
-                WAIT_FOR_REPLICATOR_REQUEST_MIN_ATTEMPTS);
+            const maxAttempts = this.waitForReplicatorRequestMaxAttempts ??
+                Math.max(WAIT_FOR_REPLICATOR_REQUEST_MIN_ATTEMPTS, Math.ceil(this.waitForReplicatorTimeout / intervalMs));
             const tick = () => {
                 if (this.closed || this._closeController.signal.aborted) {
                     this.cancelReplicationInfoRequests(peerHash);
@@ -5223,25 +5319,14 @@ let SharedLog = (() => {
             return new AbsoluteReplicas(maxValue);
         }
         removePruneRequestSent(hash, to) {
-            if (!to) {
-                this._requestIPruneSent.delete(hash);
-            }
-            else {
-                let set = this._requestIPruneSent.get(hash);
-                if (set) {
-                    set.delete(to);
-                    if (set.size === 0) {
-                        this._requestIPruneSent.delete(hash);
-                    }
-                }
-            }
+            this._checkedPrune.removeRequestSent(hash, to);
         }
         prune(entries, options) {
             if (options?.unchecked) {
                 return [...entries.values()].map((x) => {
                     this._gidPeersHistory.delete(x.entry.meta.gid);
                     this.removePruneRequestSent(x.entry.hash);
-                    this._requestIPruneResponseReplicatorSet.delete(x.entry.hash);
+                    this._checkedPrune.clearConfirmedReplicators(x.entry.hash);
                     return this.log.remove(x.entry, {
                         recursively: true,
                     });
@@ -5269,7 +5354,7 @@ let SharedLog = (() => {
                     }
                     set.push(entry.hash);
                 }
-                const pendingPrev = this._pendingDeletes.get(entry.hash);
+                const pendingPrev = this._checkedPrune.getPendingDelete(entry.hash);
                 if (pendingPrev) {
                     // If a background prune is already in-flight, an explicit prune request should
                     // still respect the caller's timeout. Otherwise, tests (and user calls) can
@@ -5298,45 +5383,62 @@ let SharedLog = (() => {
                 const minReplicas = decodeReplicas(entry);
                 const deferredPromise = pDefer();
                 const clear = () => {
-                    const pending = this._pendingDeletes.get(entry.hash);
+                    const pending = this._checkedPrune.getPendingDelete(entry.hash);
                     if (pending?.promise === deferredPromise) {
-                        this._pendingDeletes.delete(entry.hash);
+                        this._checkedPrune.deletePendingDelete(entry.hash, pending);
                     }
                     clearTimeout(timeout);
                 };
                 const resolve = () => {
-                    clear();
+                    clearTimeout(timeout);
                     this.clearCheckedPruneRetry(entry.hash);
                     cleanupTimer.push(setTimeout(async () => {
                         this._gidPeersHistory.delete(entry.meta.gid);
                         this.removePruneRequestSent(entry.hash);
-                        this._requestIPruneResponseReplicatorSet.delete(entry.hash);
-                        if (await this.isLeader({
+                        this._checkedPrune.clearConfirmedReplicators(entry.hash);
+                        const ownership = await this.revalidateCheckedPruneOwnership({
+                            hash: entry.hash,
                             entry,
-                            replicas: minReplicas.getValue(this),
-                        })) {
+                            leaders: this.checkedPruneLeadersToMap(leaders),
+                            selfReplicating: true,
+                        });
+                        if (ownership.localLeader) {
+                            clear();
+                            if (!explicitTimeout) {
+                                this.scheduleCheckedPruneRetry({ entry, leaders });
+                            }
                             deferredPromise.reject(new Error("Failed to delete, is leader again"));
                             return;
                         }
+                        this._checkedPrune.markRemoving(entry.hash);
                         return this.log
                             .remove(entry, {
                             recursively: true,
                         })
                             .then(() => {
+                            clear();
+                            this._checkedPrune.markDone(entry.hash);
                             deferredPromise.resolve();
                         })
                             .catch((e) => {
+                            clear();
+                            this._checkedPrune.markCancelled(entry.hash, {
+                                preserveRetry: false,
+                            });
                             deferredPromise.reject(e);
                         })
                             .finally(async () => {
                             this._gidPeersHistory.delete(entry.meta.gid);
                             this.removePruneRequestSent(entry.hash);
-                            this._requestIPruneResponseReplicatorSet.delete(entry.hash);
+                            this._checkedPrune.clearConfirmedReplicators(entry.hash);
                             // TODO in the case we become leader again here we need to re-add the entry
-                            if (await this.isLeader({
+                            const ownership = await this.revalidateCheckedPruneOwnership({
+                                hash: entry.hash,
                                 entry,
-                                replicas: minReplicas.getValue(this),
-                            })) {
+                                leaders: this.checkedPruneLeadersToMap(leaders),
+                                selfReplicating: true,
+                            });
+                            if (ownership.localLeader) {
                                 logger.error("Unexpected: Is leader after delete");
                             }
                         });
@@ -5347,11 +5449,9 @@ let SharedLog = (() => {
                     const isCheckedPruneTimeout = e instanceof Error &&
                         typeof e.message === "string" &&
                         e.message.startsWith("Timeout for checked pruning");
-                    if (explicitTimeout || !isCheckedPruneTimeout) {
-                        this.clearCheckedPruneRetry(entry.hash);
-                    }
-                    this.removePruneRequestSent(entry.hash);
-                    this._requestIPruneResponseReplicatorSet.delete(entry.hash);
+                    this._checkedPrune.markCancelled(entry.hash, {
+                        preserveRetry: !explicitTimeout && isCheckedPruneTimeout,
+                    });
                     deferredPromise.reject(e);
                 };
                 let cursor = undefined;
@@ -5361,7 +5461,7 @@ let SharedLog = (() => {
                 // If we time out too early we can end up with permanently prunable heads that never
                 // get retried (a common CI flake in "prune before join" tests).
                 const checkedPruneTimeoutMs = options?.timeout ??
-                    Math.max(10_000, Number(this._respondToIHaveTimeout ?? 0) +
+                    Math.max(CHECKED_PRUNE_BACKGROUND_TIMEOUT_MIN_MS, Number(this._respondToIHaveTimeout ?? 0) +
                         this.waitForReplicatorTimeout +
                         PRUNE_DEBOUNCE_INTERVAL * 2);
                 const timeout = setTimeout(() => {
@@ -5374,7 +5474,7 @@ let SharedLog = (() => {
                     reject(new Error(`Timeout for checked pruning after ${checkedPruneTimeoutMs}ms (closed=${this.closed})`));
                 }, checkedPruneTimeoutMs);
                 timeout.unref?.();
-                this._pendingDeletes.set(entry.hash, {
+                this._checkedPrune.setPendingDelete(entry.hash, {
                     promise: deferredPromise,
                     clear,
                     reject,
@@ -5394,34 +5494,24 @@ let SharedLog = (() => {
                         }))) {
                             return;
                         }
-                        let existCounter = this._requestIPruneResponseReplicatorSet.get(entry.hash);
-                        if (!existCounter) {
-                            existCounter = new Set();
-                            this._requestIPruneResponseReplicatorSet.set(entry.hash, existCounter);
-                        }
-                        existCounter.add(publicKeyHash);
+                        const existCounter = this._checkedPrune.addConfirmedReplicator(entry.hash, publicKeyHash);
                         // Seed provider hints so future remote reads can avoid extra round-trips.
                         this.remoteBlocks.hintProviders(entry.hash, [publicKeyHash]);
                         if (minReplicasValue <= existCounter.size) {
                             resolve();
                         }
                     },
-                });
+                }, entry, leaders);
                 promises.push(deferredPromise.promise);
             }
             const emitMessages = async (entries, to) => {
                 const filteredSet = [];
                 for (const entry of entries) {
-                    let set = this._requestIPruneSent.get(entry);
-                    if (!set) {
-                        set = new Set();
-                        this._requestIPruneSent.set(entry, set);
-                    }
                     /* TODO why can we not have this statement?
                     if (set.has(to)) {
                         continue;
                     } */
-                    set.add(to);
+                    this._checkedPrune.addRequestSent(entry, to);
                     filteredSet.push(entry);
                 }
                 if (filteredSet.length > 0) {
@@ -5432,7 +5522,7 @@ let SharedLog = (() => {
                             to: [to], // TODO group by peers?
                             redundancy: 1,
                         }),
-                        priority: 1,
+                        priority: CONVERGENCE_MESSAGE_PRIORITY,
                     });
                 }
             };
@@ -5453,7 +5543,7 @@ let SharedLog = (() => {
                         return;
                     const pendingByPeer = [];
                     for (const [peer, hashes] of peerToEntries) {
-                        const pending = hashes.filter((h) => this._pendingDeletes.has(h));
+                        const pending = hashes.filter((h) => this._checkedPrune.hasPendingDelete(h));
                         if (pending.length > 0) {
                             pendingByPeer.push([peer, pending]);
                         }
@@ -5779,12 +5869,21 @@ let SharedLog = (() => {
                 for (const target of [...uncheckedDeliver.keys()]) {
                     flushUncheckedDeliverTarget(target);
                 }
-                if (this._isAdaptiveReplicating && hasSelfRangeRemoval) {
-                    // Adaptive shrink/replacement can make already-indexed local heads
-                    // prunable even when the incremental rebalance scan missed them under
-                    // churn or timing pressure. Re-scan after repair dispatches are flushed
-                    // so checked prune work is enqueued before callers wait for idle.
-                    await this.pruneIndexedEntriesNoLongerLed();
+                if (this._isAdaptiveReplicating &&
+                    (hasSelfRangeRemoval ||
+                        changes.some((change) => change.type === "added" ||
+                            change.type === "removed" ||
+                            change.type === "replaced"))) {
+                    // Adaptive range changes can make already-indexed local heads prunable
+                    // even when the incremental rebalance scan misses them under churn or
+                    // timing pressure. Re-scan after repair dispatches are flushed using the
+                    // mature-role view, which matches the bounded pruning contract.
+                    await this.pruneIndexedEntriesNoLongerLed({
+                        useDefaultRoleAge: true,
+                    });
+                    await this.pruneCurrentHeadsNoLongerLed({
+                        useDefaultRoleAge: true,
+                    });
                 }
                 return changed;
             }