@peerbit/shared-log 13.1.16 → 13.1.17

package/src/index.ts

@@ -54,6 +54,8 @@ import {
 	ACK_CONTROL_PRIORITY,
 	AcknowledgeDelivery,
 	AnyWhere,
+	BACKGROUND_MESSAGE_PRIORITY,
+	CONVERGENCE_MESSAGE_PRIORITY,
 	createRequestTransportContext,
 	DataMessage,
 	MessageHeader,
@@ -72,6 +74,12 @@ import pDefer, { type DeferredPromise } from "p-defer";
 import PQueue from "p-queue";
 import { concat, fromString } from "uint8arrays";
 import { BlocksMessage } from "./blocks.js";
+import {
+	CheckedPruneCoordinator,
+	type CheckedPruneEntry,
+	type CheckedPruneLeaderMap,
+	type CheckedPruneRetryState,
+} from "./checked-prune.js";
 import { type CPUUsage, CPUUsageIntervalLag } from "./cpu.js";
 import {
 	type DebouncedAccumulatorMap,
@@ -169,7 +177,11 @@ import type {
 	Syncronizer,
 } from "./sync/index.js";
 import { RatelessIBLTSynchronizer } from "./sync/rateless-iblt.js";
-import { ConfirmEntriesMessage, SimpleSyncronizer } from "./sync/simple.js";
+import {
+	ConfirmEntriesMessage,
+	SYNC_MESSAGE_PRIORITY,
+	SimpleSyncronizer,
+} from "./sync/simple.js";
 import { groupByGid } from "./utils.js";
 
 const toLocalPublicSignKey = (
@@ -241,12 +253,6 @@ export { MAX_U32, MAX_U64, type NumberFromType };
 export const logger = loggerFn("peerbit:shared-log");
 const warn = logger.newScope("warn");
 
-type CheckedPruneLeaderMap = Map<string, { intersecting: boolean }>;
-type CheckedPruneEntry<T, R extends "u32" | "u64"> =
-	| Entry<T>
-	| ShallowEntry
-	| EntryReplicated<R>;
-
 const getLatestEntry = (
 	entries: (ShallowOrFullEntry<any> | EntryWithRefs<any>)[],
 ) => {
@@ -494,6 +500,7 @@ export const WAIT_FOR_PRUNE_DELAY = 0;
 const PRUNE_DEBOUNCE_INTERVAL = 500;
 const CHECKED_PRUNE_RESEND_INTERVAL_MIN_MS = 250;
 const CHECKED_PRUNE_RESEND_INTERVAL_MAX_MS = 5_000;
+const CHECKED_PRUNE_BACKGROUND_TIMEOUT_MIN_MS = 120_000;
 const CHECKED_PRUNE_RETRY_MAX_ATTEMPTS = 3;
 const CHECKED_PRUNE_RETRY_MAX_DELAY_MS = 30_000;
 
@@ -799,15 +806,11 @@ export class SharedLog<
 		SharedLogOptions<T, D, R>;
 	private _closeController!: AbortController;
 	private _respondToIHaveTimeout!: any;
-	private _pendingDeletes!: Map<
-		string,
-		{
-			promise: DeferredPromise<void>;
-			clear: () => void;
-			resolve: (publicKeyHash: string) => Promise<void> | void;
-			reject(reason: any): Promise<void> | void;
-		}
-	>;
+	private _checkedPrune!: CheckedPruneCoordinator<T, R>;
+
+	private get _pendingDeletes() {
+		return this._checkedPrune.pendingDeletes;
+	}
 
 	private _pendingIHave!: Map<
 		string,
@@ -879,12 +882,15 @@ export class SharedLog<
 		>
 	>;
 
-	private _requestIPruneSent!: Map<string, Set<string>>; // tracks entry hash to peer hash for requesting I prune messages
-	private _requestIPruneResponseReplicatorSet!: Map<string, Set<string>>; // tracks entry hash to peer hash
-	private _checkedPruneRetries!: Map<
-		string,
-		{ attempts: number; timer?: ReturnType<typeof setTimeout> }
-	>;
+	private get _requestIPruneSent() {
+		return this._checkedPrune.requestIPruneSent;
+	}
+	private get _requestIPruneResponseReplicatorSet() {
+		return this._checkedPrune.responseReplicatorSet;
+	}
+	private get _checkedPruneRetries() {
+		return this._checkedPrune.retries;
+	}
 
 	private replicationChangeDebounceFn!: ReturnType<
 		typeof debounceAggregationChanges<ReplicationRangeIndexable<R>>
@@ -1081,7 +1087,7 @@ export class SharedLog<
 			header: new MessageHeader({
 				session: 0,
 				mode: new AnyWhere(),
-				priority: 0,
+				priority: BACKGROUND_MESSAGE_PRIORITY,
 			}),
 		});
 		contextMessage.header.timestamp = envelope.timestamp;
@@ -1113,7 +1119,7 @@ export class SharedLog<
 			header: new MessageHeader({
 				session: 0,
 				mode: new AnyWhere(),
-				priority: 0,
+				priority: BACKGROUND_MESSAGE_PRIORITY,
 			}),
 		});
 		contextMessage.header.timestamp = detail.timestamp;
@@ -2015,7 +2021,7 @@ export class SharedLog<
 					segmentIds: rangesToUnreplicate.map((x) => x.id),
 				}),
 				{
-					priority: 1,
+					priority: CONVERGENCE_MESSAGE_PRIORITY,
 				},
 			);
 		}
@@ -2163,7 +2169,7 @@ export class SharedLog<
 			this.node.identity.publicKey,
 		);
 		await this.rpc.send(new StoppedReplicating({ segmentIds }), {
-			priority: 1,
+			priority: CONVERGENCE_MESSAGE_PRIORITY,
 		});
 	}
 
@@ -2189,7 +2195,7 @@ export class SharedLog<
 			// announce that we are no longer replicating
 
 			await this.rpc.send(new AllReplicatingSegmentsMessage({ segments: [] }), {
-				priority: 1,
+				priority: CONVERGENCE_MESSAGE_PRIORITY,
 			});
 		}
 
@@ -2673,7 +2679,7 @@ export class SharedLog<
 					return options.announce(message);
 				} else {
 					await this.rpc.send(message, {
-						priority: 1,
+						priority: CONVERGENCE_MESSAGE_PRIORITY,
 					});
 				}
 			}
@@ -2902,7 +2908,7 @@ export class SharedLog<
 				i + REPAIR_CONFIRMATION_HASH_BATCH_SIZE,
 			);
 			await this.rpc.send(new ConfirmEntriesMessage({ hashes: chunk }), {
-				priority: 1,
+				priority: CONVERGENCE_MESSAGE_PRIORITY,
 				mode: new SilentDelivery({ to: [target], redundancy: 1 }),
 			});
 		}
@@ -2918,7 +2924,7 @@ export class SharedLog<
 		)) {
 			message.reserved[0] |= EXCHANGE_HEADS_REPAIR_HINT;
 			await this.rpc.send(message, {
-				priority: 1,
+				priority: SYNC_MESSAGE_PRIORITY,
 				mode: new SilentDelivery({ to: [target], redundancy: 1 }),
 			});
 		}
@@ -3569,30 +3575,32 @@ export class SharedLog<
 		if (this.keep && (await this.keep(args.value.entry))) {
 			return false;
 		}
+		this._checkedPrune.trackCandidate(
+			args.key,
+			args.value.entry,
+			args.value.leaders,
+		);
 		void this.pruneDebouncedFn.add(args);
 		return true;
 	}
 
-	private async cancelCheckedPruneForLocalLeader(hash: string) {
+	private async cancelCheckedPruneForLocalLeader(
+		hash: string,
+		options?: { preserveRetry?: boolean },
+	) {
 		this.pruneDebouncedFn.delete(hash);
-		this.clearCheckedPruneRetry(hash);
-		this.removePruneRequestSent(hash);
-		this._requestIPruneResponseReplicatorSet.delete(hash);
-		await this._pendingDeletes
-			.get(hash)
-			?.reject(new Error("Failed to delete, is leader again"));
+		const pendingDelete = this._checkedPrune.getPendingDelete(hash);
+		this._checkedPrune.markCancelled(hash, {
+			preserveRetry: options?.preserveRetry,
+		});
+		await pendingDelete?.reject(new Error("Failed to delete, is leader again"));
 	}
 
 	private hasActiveCheckedPruneWork(hash: string) {
-		return (
-			this._pendingDeletes.has(hash) ||
-			this._requestIPruneSent.has(hash) ||
-			this._requestIPruneResponseReplicatorSet.has(hash) ||
-			this._checkedPruneRetries.has(hash)
-		);
+		return this._checkedPrune.hasActiveWork(hash);
 	}
 
-	private async resolveCheckedPruneLeaders(args: {
+	private async revalidateCheckedPruneOwnership(args: {
 		hash: string;
 		entry: CheckedPruneEntry<T, R>;
 		leaders: CheckedPruneLeaderMap;
@@ -3660,7 +3668,7 @@ export class SharedLog<
 				continue;
 			}
 
-			if (this._pendingDeletes.has(entry.hash)) {
+			if (this._checkedPrune.hasPendingDelete(entry.hash)) {
 				continue;
 			}
 
@@ -3676,7 +3684,9 @@ export class SharedLog<
 		}
 	}
 
-	private async pruneIndexedEntriesNoLongerLed() {
+	private async pruneIndexedEntriesNoLongerLed(options?: {
+		useDefaultRoleAge?: boolean;
+	}) {
 		const selfHash = this.node.identity.publicKey.hashcode();
 		const iterator = this.entryCoordinatesIndex.iterate({});
 		let enqueuedPrune = false;
@@ -3692,7 +3702,7 @@ export class SharedLog<
 					const leaders = await this.findLeaders(
 						entryReplicated.coordinates,
 						entryReplicated,
-						{ roleAge: 0 },
+						options?.useDefaultRoleAge ? undefined : { roleAge: 0 },
 					);
 
 					if (leaders.has(selfHash)) {
@@ -3700,7 +3710,7 @@ export class SharedLog<
 						continue;
 					}
 
-					if (this._pendingDeletes.has(entryReplicated.hash)) {
+					if (this._checkedPrune.hasPendingDelete(entryReplicated.hash)) {
 						continue;
 					}
 
@@ -3724,12 +3734,65 @@ export class SharedLog<
 		}
 	}
 
-	private clearCheckedPruneRetry(hash: string) {
-		const state = this._checkedPruneRetries.get(hash);
-		if (state?.timer) {
-			clearTimeout(state.timer);
+	private async pruneCurrentHeadsNoLongerLed(options?: {
+		useDefaultRoleAge?: boolean;
+	}) {
+		const selfHash = this.node.identity.publicKey.hashcode();
+		const heads = await this.log.getHeads(true).all();
+		let enqueuedPrune = false;
+
+		for (const head of heads) {
+			if (this.closed) {
+				break;
+			}
+
+			const leaders = await this.findLeadersFromEntry(
+				head,
+				maxReplicas(this, [head]),
+				options?.useDefaultRoleAge ? undefined : { roleAge: 0 },
+			);
+
+			if (leaders.has(selfHash)) {
+				await this.cancelCheckedPruneForLocalLeader(head.hash);
+				continue;
+			}
+
+			if (this._checkedPrune.hasPendingDelete(head.hash)) {
+				continue;
+			}
+
+			if (leaders.size === 0) {
+				continue;
+			}
+
+			enqueuedPrune =
+				(await this.pruneDebouncedFnAddIfNotKeeping({
+					key: head.hash,
+					value: { entry: head, leaders },
+				})) || enqueuedPrune;
+			this.responseToPruneDebouncedFn.delete(head.hash);
+		}
+
+		if (enqueuedPrune && !this.closed) {
+			await this.pruneDebouncedFn.flush();
+		}
+	}
+
+	private checkedPruneLeadersToMap(
+		leaders: CheckedPruneLeaderMap | Set<string>,
+	): CheckedPruneLeaderMap {
+		if (leaders instanceof Map) {
+			return new Map(leaders);
+		}
+		const leadersMap: CheckedPruneLeaderMap = new Map();
+		for (const leader of leaders) {
+			leadersMap.set(leader, { intersecting: true });
 		}
-		this._checkedPruneRetries.delete(hash);
+		return leadersMap;
+	}
+
+	private clearCheckedPruneRetry(hash: string) {
+		this._checkedPrune.clearRetry(hash);
 	}
 
 	private scheduleCheckedPruneRetry(args: {
@@ -3737,11 +3800,18 @@ export class SharedLog<
 		leaders: CheckedPruneLeaderMap | Set<string>;
 	}) {
 		if (this.closed) return;
-		if (this._pendingDeletes.has(args.entry.hash)) return;
+		if (this._checkedPrune.hasPendingDelete(args.entry.hash)) return;
 
 		const hash = args.entry.hash;
 		const state =
-			this._checkedPruneRetries.get(hash) ?? { attempts: 0 };
+			this._checkedPrune.getRetry(hash) ??
+			({
+				attempts: 0,
+				entry: args.entry,
+				leaders: args.leaders,
+			} satisfies CheckedPruneRetryState<T, R>);
+		state.entry = args.entry;
+		state.leaders = args.leaders;
 
 		if (state.timer) return;
 		if (state.attempts >= CHECKED_PRUNE_RETRY_MAX_ATTEMPTS) {
@@ -3759,15 +3829,17 @@ export class SharedLog<
 
 		state.attempts = attempt;
 		state.timer = setTimeout(async () => {
-			const st = this._checkedPruneRetries.get(hash);
+			const st = this._checkedPrune.getRetry(hash);
 			if (st) st.timer = undefined;
 			if (this.closed) return;
-			if (this._pendingDeletes.has(hash)) return;
+			if (this._checkedPrune.hasPendingDelete(hash)) return;
+			const retryEntry = st?.entry ?? args.entry;
+			const retryLeaders = st?.leaders ?? args.leaders;
 
 			let leadersMap: CheckedPruneLeaderMap | undefined;
 			try {
-				const replicas = decodeReplicas(args.entry).getValue(this);
-				leadersMap = await this.findLeadersFromEntry(args.entry, replicas, {
+				const replicas = decodeReplicas(retryEntry).getValue(this);
+				leadersMap = await this.findLeadersFromEntry(retryEntry, replicas, {
 					roleAge: 0,
 				});
 			} catch {
@@ -3775,14 +3847,7 @@ export class SharedLog<
 			}
 
 			if (!leadersMap || leadersMap.size === 0) {
-				if (args.leaders instanceof Map) {
-					leadersMap = args.leaders;
-				} else {
-					leadersMap = new Map<string, { intersecting: boolean }>();
-					for (const k of args.leaders) {
-						leadersMap.set(k, { intersecting: true });
-					}
-				}
+				leadersMap = this.checkedPruneLeadersToMap(retryLeaders);
 			}
 
 			try {
@@ -3790,14 +3855,81 @@ export class SharedLog<
 					leadersMap ?? new Map<string, { intersecting: boolean }>();
 				await this.pruneDebouncedFnAddIfNotKeeping({
 					key: hash,
-					value: { entry: args.entry, leaders: leadersForRetry },
+					value: { entry: retryEntry, leaders: leadersForRetry },
 				});
 			} catch {
 				// Best-effort only; pruning will be re-attempted on future changes.
 			}
 		}, delayMs);
 		state.timer.unref?.();
-		this._checkedPruneRetries.set(hash, state);
+		this._checkedPrune.setRetry(hash, state);
+	}
+
+	private async recoverCheckedPruneFromLateResponses(
+		hashes: string[],
+		publicKeyHash: string,
+	) {
+		if (this.closed) return;
+		const selfHash = this.node.identity.publicKey.hashcode();
+		const toPrune = new Map<
+			string,
+			{
+				entry: CheckedPruneEntry<T, R>;
+				leaders: CheckedPruneLeaderMap;
+			}
+		>();
+		const responseStillApplies: string[] = [];
+
+		for (const hash of hashes) {
+			if (this.closed) {
+				break;
+			}
+			if (this._checkedPrune.hasPendingDelete(hash)) {
+				continue;
+			}
+			const retry = this._checkedPrune.clearRetryTimer(hash);
+			if (!retry) {
+				continue;
+			}
+
+			const entry = retry.entry;
+			let leaders = this.checkedPruneLeadersToMap(retry.leaders);
+			try {
+				const currentLeaders = await this.findLeadersFromEntry(
+					entry,
+					decodeReplicas(entry).getValue(this),
+					{ roleAge: 0 },
+				);
+				if (currentLeaders.size > 0) {
+					leaders = currentLeaders;
+				}
+			} catch {
+				// Best-effort only; the stored retry leaders came from a previous
+				// checked-prune decision for this exact entry.
+			}
+
+			if (leaders.has(selfHash)) {
+				await this.cancelCheckedPruneForLocalLeader(hash);
+				continue;
+			}
+			if (leaders.size === 0) {
+				continue;
+			}
+
+			toPrune.set(hash, { entry, leaders });
+			if (leaders.has(publicKeyHash)) {
+				responseStillApplies.push(hash);
+			}
+		}
+
+		if (toPrune.size === 0) {
+			return;
+		}
+
+		void Promise.allSettled(this.prune(toPrune));
+		for (const hash of responseStillApplies) {
+			void this._checkedPrune.getPendingDelete(hash)?.resolve(publicKeyHash);
+		}
 	}
 
 	async append(
@@ -3947,7 +4079,7 @@ export class SharedLog<
 			this.domain.resolution,
 		);
 		this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 2e4;
-		this._pendingDeletes = new Map();
+		this._checkedPrune = new CheckedPruneCoordinator<T, R>();
 		this._pendingIHave = new Map();
 		this.latestReplicationInfoMessage = new Map();
 		this._replicationInfoBlockedPeers = new Set();
@@ -4139,10 +4271,6 @@ export class SharedLog<
 			})) > 0;
 
 		this._gidPeersHistory = new Map();
-		this._requestIPruneSent = new Map();
-		this._requestIPruneResponseReplicatorSet = new Map();
-		this._checkedPruneRetries = new Map();
-
 		this.replicationChangeDebounceFn = debounceAggregationChanges<
 			ReplicationRangeIndexable<R>
 		>(
@@ -4164,14 +4292,24 @@ export class SharedLog<
 				>();
 				const selfReplicating = await this.isReplicating();
 				for (const [hash, value] of map) {
-					const checkedPruneLeaders = await this.resolveCheckedPruneLeaders({
-						hash,
-						entry: value.entry,
-						leaders: value.leaders,
-						selfReplicating,
-					});
+						const checkedPruneLeaders =
+							await this.revalidateCheckedPruneOwnership({
+								hash,
+								entry: value.entry,
+								leaders: value.leaders,
+								selfReplicating,
+							});
 					if (checkedPruneLeaders.localLeader) {
-						await this.cancelCheckedPruneForLocalLeader(hash);
+						const preserveRetry = this._checkedPrune.hasRetry(hash);
+						await this.cancelCheckedPruneForLocalLeader(hash, {
+							preserveRetry,
+						});
+						if (preserveRetry) {
+							this.scheduleCheckedPruneRetry({
+								entry: value.entry,
+								leaders: checkedPruneLeaders.leaders,
+							});
+						}
 						continue;
 					}
 					current.set(hash, {
@@ -4217,7 +4355,7 @@ export class SharedLog<
 							to: allRequestingPeers,
 							redundancy: 1,
 						}),
-						priority: 1,
+						priority: CONVERGENCE_MESSAGE_PRIORITY,
 					});
 			},
 			() => {
@@ -4581,20 +4719,7 @@ export class SharedLog<
 		this.cancelReplicationInfoRequests(peerHash);
 		this._replicatorLivenessFailures.delete(peerHash);
 		this._replicatorLastActivityAt.delete(peerHash);
-
-		for (const [hash, peers] of this._requestIPruneSent) {
-			peers.delete(peerHash);
-			if (peers.size === 0) {
-				this._requestIPruneSent.delete(hash);
-			}
-		}
-
-		for (const [hash, peers] of this._requestIPruneResponseReplicatorSet) {
-			peers.delete(peerHash);
-			if (peers.size === 0) {
-				this._requestIPruneResponseReplicatorSet.delete(hash);
-			}
-		}
+		this._checkedPrune.cleanupPeer(peerHash);
 	}
 
 	private markReplicatorActivity(peerHash: string, now = Date.now()) {
@@ -4658,7 +4783,7 @@ export class SharedLog<
 		const self = this.node.identity.publicKey.hashcode();
 		const seed = hashToSeed32(hash);
 
-		const hinted = this._requestIPruneResponseReplicatorSet.get(hash);
+		const hinted = this._checkedPrune.getConfirmedReplicators(hash);
 		if (hinted && hinted.size > 0) {
 			const peers = [...hinted].filter((p) => p !== self);
 			return peers.length > 0
@@ -4666,7 +4791,7 @@ export class SharedLog<
 				: undefined;
 		}
 
-		const contacted = this._requestIPruneSent.get(hash);
+		const contacted = this._checkedPrune.getContactedReplicators(hash);
 		if (contacted && contacted.size > 0) {
 			const peers = [...contacted].filter((p) => p !== self);
 			return peers.length > 0
@@ -5084,25 +5209,15 @@ export class SharedLog<
 		}
 		this._appendBackfillPendingByTarget.clear();
 
-		for (const [_k, v] of this._pendingDeletes) {
-			v.clear();
-			v.promise.resolve(); // TODO or reject?
-		}
 		for (const [_k, v] of this._pendingIHave) {
 			v.clear();
 		}
-		for (const [_k, v] of this._checkedPruneRetries) {
-			if (v.timer) clearTimeout(v.timer);
-		}
+		this._checkedPrune.close();
 
 		await this.remoteBlocks.stop();
-		this._pendingDeletes.clear();
 		this._pendingIHave.clear();
-		this._checkedPruneRetries.clear();
 		this.latestReplicationInfoMessage.clear();
 		this._gidPeersHistory.clear();
-		this._requestIPruneSent.clear();
-		this._requestIPruneResponseReplicatorSet.clear();
 		// Cancel any pending debounced timers so they can't fire after we've torn down
 		// indexes/RPC state.
 		this.rebalanceParticipationDebounced?.close();
@@ -5158,7 +5273,7 @@ export class SharedLog<
 						try {
 							await this.rpc
 								.send(new AllReplicatingSegmentsMessage({ segments: [] }), {
-									priority: 1,
+									priority: CONVERGENCE_MESSAGE_PRIORITY,
 									signal: abort.signal,
 								})
 								.catch(() => {});
@@ -5205,7 +5320,7 @@ export class SharedLog<
 					try {
 						await this.rpc
 							.send(new AllReplicatingSegmentsMessage({ segments: [] }), {
-								priority: 1,
+								priority: CONVERGENCE_MESSAGE_PRIORITY,
 								signal: abort.signal,
 							})
 							.catch(() => {});
@@ -5390,7 +5505,7 @@ export class SharedLog<
 								for (const entry of entries) {
 									this.pruneDebouncedFn.delete(entry.entry.hash);
 									this.removePruneRequestSent(entry.entry.hash);
-									this._requestIPruneResponseReplicatorSet.delete(
+									this._checkedPrune.clearConfirmedReplicators(
 										entry.entry.hash,
 									);
 
@@ -5513,44 +5628,51 @@ export class SharedLog<
 
 					// if we expect the remote to be owner of this entry because we are to prune ourselves, then we need to remove the remote
 					// this is due to that the remote has previously indicated to be a replicator to help us prune but now has changed their mind
-					const outGoingPrunes =
-						this._requestIPruneResponseReplicatorSet.get(hash);
-					if (outGoingPrunes) {
-						outGoingPrunes.delete(from);
-					}
+					this._checkedPrune.removeConfirmedReplicator(hash, from);
 
 					const indexedEntry = await this.log.entryIndex.getShallow(hash);
 					let isLeader = false;
 
-					if (
-						indexedEntry &&
-						!this._pendingDeletes.has(hash) &&
-						(await this.log.blocks.has(hash))
-					) {
-						this.removePeerFromGidPeerHistory(
-							context.from!.hashcode(),
-							indexedEntry!.value.meta.gid,
-						);
+					if (indexedEntry && (await this.log.blocks.has(hash))) {
+						const pendingDelete = this._checkedPrune.getPendingDelete(hash);
+						if (pendingDelete) {
+							const ownership =
+								await this.revalidateCheckedPruneOwnership({
+									hash,
+									entry: indexedEntry.value,
+									leaders: new Map(),
+								});
+							if (ownership.localLeader) {
+								await this.cancelCheckedPruneForLocalLeader(hash);
+								isLeader = true;
+							}
+						} else {
+							this.removePeerFromGidPeerHistory(
+								context.from!.hashcode(),
+								indexedEntry!.value.meta.gid,
+							);
 
-						await this._waitForReplicators(
-							await this.createCoordinates(
+							await this._waitForReplicators(
+								await this.createCoordinates(
+									indexedEntry.value,
+									decodeReplicas(indexedEntry.value).getValue(this),
+								),
 								indexedEntry.value,
-								decodeReplicas(indexedEntry.value).getValue(this),
-							),
-							indexedEntry.value,
-							[
+								[
+									{
+										key: this.node.identity.publicKey.hashcode(),
+										replicator: true,
+									},
+								],
 								{
-									key: this.node.identity.publicKey.hashcode(),
-									replicator: true,
-								},
-							],
-							{
-								onLeader: (key) => {
-									isLeader =
-										isLeader || key === this.node.identity.publicKey.hashcode();
+									onLeader: (key) => {
+										isLeader =
+											isLeader ||
+											key === this.node.identity.publicKey.hashcode();
+									},
 								},
-							},
-						);
+							);
+						}
 					}
 
 					if (isLeader) {
@@ -5625,8 +5747,20 @@ export class SharedLog<
 					}
 				}
 			} else if (msg instanceof ResponseIPrune) {
+				const lateResponses: string[] = [];
 				for (const hash of msg.hashes) {
-					this._pendingDeletes.get(hash)?.resolve(context.from.hashcode());
+					const pendingDelete = this._checkedPrune.getPendingDelete(hash);
+					if (pendingDelete) {
+						void pendingDelete.resolve(context.from.hashcode());
+					} else {
+						lateResponses.push(hash);
+					}
+				}
+				if (lateResponses.length > 0) {
+					void this.recoverCheckedPruneFromLateResponses(
+						lateResponses,
+						context.from.hashcode(),
+					).catch((error) => logger.error(error.toString()));
 				}
 			} else if (msg instanceof ConfirmEntriesMessage) {
 				this.markEntriesKnownByPeer(msg.hashes, context.from.hashcode());
@@ -6105,7 +6239,7 @@ export class SharedLog<
 
 			if (messageToSend) {
 				await this.rpc.send(messageToSend, {
-					priority: 1,
+					priority: CONVERGENCE_MESSAGE_PRIORITY,
 				});
 			}
 		}
@@ -6892,11 +7026,12 @@ export class SharedLog<
 		this._replicationInfoRequestByPeer.set(peerHash, state);
 
 		const intervalMs = Math.max(50, this.waitForReplicatorRequestIntervalMs);
-		const maxAttempts = Math.min(
-			5,
+		const maxAttempts =
 			this.waitForReplicatorRequestMaxAttempts ??
+			Math.max(
 				WAIT_FOR_REPLICATOR_REQUEST_MIN_ATTEMPTS,
-		);
+				Math.ceil(this.waitForReplicatorTimeout / intervalMs),
+			);
 
 		const tick = () => {
 			if (this.closed || this._closeController.signal.aborted) {
@@ -7022,17 +7157,7 @@ export class SharedLog<
 	}
 
 	private removePruneRequestSent(hash: string, to?: string) {
-		if (!to) {
-			this._requestIPruneSent.delete(hash);
-		} else {
-			let set = this._requestIPruneSent.get(hash);
-			if (set) {
-				set.delete(to);
-				if (set.size === 0) {
-					this._requestIPruneSent.delete(hash);
-				}
-			}
-		}
+		this._checkedPrune.removeRequestSent(hash, to);
 	}
 
 	prune(
@@ -7049,7 +7174,7 @@ export class SharedLog<
 			return [...entries.values()].map((x) => {
 				this._gidPeersHistory.delete(x.entry.meta.gid);
 				this.removePruneRequestSent(x.entry.hash);
-				this._requestIPruneResponseReplicatorSet.delete(x.entry.hash);
+				this._checkedPrune.clearConfirmedReplicators(x.entry.hash);
 				return this.log.remove(x.entry, {
 					recursively: true,
 				});
@@ -7086,7 +7211,7 @@ export class SharedLog<
 					set.push(entry.hash);
 				}
 
-				const pendingPrev = this._pendingDeletes.get(entry.hash);
+				const pendingPrev = this._checkedPrune.getPendingDelete(entry.hash);
 				if (pendingPrev) {
 					// If a background prune is already in-flight, an explicit prune request should
 					// still respect the caller's timeout. Otherwise, tests (and user calls) can
@@ -7122,62 +7247,77 @@ export class SharedLog<
 				const deferredPromise: DeferredPromise<void> = pDefer();
 
 			const clear = () => {
-				const pending = this._pendingDeletes.get(entry.hash);
+				const pending = this._checkedPrune.getPendingDelete(entry.hash);
 				if (pending?.promise === deferredPromise) {
-					this._pendingDeletes.delete(entry.hash);
+					this._checkedPrune.deletePendingDelete(entry.hash, pending);
 				}
 				clearTimeout(timeout);
 			};
 
-				const resolve = () => {
-					clear();
-					this.clearCheckedPruneRetry(entry.hash);
-					cleanupTimer.push(
-						setTimeout(async () => {
-							this._gidPeersHistory.delete(entry.meta.gid);
-							this.removePruneRequestSent(entry.hash);
-						this._requestIPruneResponseReplicatorSet.delete(entry.hash);
-
-						if (
-							await this.isLeader({
-								entry,
-								replicas: minReplicas.getValue(this),
-							})
-						) {
-							deferredPromise.reject(
-								new Error("Failed to delete, is leader again"),
-							);
-							return;
-						}
-
-						return this.log
-							.remove(entry, {
-								recursively: true,
-							})
-							.then(() => {
-								deferredPromise.resolve();
-							})
-							.catch((e) => {
-								deferredPromise.reject(e);
-							})
-							.finally(async () => {
+					const resolve = () => {
+						clearTimeout(timeout);
+						this.clearCheckedPruneRetry(entry.hash);
+						cleanupTimer.push(
+							setTimeout(async () => {
 								this._gidPeersHistory.delete(entry.meta.gid);
 								this.removePruneRequestSent(entry.hash);
-								this._requestIPruneResponseReplicatorSet.delete(entry.hash);
-								// TODO in the case we become leader again here we need to re-add the entry
+								this._checkedPrune.clearConfirmedReplicators(entry.hash);
 
-								if (
-									await this.isLeader({
+								const ownership =
+									await this.revalidateCheckedPruneOwnership({
+										hash: entry.hash,
 										entry,
-										replicas: minReplicas.getValue(this),
-									})
-								) {
-									logger.error("Unexpected: Is leader after delete");
+										leaders: this.checkedPruneLeadersToMap(leaders),
+										selfReplicating: true,
+									});
+								if (ownership.localLeader) {
+									clear();
+									if (!explicitTimeout) {
+										this.scheduleCheckedPruneRetry({ entry, leaders });
+									}
+									deferredPromise.reject(
+										new Error("Failed to delete, is leader again"),
+									);
+									return;
 								}
-							});
-					}, this.waitForPruneDelay),
-				);
-			};
+
+								this._checkedPrune.markRemoving(entry.hash);
+								return this.log
+									.remove(entry, {
+										recursively: true,
+									})
+									.then(() => {
+										clear();
+										this._checkedPrune.markDone(entry.hash);
+										deferredPromise.resolve();
+									})
+									.catch((e) => {
+										clear();
+										this._checkedPrune.markCancelled(entry.hash, {
+											preserveRetry: false,
+										});
+										deferredPromise.reject(e);
+									})
+									.finally(async () => {
+										this._gidPeersHistory.delete(entry.meta.gid);
+										this.removePruneRequestSent(entry.hash);
+										this._checkedPrune.clearConfirmedReplicators(entry.hash);
+										// TODO in the case we become leader again here we need to re-add the entry
+
+										const ownership =
+											await this.revalidateCheckedPruneOwnership({
+												hash: entry.hash,
+												entry,
+												leaders: this.checkedPruneLeadersToMap(leaders),
+												selfReplicating: true,
+											});
+										if (ownership.localLeader) {
+											logger.error("Unexpected: Is leader after delete");
+										}
+									});
+							}, this.waitForPruneDelay),
+						);
+					};
 
 				const reject = (e: any) => {
 					clear();
@@ -7185,11 +7325,9 @@ export class SharedLog<
 						e instanceof Error &&
 						typeof e.message === "string" &&
 						e.message.startsWith("Timeout for checked pruning");
-					if (explicitTimeout || !isCheckedPruneTimeout) {
-						this.clearCheckedPruneRetry(entry.hash);
-					}
-					this.removePruneRequestSent(entry.hash);
-					this._requestIPruneResponseReplicatorSet.delete(entry.hash);
+					this._checkedPrune.markCancelled(entry.hash, {
+						preserveRetry: !explicitTimeout && isCheckedPruneTimeout,
+					});
 					deferredPromise.reject(e);
 				};
 
@@ -7203,70 +7341,65 @@ export class SharedLog<
 			const checkedPruneTimeoutMs =
 				options?.timeout ??
 				Math.max(
-					10_000,
+					CHECKED_PRUNE_BACKGROUND_TIMEOUT_MIN_MS,
 					Number(this._respondToIHaveTimeout ?? 0) +
 						this.waitForReplicatorTimeout +
 						PRUNE_DEBOUNCE_INTERVAL * 2,
 				);
 
-				const timeout = setTimeout(() => {
-					// For internal/background prune flows (no explicit timeout), retry a few times
-					// to avoid "permanently prunable" entries when `_pendingIHave` expires under
-					// heavy load.
-					if (!explicitTimeout) {
-						this.scheduleCheckedPruneRetry({ entry, leaders });
-					}
-					reject(
-						new Error(
-							`Timeout for checked pruning after ${checkedPruneTimeoutMs}ms (closed=${this.closed})`,
-						),
-					);
-				}, checkedPruneTimeoutMs);
-				timeout.unref?.();
-
-			this._pendingDeletes.set(entry.hash, {
-				promise: deferredPromise,
-				clear,
-				reject,
-				resolve: async (publicKeyHash: string) => {
-					const minReplicasObj = this.getClampedReplicas(minReplicas);
-					const minReplicasValue = minReplicasObj.getValue(this);
-
-					// TODO is this check necessary
-					if (
-						!(await this._waitForReplicators(
-							cursor ??
-								(cursor = await this.createCoordinates(
-									entry,
-									minReplicasValue,
-								)),
-							entry,
-							[
-								{ key: publicKeyHash, replicator: true },
+			const timeout = setTimeout(() => {
+				// For internal/background prune flows (no explicit timeout), retry a few times
+				// to avoid "permanently prunable" entries when `_pendingIHave` expires under
+				// heavy load.
+				if (!explicitTimeout) {
+					this.scheduleCheckedPruneRetry({ entry, leaders });
+				}
+				reject(
+					new Error(
+						`Timeout for checked pruning after ${checkedPruneTimeoutMs}ms (closed=${this.closed})`,
+					),
+				);
+			}, checkedPruneTimeoutMs);
+			timeout.unref?.();
+
+			this._checkedPrune.setPendingDelete(
+				entry.hash,
+				{
+					promise: deferredPromise,
+					clear,
+					reject,
+					resolve: async (publicKeyHash: string) => {
+						const minReplicasObj = this.getClampedReplicas(minReplicas);
+						const minReplicasValue = minReplicasObj.getValue(this);
+
+						// TODO is this check necessary
+						if (
+							!(await this._waitForReplicators(
+								cursor ??
+									(cursor = await this.createCoordinates(
+										entry,
+										minReplicasValue,
+									)),
+								entry,
+								[
+									{ key: publicKeyHash, replicator: true },
+									{
+										key: this.node.identity.publicKey.hashcode(),
+										replicator: false,
+									},
+								],
 								{
-									key: this.node.identity.publicKey.hashcode(),
-									replicator: false,
+									persist: false,
 								},
-							],
-							{
-								persist: false,
-							},
-						))
-					) {
-						return;
-					}
-
-					let existCounter = this._requestIPruneResponseReplicatorSet.get(
-						entry.hash,
-					);
-						if (!existCounter) {
-							existCounter = new Set();
-							this._requestIPruneResponseReplicatorSet.set(
-								entry.hash,
-								existCounter,
-							);
+							))
+						) {
+							return;
 						}
-						existCounter.add(publicKeyHash);
+
+						const existCounter = this._checkedPrune.addConfirmedReplicator(
+							entry.hash,
+							publicKeyHash,
+						);
 						// Seed provider hints so future remote reads can avoid extra round-trips.
 						this.remoteBlocks.hintProviders(entry.hash, [publicKeyHash]);
 
@@ -7274,7 +7407,10 @@ export class SharedLog<
 							resolve();
 						}
 					},
-				});
+				},
+				entry,
+				leaders,
+			);
 
 			promises.push(deferredPromise.promise);
 		}
@@ -7282,16 +7418,11 @@ export class SharedLog<
 		const emitMessages = async (entries: string[], to: string) => {
 			const filteredSet: string[] = [];
 			for (const entry of entries) {
-				let set = this._requestIPruneSent.get(entry);
-				if (!set) {
-					set = new Set();
-					this._requestIPruneSent.set(entry, set);
-				}
 				/* TODO why can we not have this statement? 
 				if (set.has(to)) {
 					continue;
 				} */
-				set.add(to);
+				this._checkedPrune.addRequestSent(entry, to);
 				filteredSet.push(entry);
 			}
 			if (filteredSet.length > 0) {
@@ -7304,7 +7435,7 @@ export class SharedLog<
 							to: [to], // TODO group by peers?
 							redundancy: 1,
 						}),
-						priority: 1,
+						priority: CONVERGENCE_MESSAGE_PRIORITY,
 					},
 				);
 			}
@@ -7333,7 +7464,9 @@ export class SharedLog<
 
 					const pendingByPeer: [string, string[]][] = [];
 					for (const [peer, hashes] of peerToEntries) {
-						const pending = hashes.filter((h) => this._pendingDeletes.has(h));
+						const pending = hashes.filter((h) =>
+							this._checkedPrune.hasPendingDelete(h),
+						);
 						if (pending.length > 0) {
 							pendingByPeer.push([peer, pending]);
 						}
@@ -7764,12 +7897,26 @@ export class SharedLog<
 				flushUncheckedDeliverTarget(target);
 			}
 
-			if (this._isAdaptiveReplicating && hasSelfRangeRemoval) {
-				// Adaptive shrink/replacement can make already-indexed local heads
-				// prunable even when the incremental rebalance scan missed them under
-				// churn or timing pressure. Re-scan after repair dispatches are flushed
-				// so checked prune work is enqueued before callers wait for idle.
-				await this.pruneIndexedEntriesNoLongerLed();
+			if (
+				this._isAdaptiveReplicating &&
+				(hasSelfRangeRemoval ||
+					changes.some(
+						(change) =>
+							change.type === "added" ||
+							change.type === "removed" ||
+							change.type === "replaced",
+					))
+			) {
+				// Adaptive range changes can make already-indexed local heads prunable
+				// even when the incremental rebalance scan misses them under churn or
+				// timing pressure. Re-scan after repair dispatches are flushed using the
+				// mature-role view, which matches the bounded pruning contract.
+				await this.pruneIndexedEntriesNoLongerLed({
+					useDefaultRoleAge: true,
+				});
+				await this.pruneCurrentHeadsNoLongerLed({
+					useDefaultRoleAge: true,
+				});
 			}
 
 			return changed;