@peerbit/shared-log 16.0.25 → 16.0.26

package/src/index.ts

@@ -4087,10 +4087,15 @@ export class SharedLog<
 						: 0),
 			}),
 			{
-				mode: new AcknowledgeDelivery({
-					redundancy: 1,
-					to: [properties.target],
-				}),
+				// The transient rearm is only a bounded recovery hint. The ensuing
+				// authenticated Full/Applied exchange is the readiness proof, so do not
+				// let an ACK-delivery promise park the recovery scheduler itself.
+				mode: properties.requestRemoteFullRearm
+					? new SilentDelivery({ redundancy: 1, to: [properties.target] })
+					: new AcknowledgeDelivery({
+							redundancy: 1,
+							to: [properties.target],
+						}),
 				priority: CONVERGENCE_MESSAGE_PRIORITY,
 				signal: properties.signal,
 			},
@@ -24381,6 +24386,11 @@ export class SharedLog<
 	private nudgePersistedReceiptPeerReadiness(
 		publicKey: PublicSignKey,
 		signal: AbortSignal,
+		rearmCurrentUnconfirmed?: {
+			peerSession: PeerSession;
+			receiveEpoch: object | null;
+			capabilitySession: bigint;
+		},
 	): void {
 		if (this.closed || signal.aborted) return;
 		const peerHash = publicKey.hashcode();
@@ -24416,11 +24426,20 @@ export class SharedLog<
 		const receiptTarget = this.persistedReceiptPeerSession(peerHash);
 		if (
 			receiptTarget &&
-			!this._v2Send.hasCurrentStateForPeer({
+			(!this._v2Send.hasCurrentStateForPeer({
 				peerHash,
 				peerSession: receiptTarget.peerSession,
 				receiverTransportSession: receiptTarget.capabilitySession,
-			})
+			}) ||
+				(rearmCurrentUnconfirmed?.peerSession === receiptTarget.peerSession &&
+					rearmCurrentUnconfirmed.receiveEpoch === receiptTarget.receiveEpoch &&
+					rearmCurrentUnconfirmed.capabilitySession ===
+						receiptTarget.capabilitySession &&
+					!this._v2Send.isLatestConfirmedForPeer({
+						peerHash,
+						peerSession: receiptTarget.peerSession,
+						receiverTransportSession: receiptTarget.capabilitySession,
+					})))
 		) {
 			this._v2Receive.reAdvertiseLocalCapabilityForRemoteFull({
 				peerHash,
@@ -24563,7 +24582,9 @@ export class SharedLog<
 	 * Wait for a public key's current (or replacement) connection generation to
 	 * become persisted-receipt ready. Transition listeners are installed before
 	 * the first asynchronous inspection, and a bounded recovery tick repairs
-	 * missed subscriber/capability wakes without retaining stale PeerSessions.
+	 * missed subscriber/capability wakes without retaining stale PeerSessions. A
+	 * current sender gets one normal confirmation interval before an exact-
+	 * generation watchdog requests a bounded Full rearm.
 	 * This waiter is advisory only; the following persisted delivery remains the
 	 * operation that proves the requested remote durability quorum.
 	 */
@@ -24621,6 +24642,14 @@ export class SharedLog<
 		let rerun = false;
 		let recoveryTimer: ReturnType<typeof setTimeout> | undefined;
 		let confirmationController: AbortController | undefined;
+		let confirmationRecoveryTarget:
+			| {
+					peerSession: PeerSession;
+					receiveEpoch: object | null;
+					capabilitySession: bigint;
+					notBefore: number;
+			  }
+			| undefined;
 		let lastSnapshot: PersistedReceiptPeerReadiness | undefined;
 		const createTimeoutError = () => {
 			const suffix = lastSnapshot
@@ -24651,6 +24680,7 @@ export class SharedLog<
 				new AbortError("Persisted-receipt readiness generation changed"),
 			);
 			confirmationController = undefined;
+			confirmationRecoveryTarget = undefined;
 			operationController.abort(
 				new AbortError("Persisted-receipt readiness wait settled"),
 			);
@@ -24706,7 +24736,14 @@ export class SharedLog<
 			recoveryTimer = setTimeout(() => {
 				recoveryTimer = undefined;
 				if (!continueWait()) return;
-				this.nudgePersistedReceiptPeerReadiness(key, operationSignal);
+				const recoveryTarget = confirmationRecoveryTarget;
+				this.nudgePersistedReceiptPeerReadiness(
+					key,
+					operationSignal,
+					recoveryTarget && Date.now() >= recoveryTarget.notBefore
+						? recoveryTarget
+						: undefined,
+				);
 				if (checkInFlight) {
 					// Confirmation can legitimately span several recovery intervals.
 					// Keep repairing the exact current generation without aborting its
@@ -24719,6 +24756,34 @@ export class SharedLog<
 			}, recoveryDelayMs);
 			recoveryTimer.unref?.();
 		};
+		const prepareConfirmationRecovery = () => {
+			const target = this.persistedReceiptPeerSession(peerHash);
+			if (!target) {
+				confirmationRecoveryTarget = undefined;
+				return undefined;
+			}
+			const retained = confirmationRecoveryTarget;
+			if (
+				retained?.peerSession !== target.peerSession ||
+				retained.receiveEpoch !== target.receiveEpoch ||
+				retained.capabilitySession !== target.capabilitySession
+			) {
+				confirmationRecoveryTarget = {
+					peerSession: target.peerSession,
+					receiveEpoch: target.receiveEpoch,
+					capabilitySession: target.capabilitySession,
+					notBefore: Date.now() + recoveryDelayMs,
+				};
+			}
+			const recoveryTarget = confirmationRecoveryTarget!;
+			this.nudgePersistedReceiptPeerReadiness(
+				key,
+				operationSignal,
+				Date.now() >= recoveryTarget.notBefore ? recoveryTarget : undefined,
+			);
+			armRecoveryTick();
+			return target;
+		};
 		const runCheck = async () => {
 			checkScheduled = false;
 			if (!continueWait()) return;
@@ -24735,8 +24800,9 @@ export class SharedLog<
 				);
 				lastSnapshot = snapshot;
 				if (!continueWait()) return;
-				if (rerun) return;
 				if (snapshot.status === "ready") {
+					confirmationRecoveryTarget = undefined;
+					if (rerun) return;
 					// A wake observed while the asynchronous inspection was running may
 					// already have invalidated this snapshot. Drain that coalesced wake
 					// before publishing readiness.
@@ -24747,15 +24813,17 @@ export class SharedLog<
 					snapshot.status === "pending" &&
 					snapshot.reason === "replication-confirmation-pending"
 				) {
-					const target = this.persistedReceiptPeerSession(peerHash);
+					// Retain one watchdog deadline for the exact generation across
+					// coalesced inspection wakes. A sub-interval event stream must drive,
+					// rather than indefinitely postpone, the mature recovery nudge.
+					const target = prepareConfirmationRecovery();
+					if (rerun) return;
 					if (target) {
-						// A confirmation waiter cannot create an outbound V2 stream when
-						// that exact session state is missing. Nudge first, then keep the
-						// recovery tick alive while the single confirmation wait remains.
-						this.nudgePersistedReceiptPeerReadiness(key, operationSignal);
-						armRecoveryTick();
 						const currentConfirmationController = new AbortController();
 						confirmationController = currentConfirmationController;
+						const currentConfirmationRecoveryTarget =
+							confirmationRecoveryTarget;
+						let confirmationSucceeded = false;
 						try {
 							await this._v2Send.confirmLatestForPeer(
 								{
@@ -24771,6 +24839,7 @@ export class SharedLog<
 									]),
 								},
 							);
+							confirmationSucceeded = true;
 						} catch (error) {
 							if (!continueWait()) return;
 							if (!(error instanceof AbortError)) {
@@ -24781,6 +24850,12 @@ export class SharedLog<
 							if (confirmationController === currentConfirmationController) {
 								confirmationController = undefined;
 							}
+							if (
+								confirmationSucceeded &&
+								confirmationRecoveryTarget === currentConfirmationRecoveryTarget
+							) {
+								confirmationRecoveryTarget = undefined;
+							}
 						}
 						if (!continueWait()) return;
 						snapshot = await this.inspectPersistedReceiptPeerReadiness(
@@ -24790,13 +24865,26 @@ export class SharedLog<
 						);
 						lastSnapshot = snapshot;
 						if (!continueWait()) return;
-						if (rerun) return;
 						if (snapshot.status === "ready") {
+							confirmationRecoveryTarget = undefined;
+							if (rerun) return;
 							resolve(snapshot);
 							return;
 						}
+						if (
+							snapshot.status === "pending" &&
+							snapshot.reason === "replication-confirmation-pending"
+						) {
+							prepareConfirmationRecovery();
+						} else {
+							confirmationRecoveryTarget = undefined;
+						}
+						if (rerun) return;
 					}
+				} else {
+					confirmationRecoveryTarget = undefined;
 				}
+				if (rerun) return;
 				if (!continueWait()) return;
 				this.nudgePersistedReceiptPeerReadiness(key, operationSignal);
 			} catch (error) {

package/src/replication-info-v2-receive.ts

@@ -25,6 +25,11 @@ const REQUIRED_SENDER_CAPABILITIES =
 const DEFAULT_REQUEST_RETRY_MS = 1_000;
 const DEFAULT_MAX_REQUEST_RETRY_MS = 30_000;
 const DEFAULT_REQUEST_MAX_ATTEMPTS = 7;
+const DEFAULT_REMOTE_FULL_REARM_ATTEMPT_TIMEOUT_MS = 2_000;
+const DEFAULT_REMOTE_FULL_REARM_COOLDOWN_MS = 5_000;
+const MAX_REMOTE_FULL_REARM_OUTSTANDING_PER_SESSION = 2;
+const MAX_REMOTE_FULL_REARM_OUTSTANDING_GLOBAL = 64;
+const MAX_TIMER_MS = 2_147_483_647;
 const MAX_U64 = (1n << 64n) - 1n;
 const MAX_BACKOFF_EXPONENT = 20;
 
@@ -113,8 +118,13 @@ export type ReplicationInfoV2LocalCapabilityAdvertisement = {
 };
 
 type ReplicationInfoV2RemoteFullRearm = {
-	lastAttemptAt: number;
+	peerHash: string;
+	receiveEpoch: object | null;
+	receiverTransportSession: bigint;
+	nextAttemptAt: number;
+	outstandingAttempts: number;
 	inFlight?: Promise<void>;
+	controller?: AbortController;
 };
 
 export type ReplicationInfoV2LocalCapabilityRefresh = {
@@ -196,6 +206,9 @@ export type ReplicationInfoV2ReceiveDeps = {
 	requestRetryMs?: number;
 	maxRequestRetryMs?: number;
 	requestMaxAttempts?: number;
+	remoteFullRearmAttemptTimeoutMs?: number;
+	remoteFullRearmCooldownMs?: number;
+	maxRemoteFullRearmOutstandingGlobal?: number;
 };
 
 /**
@@ -224,12 +237,17 @@ export class ReplicationInfoV2ReceiveCoordinator {
 		ReplicationInfoV2LocalCapabilityAdvertisement
 	>;
 	_remoteFullRearmBySession!: WeakMap<object, ReplicationInfoV2RemoteFullRearm>;
+	_remoteFullRearmsInFlight!: Set<ReplicationInfoV2RemoteFullRearm>;
+	_remoteFullRearmOutstanding!: Set<object>;
 	_reservedAdmissionsByPeer!: Map<string, ReplicationInfoV2ReceiveAdmission>;
 
 	private readonly now: () => number;
 	private readonly requestRetryMs: number;
 	private readonly maxRequestRetryMs: number;
 	private readonly requestMaxAttempts: number;
+	private readonly remoteFullRearmAttemptTimeoutMs: number;
+	private readonly remoteFullRearmCooldownMs: number;
+	private readonly maxRemoteFullRearmOutstandingGlobal: number;
 
 	constructor(private readonly deps: ReplicationInfoV2ReceiveDeps) {
 		this.now = deps.now ?? Date.now;
@@ -245,12 +263,46 @@ export class ReplicationInfoV2ReceiveCoordinator {
 			1,
 			Math.floor(deps.requestMaxAttempts ?? DEFAULT_REQUEST_MAX_ATTEMPTS),
 		);
+		this.remoteFullRearmAttemptTimeoutMs = Math.max(
+			1,
+			Math.min(
+				MAX_TIMER_MS,
+				Math.floor(
+					deps.remoteFullRearmAttemptTimeoutMs ??
+						DEFAULT_REMOTE_FULL_REARM_ATTEMPT_TIMEOUT_MS,
+				),
+			),
+		);
+		this.remoteFullRearmCooldownMs = Math.max(
+			this.requestRetryMs,
+			Math.min(
+				MAX_TIMER_MS,
+				Math.floor(
+					deps.remoteFullRearmCooldownMs ??
+						DEFAULT_REMOTE_FULL_REARM_COOLDOWN_MS,
+				),
+			),
+		);
+		// The injected value is a test seam that may only tighten this lifetime
+		// resource bound, never widen it.
+		this.maxRemoteFullRearmOutstandingGlobal = Math.min(
+			MAX_REMOTE_FULL_REARM_OUTSTANDING_GLOBAL,
+			Math.max(
+				1,
+				Math.floor(
+					deps.maxRemoteFullRearmOutstandingGlobal ??
+						MAX_REMOTE_FULL_REARM_OUTSTANDING_GLOBAL,
+				),
+			),
+		);
 		this._receiveStates = new Map();
 		this._cutoverPeerSessions = new WeakSet();
 		this._localCapabilityReadyBySession = new WeakMap();
 		this._localCapabilityContextBySession = new WeakMap();
 		this._localCapabilityAdvertisementsByPeer = new Map();
 		this._remoteFullRearmBySession = new WeakMap();
+		this._remoteFullRearmsInFlight = new Set();
+		this._remoteFullRearmOutstanding = new Set();
 		this._reservedAdmissionsByPeer = new Map();
 	}
 
@@ -262,10 +314,19 @@ export class ReplicationInfoV2ReceiveCoordinator {
 		this._localCapabilityContextBySession = new WeakMap();
 		this._localCapabilityAdvertisementsByPeer = new Map();
 		this._remoteFullRearmBySession = new WeakMap();
+		this._remoteFullRearmsInFlight = new Set();
 		this._reservedAdmissionsByPeer = new Map();
 	}
 
 	clearForClose(): void {
+		for (const rearm of this._remoteFullRearmsInFlight ?? []) {
+			rearm.controller?.abort(
+				new Error(
+					"Replication-info V2 receiver closed during remote Full rearm",
+				),
+			);
+		}
+		this._remoteFullRearmsInFlight?.clear();
 		for (const advertisement of [
 			...(this._localCapabilityAdvertisementsByPeer?.values() ?? []),
 		]) {
@@ -298,10 +359,28 @@ export class ReplicationInfoV2ReceiveCoordinator {
 			this._localCapabilityContextBySession.delete(state.peerSession);
 			this._cutoverPeerSessions.delete(state.peerSession);
 		}
+		if (!expectedSession) {
+			for (const rearm of this._remoteFullRearmsInFlight) {
+				if (rearm.peerHash === peerHash) {
+					rearm.controller?.abort(
+						new Error("Replication-info V2 peer cleared during rearm"),
+					);
+				}
+			}
+		}
+		const rearmSession =
+			expectedSession ?? state?.peerSession ?? advertisement?.peerSession;
+		if (rearmSession) {
+			this._remoteFullRearmBySession
+				.get(rearmSession)
+				?.controller?.abort(
+					new Error("Replication-info V2 peer session cleared during rearm"),
+				);
+			this._remoteFullRearmBySession.delete(rearmSession);
+		}
 		if (expectedSession) {
 			this._localCapabilityReadyBySession.delete(expectedSession);
 			this._localCapabilityContextBySession.delete(expectedSession);
-			this._remoteFullRearmBySession.delete(expectedSession);
 			this._cutoverPeerSessions.delete(expectedSession);
 		}
 	}
@@ -643,9 +722,14 @@ export class ReplicationInfoV2ReceiveCoordinator {
 	 * stream that was lost while both directions otherwise remained current.
 	 *
 	 * This deliberately does not mutate the steady advertisement worker. One
-	 * attempt is coalesced per PeerSession and bound to the caller's signal; the
-	 * readiness wait's bounded recovery tick decides whether another attempt is
-	 * needed.
+	 * attempt is coalesced per exact PeerSession/receive generation and bound to
+	 * its own deadline as well as the caller's signal. A cooldown avoids rotating
+	 * a healthy in-flight challenge on every recovery tick. The remote rotates
+	 * only when its receive state is active; later hints during resync only nudge
+	 * that same bounded request generation. At most two transports that disregard
+	 * abort remain outstanding for one exact session (64 across this coordinator's
+	 * lifetime); reaching either cap stops new hints while the persisted-readiness
+	 * gate remains fail closed.
 	 */
 	reAdvertiseLocalCapabilityForRemoteFull(properties: {
 		peerHash: string;
@@ -679,47 +763,130 @@ export class ReplicationInfoV2ReceiveCoordinator {
 		}
 
 		const now = this.now();
+		const receiverTransportSession = ready.receiverTransportSession;
 		let rearm = this._remoteFullRearmBySession.get(properties.peerSession);
+		if (
+			rearm &&
+			(rearm.peerHash !== properties.peerHash ||
+				rearm.receiveEpoch !== properties.receiveEpoch ||
+				rearm.receiverTransportSession !== receiverTransportSession)
+		) {
+			rearm.controller?.abort(
+				new Error("Replication-info V2 remote Full rearm generation changed"),
+			);
+			rearm = undefined;
+		}
 		if (rearm?.inFlight) {
 			return true;
 		}
+		if (rearm !== undefined && now < rearm.nextAttemptAt) {
+			return true;
+		}
 		if (
-			rearm !== undefined &&
-			now - rearm.lastAttemptAt < this.requestRetryMs
+			(rearm?.outstandingAttempts ?? 0) >=
+				MAX_REMOTE_FULL_REARM_OUTSTANDING_PER_SESSION ||
+			this._remoteFullRearmOutstanding.size >=
+				this.maxRemoteFullRearmOutstandingGlobal
 		) {
 			return true;
 		}
 		if (!rearm) {
-			rearm = { lastAttemptAt: now };
+			rearm = {
+				peerHash: properties.peerHash,
+				receiveEpoch: properties.receiveEpoch,
+				receiverTransportSession,
+				nextAttemptAt: now + this.remoteFullRearmCooldownMs,
+				outstandingAttempts: 0,
+			};
 			this._remoteFullRearmBySession.set(properties.peerSession, rearm);
 		} else {
-			rearm.lastAttemptAt = now;
-		}
+			rearm.nextAttemptAt = now + this.remoteFullRearmCooldownMs;
+		}
+		const attemptState = rearm;
+		const outstandingReservation = {};
+		attemptState.outstandingAttempts++;
+		this._remoteFullRearmOutstanding.add(outstandingReservation);
+
+		const attemptController = new AbortController();
+		const operationSignal = attemptController.signal;
+		attemptState.controller = attemptController;
+		this._remoteFullRearmsInFlight.add(attemptState);
+		const sourceSignals = Array.from(
+			new Set([context.lifecycleSignal, properties.signal]),
+		);
+		const onSourceAbort = (event: Event) => {
+			const source = event.currentTarget as AbortSignal;
+			attemptController.abort(source.reason);
+		};
+		for (const source of sourceSignals) {
+			source.addEventListener("abort", onSourceAbort, { once: true });
+		}
+		const attemptTimer = setTimeout(
+			() =>
+				attemptController.abort(
+					new Error("Replication-info V2 remote Full rearm attempt timed out"),
+				),
+			this.remoteFullRearmAttemptTimeoutMs,
+		);
+		attemptTimer.unref?.();
+		let releasedOutstanding = false;
+		const releaseOutstanding = () => {
+			if (releasedOutstanding) return;
+			releasedOutstanding = true;
+			this._remoteFullRearmOutstanding.delete(outstandingReservation);
+			attemptState.outstandingAttempts--;
+		};
+		const refresh = Promise.resolve().then(() => {
+			if (operationSignal.aborted) {
+				throw operationSignal.reason;
+			}
+			return this.deps.refreshLocalCapability({
+				peerHash: properties.peerHash,
+				target: context.target,
+				peerSession: properties.peerSession,
+				receiveEpoch: properties.receiveEpoch,
+				signal: operationSignal,
+				requestRemoteFullRearm: true,
+			});
+		});
+		void refresh.then(releaseOutstanding, releaseOutstanding);
 
-		const operationSignal = AbortSignal.any([
-			context.lifecycleSignal,
-			properties.signal,
-		]);
 		let operation: Promise<void>;
 		const detach = () => {
-			if (rearm?.inFlight === operation) {
-				rearm.inFlight = undefined;
+			clearTimeout(attemptTimer);
+			operationSignal.removeEventListener("abort", detach);
+			for (const source of sourceSignals) {
+				source.removeEventListener("abort", onSourceAbort);
+			}
+			if (attemptState.inFlight === operation) {
+				this._remoteFullRearmsInFlight.delete(attemptState);
+				attemptState.inFlight = undefined;
+				attemptState.controller = undefined;
 			}
 		};
-		operation = Promise.resolve()
-			.then(() => {
+		const raceWithAttemptSignal = <T>(promise: Promise<T>): Promise<T> =>
+			new Promise<T>((resolve, reject) => {
+				const onAbort = () => {
+					operationSignal.removeEventListener("abort", onAbort);
+					reject(operationSignal.reason);
+				};
+				operationSignal.addEventListener("abort", onAbort, { once: true });
 				if (operationSignal.aborted) {
-					throw operationSignal.reason;
+					onAbort();
+					return;
 				}
-				return this.deps.refreshLocalCapability({
-					peerHash: properties.peerHash,
-					target: context.target,
-					peerSession: properties.peerSession,
-					receiveEpoch: properties.receiveEpoch,
-					signal: operationSignal,
-					requestRemoteFullRearm: true,
-				});
-			})
+				promise.then(
+					(value) => {
+						operationSignal.removeEventListener("abort", onAbort);
+						resolve(value);
+					},
+					(error) => {
+						operationSignal.removeEventListener("abort", onAbort);
+						reject(error);
+					},
+				);
+			});
+		operation = raceWithAttemptSignal(refresh)
 			.then(() => undefined)
 			.catch((error) => {
 				if (
@@ -735,13 +902,15 @@ export class ReplicationInfoV2ReceiveCoordinator {
 				}
 			})
 			.finally(() => {
-				operationSignal.removeEventListener("abort", detach);
 				detach();
 			});
-		rearm.inFlight = operation;
+		attemptState.inFlight = operation;
 		operationSignal.addEventListener("abort", detach, { once: true });
-		if (operationSignal.aborted) {
-			detach();
+		for (const source of sourceSignals) {
+			if (source.aborted) {
+				attemptController.abort(source.reason);
+				break;
+			}
 		}
 		void operation;
 		return true;