@peerbit/shared-log 16.0.25 → 16.0.27

package/src/index.ts

@@ -260,10 +260,14 @@ import type {
 	AddedReplicationInfoMutation,
 	FullReplicationInfoMutation,
 } from "./replication-info-mutation.js";
-import { ReplicationInfoV2ReceiveCoordinator } from "./replication-info-v2-receive.js";
+import {
+	ReplicationInfoV2ReceiveCoordinator,
+	type ReplicationInfoV2ReceiveDiagnostic,
+} from "./replication-info-v2-receive.js";
 import {
 	type ReplicationInfoV2ConfirmationOptions,
 	ReplicationInfoV2SendCoordinator,
+	type ReplicationInfoV2SendDiagnostic,
 } from "./replication-info-v2-send.js";
 import {
 	type ReplicationStatus,
@@ -1860,6 +1864,39 @@ export type PersistedReceiptPeerReadinessUnsupportedReason =
 	| "persisted-receipts-unsupported"
 	| "replication-confirmation-unsupported";
 
+/**
+ * Bounded, payload-free diagnostics for one persisted-receipt readiness
+ * inspection. Session and transport identities remain opaque; revision and
+ * sequence values are decimal strings so the snapshot is JSON-safe. `reason`
+ * fields classify the current state; rejected-packet and disconnected-peer
+ * history is deliberately not retained.
+ */
+export type PersistedReceiptPeerReadinessDiagnostic = Readonly<{
+	version: 1;
+	log: "open" | "closed";
+	session: Readonly<{
+		state: "absent" | "current";
+		phase?: "opening" | "open" | "departing" | "superseded";
+		established: boolean;
+		suspended: boolean;
+		openingBarrierActive: boolean;
+		replicationInfoBlocked: boolean;
+		receiveCleanupGateOpen: boolean;
+	}>;
+	capability: Readonly<{
+		state: "absent" | "observed";
+		persistedReceipts: boolean;
+		replicationConfirmation: boolean;
+	}>;
+	sender: ReplicationInfoV2SendDiagnostic;
+	receiver: ReplicationInfoV2ReceiveDiagnostic;
+}>;
+
+type PersistedReceiptPeerReadinessDiagnosticAttachment = Readonly<{
+	/** Present only when `diagnostics: true` was requested. */
+	diagnostic?: PersistedReceiptPeerReadinessDiagnostic;
+}>;
+
 /**
  * Detached view of one public key's current persisted-receipt generation.
  * `generation` is opaque: callers may compare it for equality, but must not
@@ -1868,20 +1905,23 @@ export type PersistedReceiptPeerReadinessUnsupportedReason =
  * leadership and outbound confirmation can still change within a generation.
  */
 export type PersistedReceiptPeerReadiness =
-	| Readonly<{
+	| (Readonly<{
 			status: "ready";
 			generation: string;
-	  }>
-	| Readonly<{
+	  }> &
+			PersistedReceiptPeerReadinessDiagnosticAttachment)
+	| (Readonly<{
 			status: "pending";
 			reason: PersistedReceiptPeerReadinessPendingReason;
 			generation?: string;
-	  }>
-	| Readonly<{
+	  }> &
+			PersistedReceiptPeerReadinessDiagnosticAttachment)
+	| (Readonly<{
 			status: "unsupported";
 			reason: PersistedReceiptPeerReadinessUnsupportedReason;
 			generation: string;
-	  }>;
+	  }> &
+			PersistedReceiptPeerReadinessDiagnosticAttachment);
 
 export type PersistedReceiptPeerReady = Extract<
 	PersistedReceiptPeerReadiness,
@@ -1892,6 +1932,8 @@ export type PersistedReceiptPeerReadinessOptions<
 	T,
 	R extends "u32" | "u64",
 > = Readonly<{
+	/** Include a bounded, JSON-safe diagnostic snapshot in the result. */
+	diagnostics?: boolean;
 	/** Require this peer to be a freshly planned leader for every entry. */
 	entries?: readonly (ShallowOrFullEntry<T> | EntryReplicated<R>)[];
 	/**
@@ -1952,6 +1994,7 @@ const MAX_PERSISTED_RECEIPT_ATTEMPT_MS = 2_000;
 const MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL = 8;
 const MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER = 2;
 const MAX_PERSISTED_RECEIPT_READINESS_WAITERS = 1_024;
+const MAX_PERSISTED_RECEIPT_READINESS_DIAGNOSTIC_SUMMARY_CHARS = 768;
 const PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY = 16;
 const PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY = 8_192;
 const PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND = 8;
@@ -4087,10 +4130,15 @@ export class SharedLog<
 						: 0),
 			}),
 			{
-				mode: new AcknowledgeDelivery({
-					redundancy: 1,
-					to: [properties.target],
-				}),
+				// The transient rearm is only a bounded recovery hint. The ensuing
+				// authenticated Full/Applied exchange is the readiness proof, so do not
+				// let an ACK-delivery promise park the recovery scheduler itself.
+				mode: properties.requestRemoteFullRearm
+					? new SilentDelivery({ redundancy: 1, to: [properties.target] })
+					: new AcknowledgeDelivery({
+							redundancy: 1,
+							to: [properties.target],
+						}),
 				priority: CONVERGENCE_MESSAGE_PRIORITY,
 				signal: properties.signal,
 			},
@@ -4442,8 +4490,7 @@ export class SharedLog<
 
 		const profile = this._logProperties?.sync?.profile;
 		const startedAt = syncProfileStart(profile);
-		const mode =
-			resolvedRoot === fanoutService.publicKeyHash ? "root" : "node";
+		const mode = resolvedRoot === fanoutService.publicKeyHash ? "root" : "node";
 		const before = profile
 			? snapshotFanoutOpenMetrics(fanoutService, this.topic, resolvedRoot)
 			: undefined;
@@ -5765,6 +5812,117 @@ export class SharedLog<
 		});
 	}
 
+	private persistedReceiptReadinessDiagnostic(
+		peerHash: string,
+	): PersistedReceiptPeerReadinessDiagnostic {
+		const peerSession = this._peerSessions.current(peerHash);
+		const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
+		const capabilitySession = this._peerSyncCapabilitySessions.get(peerHash);
+		const capabilities = this._peerSyncCapabilities.get(peerHash) ?? 0;
+		const replicationInfoBlocked =
+			this._peerSessions.isReplicationInfoBlocked(peerHash);
+		const receiveCleanupGateOpen =
+			this._peerSessions.isReceiveCleanupGateOpen(peerHash);
+		const active = peerSession?.isActive() ?? false;
+		const established =
+			!this.closed &&
+			peerSession?.phase === "open" &&
+			active &&
+			!replicationInfoBlocked &&
+			receiveCleanupGateOpen;
+		const session = Object.freeze({
+			state: peerSession ? ("current" as const) : ("absent" as const),
+			...(peerSession ? { phase: peerSession.phase } : {}),
+			established,
+			suspended: !established,
+			openingBarrierActive: peerSession?.openingBarrierActive ?? false,
+			replicationInfoBlocked,
+			receiveCleanupGateOpen,
+		});
+		const capabilityObserved =
+			capabilitySession !== undefined &&
+			this._peerSyncCapabilityTimestamps.has(peerHash);
+		const capability = Object.freeze({
+			state: capabilityObserved ? ("observed" as const) : ("absent" as const),
+			persistedReceipts:
+				(capabilities & SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS) !== 0,
+			replicationConfirmation:
+				(capabilities & SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM) !== 0,
+		});
+		return Object.freeze({
+			version: 1 as const,
+			log: this.closed ? ("closed" as const) : ("open" as const),
+			session,
+			capability,
+			sender: this._v2Send.diagnosePeer({
+				peerHash,
+				...(peerSession ? { peerSession } : {}),
+				...(capabilitySession === undefined
+					? {}
+					: { receiverTransportSession: capabilitySession }),
+			}),
+			receiver: this._v2Receive.diagnosePeer({
+				peerHash,
+				...(peerSession ? { peerSession } : {}),
+				receiveEpoch,
+				...(capabilitySession === undefined
+					? {}
+					: { senderTransportSession: capabilitySession }),
+			}),
+		});
+	}
+
+	private attachPersistedReceiptReadinessDiagnostic<
+		Snapshot extends PersistedReceiptPeerReadiness,
+	>(
+		peerHash: string,
+		snapshot: Snapshot,
+		enabled: boolean | undefined,
+	): Snapshot {
+		if (enabled !== true) return snapshot;
+		return Object.freeze({
+			...snapshot,
+			diagnostic: this.persistedReceiptReadinessDiagnostic(peerHash),
+		}) as Snapshot;
+	}
+
+	private formatPersistedReceiptReadinessDiagnostic(peerHash: string): string {
+		const diagnostic = this.persistedReceiptReadinessDiagnostic(peerHash);
+		const { session, capability, sender, receiver } = diagnostic;
+		const summary = [
+			`session=${session.state}/${session.phase ?? "none"}/${
+				session.established ? "established" : "suspended"
+			}`,
+			`capability=${capability.state}/persisted:${
+				capability.persistedReceipts ? 1 : 0
+			}/confirm:${capability.replicationConfirmation ? 1 : 0}`,
+			`sender=${sender.state}/${sender.reason}/revision:${
+				sender.currentRevision
+			}/pending:${sender.pendingRevision ?? "none"}/inflight:${
+				sender.inFlightSequence ?? "none"
+			}@${sender.inFlightRevision ?? "none"}/applied:${
+				sender.appliedRevision ?? "none"
+			}/confirmation:${sender.confirmationSequence ?? "none"}@${
+				sender.confirmationRevision ?? "none"
+			}/waiters:${sender.confirmationWaiters}/oldest-ms:${
+				sender.oldestConfirmationAgeMs ?? "none"
+			}/retries:${sender.retryAttempts ?? 0}`,
+			`receiver=${receiver.state}/${receiver.reason}/phase:${
+				receiver.phase ?? "none"
+			}/request:${receiver.requestState ?? "none"}/applied-sequence:${
+				receiver.lastAppliedSequence ?? "none"
+			}/retries:${receiver.requestAttempts ?? 0}/rearm:${
+				receiver.remoteFullRearm
+			}/rearm-attempts:${receiver.remoteFullRearmAttempts}/rearm-outstanding:${
+				receiver.remoteFullRearmOutstanding
+			}/rearm-global:${receiver.remoteFullRearmGlobalOutstanding}`,
+		].join(" ");
+		return summary.slice(
+			0,
+			MAX_PERSISTED_RECEIPT_READINESS_DIAGNOSTIC_SUMMARY_CHARS,
+		);
+	}
+
 	private dispatchPersistedReceiptReadinessChange(peerHash: string): void {
 		this.events.dispatchEvent(
 			new CustomEvent<PersistedReceiptPeerReadinessEvent>(
@@ -5856,9 +6014,7 @@ export class SharedLog<
 		};
 	}
 
-	private persistedReceiptPeerSession(
-		peerHash: string,
-	):
+	private persistedReceiptPeerSession(peerHash: string):
 		| {
 				capabilitySession: bigint;
 				peerSession: PeerSession;
@@ -24381,6 +24537,11 @@ export class SharedLog<
 	private nudgePersistedReceiptPeerReadiness(
 		publicKey: PublicSignKey,
 		signal: AbortSignal,
+		rearmCurrentUnconfirmed?: {
+			peerSession: PeerSession;
+			receiveEpoch: object | null;
+			capabilitySession: bigint;
+		},
 	): void {
 		if (this.closed || signal.aborted) return;
 		const peerHash = publicKey.hashcode();
@@ -24388,8 +24549,7 @@ export class SharedLog<
 		if (
 			!peerSession ||
 			peerSession.phase === "departing" ||
-			(peerSession.phase === "opening" &&
-				!peerSession.openingBarrierActive)
+			(peerSession.phase === "opening" && !peerSession.openingBarrierActive)
 		) {
 			// A barrier rejection deliberately leaves the current session in its
 			// fail-closed opening phase after the barrier window has settled. Ask the
@@ -24416,11 +24576,20 @@ export class SharedLog<
 		const receiptTarget = this.persistedReceiptPeerSession(peerHash);
 		if (
 			receiptTarget &&
-			!this._v2Send.hasCurrentStateForPeer({
+			(!this._v2Send.hasCurrentStateForPeer({
 				peerHash,
 				peerSession: receiptTarget.peerSession,
 				receiverTransportSession: receiptTarget.capabilitySession,
-			})
+			}) ||
+				(rearmCurrentUnconfirmed?.peerSession === receiptTarget.peerSession &&
+					rearmCurrentUnconfirmed.receiveEpoch === receiptTarget.receiveEpoch &&
+					rearmCurrentUnconfirmed.capabilitySession ===
+						receiptTarget.capabilitySession &&
+					!this._v2Send.isLatestConfirmedForPeer({
+						peerHash,
+						peerSession: receiptTarget.peerSession,
+						receiverTransportSession: receiptTarget.capabilitySession,
+					})))
 		) {
 			this._v2Receive.reAdvertiseLocalCapabilityForRemoteFull({
 				peerHash,
@@ -24446,7 +24615,18 @@ export class SharedLog<
 		key: PublicSignKey,
 		options: PersistedReceiptPeerReadinessOptions<T, R> = {},
 	): Promise<PersistedReceiptPeerReadiness> {
-		return this.inspectPersistedReceiptPeerReadiness(key, options);
+		if (options.diagnostics !== true) {
+			return this.inspectPersistedReceiptPeerReadiness(key, options);
+		}
+		const snapshot = await this.inspectPersistedReceiptPeerReadiness(
+			key,
+			options,
+		);
+		return this.attachPersistedReceiptReadinessDiagnostic(
+			key.hashcode(),
+			snapshot,
+			options.diagnostics,
+		);
 	}
 
 	private async inspectPersistedReceiptPeerReadiness(
@@ -24563,7 +24743,9 @@ export class SharedLog<
 	 * Wait for a public key's current (or replacement) connection generation to
 	 * become persisted-receipt ready. Transition listeners are installed before
 	 * the first asynchronous inspection, and a bounded recovery tick repairs
-	 * missed subscriber/capability wakes without retaining stale PeerSessions.
+	 * missed subscriber/capability wakes without retaining stale PeerSessions. A
+	 * current sender gets one normal confirmation interval before an exact-
+	 * generation watchdog requests a bounded Full rearm.
 	 * This waiter is advisory only; the following persisted delivery remains the
 	 * operation that proves the requested remote durability quorum.
 	 */
@@ -24621,6 +24803,14 @@ export class SharedLog<
 		let rerun = false;
 		let recoveryTimer: ReturnType<typeof setTimeout> | undefined;
 		let confirmationController: AbortController | undefined;
+		let confirmationRecoveryTarget:
+			| {
+					peerSession: PeerSession;
+					receiveEpoch: object | null;
+					capabilitySession: bigint;
+					notBefore: number;
+			  }
+			| undefined;
 		let lastSnapshot: PersistedReceiptPeerReadiness | undefined;
 		const createTimeoutError = () => {
 			const suffix = lastSnapshot
@@ -24628,8 +24818,10 @@ export class SharedLog<
 						"reason" in lastSnapshot ? `/${lastSnapshot.reason}` : ""
 					})`
 				: "";
+			const diagnostic =
+				this.formatPersistedReceiptReadinessDiagnostic(peerHash);
 			return new TimeoutError(
-				`Timeout waiting for persisted-receipt readiness from ${peerHash}${suffix}`,
+				`Timeout waiting for persisted-receipt readiness from ${peerHash}${suffix}; diagnostic: ${diagnostic}`,
 			);
 		};
 
@@ -24651,6 +24843,7 @@ export class SharedLog<
 				new AbortError("Persisted-receipt readiness generation changed"),
 			);
 			confirmationController = undefined;
+			confirmationRecoveryTarget = undefined;
 			operationController.abort(
 				new AbortError("Persisted-receipt readiness wait settled"),
 			);
@@ -24706,7 +24899,14 @@ export class SharedLog<
 			recoveryTimer = setTimeout(() => {
 				recoveryTimer = undefined;
 				if (!continueWait()) return;
-				this.nudgePersistedReceiptPeerReadiness(key, operationSignal);
+				const recoveryTarget = confirmationRecoveryTarget;
+				this.nudgePersistedReceiptPeerReadiness(
+					key,
+					operationSignal,
+					recoveryTarget && Date.now() >= recoveryTarget.notBefore
+						? recoveryTarget
+						: undefined,
+				);
 				if (checkInFlight) {
 					// Confirmation can legitimately span several recovery intervals.
 					// Keep repairing the exact current generation without aborting its
@@ -24719,6 +24919,34 @@ export class SharedLog<
 			}, recoveryDelayMs);
 			recoveryTimer.unref?.();
 		};
+		const prepareConfirmationRecovery = () => {
+			const target = this.persistedReceiptPeerSession(peerHash);
+			if (!target) {
+				confirmationRecoveryTarget = undefined;
+				return undefined;
+			}
+			const retained = confirmationRecoveryTarget;
+			if (
+				retained?.peerSession !== target.peerSession ||
+				retained.receiveEpoch !== target.receiveEpoch ||
+				retained.capabilitySession !== target.capabilitySession
+			) {
+				confirmationRecoveryTarget = {
+					peerSession: target.peerSession,
+					receiveEpoch: target.receiveEpoch,
+					capabilitySession: target.capabilitySession,
+					notBefore: Date.now() + recoveryDelayMs,
+				};
+			}
+			const recoveryTarget = confirmationRecoveryTarget!;
+			this.nudgePersistedReceiptPeerReadiness(
+				key,
+				operationSignal,
+				Date.now() >= recoveryTarget.notBefore ? recoveryTarget : undefined,
+			);
+			armRecoveryTick();
+			return target;
+		};
 		const runCheck = async () => {
 			checkScheduled = false;
 			if (!continueWait()) return;
@@ -24735,27 +24963,36 @@ export class SharedLog<
 				);
 				lastSnapshot = snapshot;
 				if (!continueWait()) return;
-				if (rerun) return;
 				if (snapshot.status === "ready") {
+					confirmationRecoveryTarget = undefined;
+					if (rerun) return;
 					// A wake observed while the asynchronous inspection was running may
 					// already have invalidated this snapshot. Drain that coalesced wake
 					// before publishing readiness.
-					resolve(snapshot);
+					resolve(
+						this.attachPersistedReceiptReadinessDiagnostic(
+							peerHash,
+							snapshot,
+							options.diagnostics,
+						),
+					);
 					return;
 				}
 				if (
 					snapshot.status === "pending" &&
 					snapshot.reason === "replication-confirmation-pending"
 				) {
-					const target = this.persistedReceiptPeerSession(peerHash);
+					// Retain one watchdog deadline for the exact generation across
+					// coalesced inspection wakes. A sub-interval event stream must drive,
+					// rather than indefinitely postpone, the mature recovery nudge.
+					const target = prepareConfirmationRecovery();
+					if (rerun) return;
 					if (target) {
-						// A confirmation waiter cannot create an outbound V2 stream when
-						// that exact session state is missing. Nudge first, then keep the
-						// recovery tick alive while the single confirmation wait remains.
-						this.nudgePersistedReceiptPeerReadiness(key, operationSignal);
-						armRecoveryTick();
 						const currentConfirmationController = new AbortController();
 						confirmationController = currentConfirmationController;
+						const currentConfirmationRecoveryTarget =
+							confirmationRecoveryTarget;
+						let confirmationSucceeded = false;
 						try {
 							await this._v2Send.confirmLatestForPeer(
 								{
@@ -24771,6 +25008,7 @@ export class SharedLog<
 									]),
 								},
 							);
+							confirmationSucceeded = true;
 						} catch (error) {
 							if (!continueWait()) return;
 							if (!(error instanceof AbortError)) {
@@ -24781,6 +25019,12 @@ export class SharedLog<
 							if (confirmationController === currentConfirmationController) {
 								confirmationController = undefined;
 							}
+							if (
+								confirmationSucceeded &&
+								confirmationRecoveryTarget === currentConfirmationRecoveryTarget
+							) {
+								confirmationRecoveryTarget = undefined;
+							}
 						}
 						if (!continueWait()) return;
 						snapshot = await this.inspectPersistedReceiptPeerReadiness(
@@ -24790,13 +25034,32 @@ export class SharedLog<
 						);
 						lastSnapshot = snapshot;
 						if (!continueWait()) return;
-						if (rerun) return;
 						if (snapshot.status === "ready") {
-							resolve(snapshot);
+							confirmationRecoveryTarget = undefined;
+							if (rerun) return;
+							resolve(
+								this.attachPersistedReceiptReadinessDiagnostic(
+									peerHash,
+									snapshot,
+									options.diagnostics,
+								),
+							);
 							return;
 						}
+						if (
+							snapshot.status === "pending" &&
+							snapshot.reason === "replication-confirmation-pending"
+						) {
+							prepareConfirmationRecovery();
+						} else {
+							confirmationRecoveryTarget = undefined;
+						}
+						if (rerun) return;
 					}
+				} else {
+					confirmationRecoveryTarget = undefined;
 				}
+				if (rerun) return;
 				if (!continueWait()) return;
 				this.nudgePersistedReceiptPeerReadiness(key, operationSignal);
 			} catch (error) {