@peerbit/shared-log 16.0.26 → 16.0.28

package/src/index.ts

@@ -260,10 +260,14 @@ import type {
 	AddedReplicationInfoMutation,
 	FullReplicationInfoMutation,
 } from "./replication-info-mutation.js";
-import { ReplicationInfoV2ReceiveCoordinator } from "./replication-info-v2-receive.js";
+import {
+	ReplicationInfoV2ReceiveCoordinator,
+	type ReplicationInfoV2ReceiveDiagnostic,
+} from "./replication-info-v2-receive.js";
 import {
 	type ReplicationInfoV2ConfirmationOptions,
 	ReplicationInfoV2SendCoordinator,
+	type ReplicationInfoV2SendDiagnostic,
 } from "./replication-info-v2-send.js";
 import {
 	type ReplicationStatus,
@@ -1860,6 +1864,39 @@ export type PersistedReceiptPeerReadinessUnsupportedReason =
 	| "persisted-receipts-unsupported"
 	| "replication-confirmation-unsupported";
 
+/**
+ * Bounded, payload-free diagnostics for one persisted-receipt readiness
+ * inspection. Session and transport identities remain opaque; revision and
+ * sequence values are decimal strings so the snapshot is JSON-safe. `reason`
+ * fields classify the current state; rejected-packet and disconnected-peer
+ * history is deliberately not retained.
+ */
+export type PersistedReceiptPeerReadinessDiagnostic = Readonly<{
+	version: 1;
+	log: "open" | "closed";
+	session: Readonly<{
+		state: "absent" | "current";
+		phase?: "opening" | "open" | "departing" | "superseded";
+		established: boolean;
+		suspended: boolean;
+		openingBarrierActive: boolean;
+		replicationInfoBlocked: boolean;
+		receiveCleanupGateOpen: boolean;
+	}>;
+	capability: Readonly<{
+		state: "absent" | "observed";
+		persistedReceipts: boolean;
+		replicationConfirmation: boolean;
+	}>;
+	sender: ReplicationInfoV2SendDiagnostic;
+	receiver: ReplicationInfoV2ReceiveDiagnostic;
+}>;
+
+type PersistedReceiptPeerReadinessDiagnosticAttachment = Readonly<{
+	/** Present only when `diagnostics: true` was requested. */
+	diagnostic?: PersistedReceiptPeerReadinessDiagnostic;
+}>;
+
 /**
  * Detached view of one public key's current persisted-receipt generation.
  * `generation` is opaque: callers may compare it for equality, but must not
@@ -1868,20 +1905,23 @@ export type PersistedReceiptPeerReadinessUnsupportedReason =
  * leadership and outbound confirmation can still change within a generation.
  */
 export type PersistedReceiptPeerReadiness =
-	| Readonly<{
+	| (Readonly<{
 			status: "ready";
 			generation: string;
-	  }>
-	| Readonly<{
+	  }> &
+			PersistedReceiptPeerReadinessDiagnosticAttachment)
+	| (Readonly<{
 			status: "pending";
 			reason: PersistedReceiptPeerReadinessPendingReason;
 			generation?: string;
-	  }>
-	| Readonly<{
+	  }> &
+			PersistedReceiptPeerReadinessDiagnosticAttachment)
+	| (Readonly<{
 			status: "unsupported";
 			reason: PersistedReceiptPeerReadinessUnsupportedReason;
 			generation: string;
-	  }>;
+	  }> &
+			PersistedReceiptPeerReadinessDiagnosticAttachment);
 
 export type PersistedReceiptPeerReady = Extract<
 	PersistedReceiptPeerReadiness,
@@ -1892,6 +1932,8 @@ export type PersistedReceiptPeerReadinessOptions<
 	T,
 	R extends "u32" | "u64",
 > = Readonly<{
+	/** Include a bounded, JSON-safe diagnostic snapshot in the result. */
+	diagnostics?: boolean;
 	/** Require this peer to be a freshly planned leader for every entry. */
 	entries?: readonly (ShallowOrFullEntry<T> | EntryReplicated<R>)[];
 	/**
@@ -1952,6 +1994,7 @@ const MAX_PERSISTED_RECEIPT_ATTEMPT_MS = 2_000;
 const MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL = 8;
 const MAX_PERSISTED_RECEIPT_REQUESTS_PER_PEER = 2;
 const MAX_PERSISTED_RECEIPT_READINESS_WAITERS = 1_024;
+const MAX_PERSISTED_RECEIPT_READINESS_DIAGNOSTIC_SUMMARY_CHARS = 768;
 const PERSISTED_RECEIPT_INGRESS_PEER_REQUEST_CAPACITY = 16;
 const PERSISTED_RECEIPT_INGRESS_PEER_HASH_CAPACITY = 8_192;
 const PERSISTED_RECEIPT_INGRESS_PEER_REQUESTS_PER_SECOND = 8;
@@ -2680,6 +2723,10 @@ export class SharedLog<
 	// The epoch tokens are PeerSession objects (src/peer-session.ts); the map
 	// itself lives on the registry. Stage-2 of the fence refactor.
 	private _peerSessions!: PeerSessionRegistry;
+	// An exact successor session gets at most one unchanged-Full repair. Weak
+	// ownership keeps retired transport generations collectible without an
+	// identity-keyed cleanup path.
+	private _successorFullRepairSessions!: WeakSet<PeerSession>;
 	// A superseded removal may be the queue item that actually observed an active
 	// replicator. Carry that leave obligation to the transition that ultimately
 	// wins, while a winning reconnect clears it without emitting a stale leave.
@@ -4264,6 +4311,7 @@ export class SharedLog<
 		// replication-info blocked set (fence B5) alongside the session maps —
 		// the legacy inline `new Set()` that sat above moved there.
 		this._peerSessions = this.createPeerSessionRegistry();
+		this._successorFullRepairSessions = new WeakSet();
 		this._pendingReplicatorLeaveByPeer = new Set();
 		this._activeReceiveHandlersByPeer = new Map();
 		this._receiveHandlerDrainByPeer = new Map();
@@ -4447,8 +4495,7 @@ export class SharedLog<
 
 		const profile = this._logProperties?.sync?.profile;
 		const startedAt = syncProfileStart(profile);
-		const mode =
-			resolvedRoot === fanoutService.publicKeyHash ? "root" : "node";
+		const mode = resolvedRoot === fanoutService.publicKeyHash ? "root" : "node";
 		const before = profile
 			? snapshotFanoutOpenMetrics(fanoutService, this.topic, resolvedRoot)
 			: undefined;
@@ -5770,6 +5817,117 @@ export class SharedLog<
 		});
 	}
 
+	private persistedReceiptReadinessDiagnostic(
+		peerHash: string,
+	): PersistedReceiptPeerReadinessDiagnostic {
+		const peerSession = this._peerSessions.current(peerHash);
+		const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
+		const capabilitySession = this._peerSyncCapabilitySessions.get(peerHash);
+		const capabilities = this._peerSyncCapabilities.get(peerHash) ?? 0;
+		const replicationInfoBlocked =
+			this._peerSessions.isReplicationInfoBlocked(peerHash);
+		const receiveCleanupGateOpen =
+			this._peerSessions.isReceiveCleanupGateOpen(peerHash);
+		const active = peerSession?.isActive() ?? false;
+		const established =
+			!this.closed &&
+			peerSession?.phase === "open" &&
+			active &&
+			!replicationInfoBlocked &&
+			receiveCleanupGateOpen;
+		const session = Object.freeze({
+			state: peerSession ? ("current" as const) : ("absent" as const),
+			...(peerSession ? { phase: peerSession.phase } : {}),
+			established,
+			suspended: !established,
+			openingBarrierActive: peerSession?.openingBarrierActive ?? false,
+			replicationInfoBlocked,
+			receiveCleanupGateOpen,
+		});
+		const capabilityObserved =
+			capabilitySession !== undefined &&
+			this._peerSyncCapabilityTimestamps.has(peerHash);
+		const capability = Object.freeze({
+			state: capabilityObserved ? ("observed" as const) : ("absent" as const),
+			persistedReceipts:
+				(capabilities & SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS) !== 0,
+			replicationConfirmation:
+				(capabilities & SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM) !== 0,
+		});
+		return Object.freeze({
+			version: 1 as const,
+			log: this.closed ? ("closed" as const) : ("open" as const),
+			session,
+			capability,
+			sender: this._v2Send.diagnosePeer({
+				peerHash,
+				...(peerSession ? { peerSession } : {}),
+				...(capabilitySession === undefined
+					? {}
+					: { receiverTransportSession: capabilitySession }),
+			}),
+			receiver: this._v2Receive.diagnosePeer({
+				peerHash,
+				...(peerSession ? { peerSession } : {}),
+				receiveEpoch,
+				...(capabilitySession === undefined
+					? {}
+					: { senderTransportSession: capabilitySession }),
+			}),
+		});
+	}
+
+	private attachPersistedReceiptReadinessDiagnostic<
+		Snapshot extends PersistedReceiptPeerReadiness,
+	>(
+		peerHash: string,
+		snapshot: Snapshot,
+		enabled: boolean | undefined,
+	): Snapshot {
+		if (enabled !== true) return snapshot;
+		return Object.freeze({
+			...snapshot,
+			diagnostic: this.persistedReceiptReadinessDiagnostic(peerHash),
+		}) as Snapshot;
+	}
+
+	private formatPersistedReceiptReadinessDiagnostic(peerHash: string): string {
+		const diagnostic = this.persistedReceiptReadinessDiagnostic(peerHash);
+		const { session, capability, sender, receiver } = diagnostic;
+		const summary = [
+			`session=${session.state}/${session.phase ?? "none"}/${
+				session.established ? "established" : "suspended"
+			}`,
+			`capability=${capability.state}/persisted:${
+				capability.persistedReceipts ? 1 : 0
+			}/confirm:${capability.replicationConfirmation ? 1 : 0}`,
+			`sender=${sender.state}/${sender.reason}/revision:${
+				sender.currentRevision
+			}/pending:${sender.pendingRevision ?? "none"}/inflight:${
+				sender.inFlightSequence ?? "none"
+			}@${sender.inFlightRevision ?? "none"}/applied:${
+				sender.appliedRevision ?? "none"
+			}/confirmation:${sender.confirmationSequence ?? "none"}@${
+				sender.confirmationRevision ?? "none"
+			}/waiters:${sender.confirmationWaiters}/oldest-ms:${
+				sender.oldestConfirmationAgeMs ?? "none"
+			}/retries:${sender.retryAttempts ?? 0}`,
+			`receiver=${receiver.state}/${receiver.reason}/phase:${
+				receiver.phase ?? "none"
+			}/request:${receiver.requestState ?? "none"}/applied-sequence:${
+				receiver.lastAppliedSequence ?? "none"
+			}/retries:${receiver.requestAttempts ?? 0}/rearm:${
+				receiver.remoteFullRearm
+			}/rearm-attempts:${receiver.remoteFullRearmAttempts}/rearm-outstanding:${
+				receiver.remoteFullRearmOutstanding
+			}/rearm-global:${receiver.remoteFullRearmGlobalOutstanding}`,
+		].join(" ");
+		return summary.slice(
+			0,
+			MAX_PERSISTED_RECEIPT_READINESS_DIAGNOSTIC_SUMMARY_CHARS,
+		);
+	}
+
 	private dispatchPersistedReceiptReadinessChange(peerHash: string): void {
 		this.events.dispatchEvent(
 			new CustomEvent<PersistedReceiptPeerReadinessEvent>(
@@ -5861,9 +6019,7 @@ export class SharedLog<
 		};
 	}
 
-	private persistedReceiptPeerSession(
-		peerHash: string,
-	):
+	private persistedReceiptPeerSession(peerHash: string):
 		| {
 				capabilitySession: bigint;
 				peerSession: PeerSession;
@@ -17808,6 +17964,7 @@ export class SharedLog<
 			})) > 0;
 
 		this._gidPeersHistory = new Map();
+		this._successorFullRepairSessions = new WeakSet();
 		this.resetGidPeerHistoryCleanupState();
 		const replicationChangeOwnershipLifecycleController =
 			this.captureReplicationOwnershipLifecycle();
@@ -20607,6 +20764,7 @@ export class SharedLog<
 			this._receiveHandlerDrainByPeer?.clear();
 			this._openingSyncCapabilitiesByPeer?.clear();
 			this._gidPeersHistory?.clear();
+			this._successorFullRepairSessions = new WeakSet();
 			this._peerSyncCapabilities?.clear();
 			this._peerSyncCapabilitySessions?.clear();
 			this._peerSyncCapabilityTimestamps?.clear();
@@ -23875,6 +24033,10 @@ export class SharedLog<
 		) {
 			return;
 		}
+		const isFirstSuccessorFull =
+			msg instanceof FullReplicationInfoV2Message &&
+			receiveSession.hasPredecessor &&
+			!this._successorFullRepairSessions.has(receiveSession);
 		const admission = this._v2Receive.reserve(msg, {
 			from,
 			peerSession: receiveSession,
@@ -23908,6 +24070,8 @@ export class SharedLog<
 				const exactGate = () =>
 					hostGate() && this._v2Receive.isAdmissionCurrent(admission);
 				let durableCommitted = false;
+				let committedSuccessorFull = false;
+				let repairUnchangedSuccessorFull = false;
 
 				try {
 					if (
@@ -23951,6 +24115,12 @@ export class SharedLog<
 						) {
 							return;
 						}
+						committedSuccessorFull =
+							isFirstSuccessorFull &&
+							msg instanceof FullReplicationInfoV2Message &&
+							msg.segments.length > 0;
+						repairUnchangedSuccessorFull =
+							committedSuccessorFull && result.length === 0;
 					} else {
 						if (!exactGate()) {
 							return;
@@ -24024,6 +24194,30 @@ export class SharedLog<
 				if (!hostGate()) {
 					return;
 				}
+				if (committedSuccessorFull) {
+					if (!exactGate()) {
+						return;
+					}
+					this._successorFullRepairSessions.add(receiveSession);
+					if (repairUnchangedSuccessorFull) {
+						// The unchanged range rows belong to the public-key principal, but
+						// delivery evidence may belong to the superseded process. Reuse the
+						// receipt-driven authoritative repair path and bypass only those stale
+						// hints; do not synthesize public membership or range-change events.
+						this._repairFrontierBypassKnownPeersByMode
+							.get("join-authoritative")
+							?.add(fromHash);
+						const peers = new Set([fromHash]);
+						this.scheduleRepairSweep(
+							{ mode: "join-authoritative", peers },
+							lane.ownershipLifecycleController,
+						);
+						this.scheduleJoinAuthoritativeRepair(
+							peers,
+							lane.ownershipLifecycleController,
+						);
+					}
+				}
 				this._liveness.markReplicatorActivity(fromHash);
 				// A committed V2 announcement is applied progress: the peer answers,
 				// so recovery re-solicitation may restart from the base interval.
@@ -24398,8 +24592,7 @@ export class SharedLog<
 		if (
 			!peerSession ||
 			peerSession.phase === "departing" ||
-			(peerSession.phase === "opening" &&
-				!peerSession.openingBarrierActive)
+			(peerSession.phase === "opening" && !peerSession.openingBarrierActive)
 		) {
 			// A barrier rejection deliberately leaves the current session in its
 			// fail-closed opening phase after the barrier window has settled. Ask the
@@ -24465,7 +24658,18 @@ export class SharedLog<
 		key: PublicSignKey,
 		options: PersistedReceiptPeerReadinessOptions<T, R> = {},
 	): Promise<PersistedReceiptPeerReadiness> {
-		return this.inspectPersistedReceiptPeerReadiness(key, options);
+		if (options.diagnostics !== true) {
+			return this.inspectPersistedReceiptPeerReadiness(key, options);
+		}
+		const snapshot = await this.inspectPersistedReceiptPeerReadiness(
+			key,
+			options,
+		);
+		return this.attachPersistedReceiptReadinessDiagnostic(
+			key.hashcode(),
+			snapshot,
+			options.diagnostics,
+		);
 	}
 
 	private async inspectPersistedReceiptPeerReadiness(
@@ -24657,8 +24861,10 @@ export class SharedLog<
 						"reason" in lastSnapshot ? `/${lastSnapshot.reason}` : ""
 					})`
 				: "";
+			const diagnostic =
+				this.formatPersistedReceiptReadinessDiagnostic(peerHash);
 			return new TimeoutError(
-				`Timeout waiting for persisted-receipt readiness from ${peerHash}${suffix}`,
+				`Timeout waiting for persisted-receipt readiness from ${peerHash}${suffix}; diagnostic: ${diagnostic}`,
 			);
 		};
 
@@ -24806,7 +25012,13 @@ export class SharedLog<
 					// A wake observed while the asynchronous inspection was running may
 					// already have invalidated this snapshot. Drain that coalesced wake
 					// before publishing readiness.
-					resolve(snapshot);
+					resolve(
+						this.attachPersistedReceiptReadinessDiagnostic(
+							peerHash,
+							snapshot,
+							options.diagnostics,
+						),
+					);
 					return;
 				}
 				if (
@@ -24868,7 +25080,13 @@ export class SharedLog<
 						if (snapshot.status === "ready") {
 							confirmationRecoveryTarget = undefined;
 							if (rerun) return;
-							resolve(snapshot);
+							resolve(
+								this.attachPersistedReceiptReadinessDiagnostic(
+									peerHash,
+									snapshot,
+									options.diagnostics,
+								),
+							);
 							return;
 						}
 						if (
@@ -24966,6 +25184,11 @@ export class SharedLog<
 			timeout?: number;
 		},
 	) {
+		if (this.closed) {
+			throw new ClosedError();
+		}
+		const closeSignal = this._closeController.signal;
+		if (closeSignal.aborted) throw new ClosedError();
 		if (options?.signal?.aborted) {
 			throw new AbortError();
 		}
@@ -24974,6 +25197,9 @@ export class SharedLog<
 		const resolvedRoleAge = options?.eager
 			? undefined
 			: (options?.roleAge ?? (await this.getDefaultMinRoleAge()));
+		if (this.closed || closeSignal.aborted) {
+			throw new ClosedError();
+		}
 		if (options?.signal?.aborted) {
 			throw new AbortError();
 		}
@@ -24989,6 +25215,7 @@ export class SharedLog<
 			this.events.removeEventListener("replicator:mature", runCheck);
 			this.events.removeEventListener("replication:change", runCheck);
 			options?.signal?.removeEventListener("abort", onAbort);
+			closeSignal.removeEventListener("abort", onClose);
 			if (timer != null) {
 				clearTimeout(timer);
 				timer = undefined;
@@ -25028,9 +25255,11 @@ export class SharedLog<
 		};
 
 		const onAbort = () => reject(new AbortError());
+		const onClose = () => reject(new ClosedError());
 		if (options?.signal) {
-			options.signal.addEventListener("abort", onAbort);
+			options.signal.addEventListener("abort", onAbort, { once: true });
 		}
+		closeSignal.addEventListener("abort", onClose, { once: true });
 
 		timer = setTimeout(() => {
 			reject(
@@ -25073,31 +25302,32 @@ export class SharedLog<
 				return;
 			}
 
-			if (requestAttempts >= maxRequestAttempts) {
-				return;
-			}
-
-			requestAttempts++;
+			if (requestAttempts < maxRequestAttempts) {
+				requestAttempts++;
 
-			const peerHash = key.hashcode();
-			const peerSession = this._peerSessions.current(peerHash);
-			if (peerSession?.phase === "open") {
-				this._v2Receive.ensureRequestProgress({
-					peerHash,
-					peerSession,
-					receiveEpoch: this._peerSessions.receiveEpoch(peerHash),
-				});
-			} else if (peerSession === null || peerSession.phase === "departing") {
-				// A peer can be known to routing before its SharedLog topic
-				// subscription has been observed. Legacy requests used to bootstrap
-				// that case directly; V2 needs an authoritative Subscribe snapshot
-				// before it can create a fenced PeerSession and request a Full.
-				requestSubscriberSnapshot();
+				const peerHash = key.hashcode();
+				const peerSession = this._peerSessions.current(peerHash);
+				if (peerSession?.phase === "open") {
+					this._v2Receive.ensureRequestProgress({
+						peerHash,
+						peerSession,
+						receiveEpoch: this._peerSessions.receiveEpoch(peerHash),
+					});
+				} else if (peerSession === null || peerSession.phase === "departing") {
+					// A peer can be known to routing before its SharedLog topic
+					// subscription has been observed. Legacy requests used to bootstrap
+					// that case directly; V2 needs an authoritative Subscribe snapshot
+					// before it can create a fenced PeerSession and request a Full.
+					requestSubscriberSnapshot();
+				}
 			}
 
-			if (requestAttempts < maxRequestAttempts) {
-				requestTimer = setTimeout(requestReplicationInfo, requestIntervalMs);
-			}
+			// Replication change events are wake hints, not replayable state. Keep
+			// checking the authoritative index after the bounded network recovery
+			// attempts are exhausted so a transiently empty transition read cannot
+			// strand an otherwise-ready waiter until its outer timeout.
+			runCheck();
+			requestTimer = setTimeout(requestReplicationInfo, requestIntervalMs);
 		};
 
 		const check = async () => {
@@ -25149,8 +25379,9 @@ export class SharedLog<
 		// replay a maturity/change event that fires while that read is in flight.
 		this.events.addEventListener("replicator:mature", runCheck);
 		this.events.addEventListener("replication:change", runCheck);
-		requestReplicationInfo();
-		runCheck();
+		if (closeSignal.aborted) onClose();
+		else if (options?.signal?.aborted) onAbort();
+		else requestReplicationInfo();
 
 		return deferred.promise.finally(clear);
 	}