@peerbit/shared-log 16.0.28 → 16.0.30

package/src/index.ts

@@ -564,7 +564,7 @@ const emitAdvisorySyncProfileDuration = (
 	try {
 		emitSyncProfileDuration(profile, startedAt, event);
 	} catch {
-		// Diagnostics must not change open or provider-resolution correctness.
+		// Advisory diagnostics must not change replication or lifecycle behavior.
 	}
 };
 
@@ -4033,7 +4033,13 @@ export class SharedLog<
 				this._repairMetrics["join-warmup"].simpleFallbackPasses += 1;
 			},
 			sendEntriesSimple: (target, entries, options) =>
-				this.sendRepairEntriesWithTransport(target, entries, "simple", options),
+				this.sendRepairEntriesWithTransport(
+					target,
+					entries,
+					"simple",
+					options,
+					"join-warmup",
+				),
 			logError: (error) => logger.error(error),
 		});
 	}
@@ -6524,6 +6530,24 @@ export class SharedLog<
 								);
 							};
 							try {
+								if (!isPeerRoundCurrent()) return;
+								if (
+									!this._v2Send.hasCurrentStateForPeer({
+										peerHash: peer,
+										peerSession: captured.peerSession,
+										receiverTransportSession: captured.capabilitySession,
+									})
+								) {
+									// Preflight can precede replacement or loss of the sender
+									// stream. Delivery must recover its own exact current
+									// binding before waiting for application confirmation.
+									this._v2Receive.reAdvertiseLocalCapabilityForRemoteFull({
+										peerHash: peer,
+										peerSession: captured.peerSession,
+										receiveEpoch: captured.receiveEpoch,
+										signal: roundSignal,
+									});
+								}
 								await this._v2Send.confirmLatestForPeer(
 									{
 										peerHash: peer,
@@ -10621,6 +10645,7 @@ export class SharedLog<
 			isStillCurrent?: () => boolean;
 			signal?: AbortSignal;
 		},
+		mode?: RepairDispatchMode,
 	) {
 		const isStillCurrent = options?.isStillCurrent ?? (() => true);
 		if (!isStillCurrent()) {
@@ -10628,53 +10653,87 @@ export class SharedLog<
 		}
 		const unknownEntries = new Map<string, RepairDispatchEntry<R>>();
 		const knownHashes: string[] = [];
-		for (const [hash, entry] of entries) {
-			if (
-				(options?.bypassRecentKnownPeers ||
-					!this.isEntryRecentlyKnownByPeer(
-						hash,
-						target,
-						RECENT_KNOWN_REPAIR_SUPPRESSION_MS,
-					)) &&
-				(options?.bypassKnownPeers || !this.isEntryKnownByPeer(hash, target))
-			) {
-				unknownEntries.set(hash, entry);
-			} else {
-				knownHashes.push(hash);
+		const profile = this._logProperties?.sync?.profile;
+		const startedAt = syncProfileStart(profile);
+		const inputEntries = profile ? entries.size : 0;
+		let selectedEntries = 0;
+		let lastObservedCurrent = true;
+		let outcome = "stale";
+		try {
+			for (const [hash, entry] of entries) {
+				if (
+					(options?.bypassRecentKnownPeers ||
+						!this.isEntryRecentlyKnownByPeer(
+							hash,
+							target,
+							RECENT_KNOWN_REPAIR_SUPPRESSION_MS,
+						)) &&
+					(options?.bypassKnownPeers || !this.isEntryKnownByPeer(hash, target))
+				) {
+					unknownEntries.set(hash, entry);
+				} else {
+					knownHashes.push(hash);
+				}
 			}
-		}
-		if (!isStillCurrent()) {
-			return;
-		}
-		this.clearRepairFrontierHashes(target, knownHashes);
-		if (unknownEntries.size === 0) {
-			return;
-		}
-		if (transport === "simple") {
-			// Fallback repair should not depend on the target completing the
-			// RequestMaybeSync -> ResponseMaybeSync round trip.
-			await this.pushRepairEntries(
-				target,
-				unknownEntries,
-				isStillCurrent,
-				options?.signal,
-			);
-			return;
-		}
-
-		const syncEntries = this._logProperties?.sync?.priority
-			? (this._coordinates.materializeRepairDispatchEntries(
+			// A custom synchronizer may mutate the Map once it receives it.
+			if (profile) selectedEntries = unknownEntries.size;
+			if (!isStillCurrent()) return;
+			this.clearRepairFrontierHashes(target, knownHashes);
+			if (unknownEntries.size === 0) {
+				outcome = "known-suppressed";
+				return;
+			}
+			if (transport === "simple") {
+				// Observe only checks the lower path already makes, without adding
+				// lifecycle decisions or wrapping the disabled-profiling path.
+				const dispatchIsStillCurrent = profile
+					? () => (lastObservedCurrent = isStillCurrent())
+					: isStillCurrent;
+				// Fallback repair does not wait for the maybe-sync round trip.
+				await this.pushRepairEntries(
+					target,
 					unknownEntries,
-				) as unknown as Map<string, SyncEntryCoordinates<R>>)
-			: (unknownEntries as Map<string, SyncEntryCoordinates<R>>);
-		if (!isStillCurrent()) {
-			return;
+					dispatchIsStillCurrent,
+					options?.signal,
+				);
+			} else {
+				const syncEntries = this._logProperties?.sync?.priority
+					? (this._coordinates.materializeRepairDispatchEntries(
+							unknownEntries,
+						) as unknown as Map<string, SyncEntryCoordinates<R>>)
+					: (unknownEntries as Map<string, SyncEntryCoordinates<R>>);
+				if (!isStillCurrent()) return;
+				await this.syncronizer.onMaybeMissingEntries({
+					entries: syncEntries,
+					targets: [target],
+					signal: options?.signal,
+				});
+			}
+			outcome = !lastObservedCurrent
+				? "stale"
+				: options?.signal?.aborted
+					? "cancelled"
+					: "dispatched";
+		} catch (error) {
+			outcome = "error";
+			throw error;
+		} finally {
+			if (profile) {
+				emitAdvisorySyncProfileDuration(profile, startedAt, {
+					name: "sharedLog.repair.dispatch",
+					component: "shared-log",
+					entries: inputEntries,
+					count: selectedEntries,
+					targets: 1,
+					details: {
+						mode,
+						transport,
+						outcome,
+						knownSuppressedEntries: knownHashes.length,
+					},
+				});
+			}
 		}
-		await this.syncronizer.onMaybeMissingEntries({
-			entries: syncEntries,
-			targets: [target],
-			signal: options?.signal,
-		});
 	}
 
 	private async sendMaybeMissingEntriesNow(
@@ -10758,6 +10817,7 @@ export class SharedLog<
 						this.isRepairLifecycleActive(repairLifecycleController),
 					signal: repairLifecycleController.signal,
 				},
+				options.mode,
 			),
 		).catch((error: any) => logger.error(error));
 	}
@@ -11148,6 +11208,7 @@ export class SharedLog<
 							this.isRepairLifecycleActive(repairLifecycleController),
 						signal: repairLifecycleController.signal,
 					},
+					options.mode,
 				),
 			).catch((error: any) => logger.error(error));
 		};
@@ -11291,6 +11352,18 @@ export class SharedLog<
 		repairLifecycleController: AbortController = this._instanceLifecycle
 			?.ownershipLifecycleController as AbortController,
 	) {
+		const profile = this._logProperties?.sync?.profile;
+		const startedAt = syncProfileStart(profile);
+		const profileCounts = profile
+			? {
+					passes: 0,
+					inputEntries: 0,
+					nativePasses: 0,
+					repairCandidates: 0,
+					repairBatches: 0,
+					outcome: "stale",
+				}
+			: undefined;
 		try {
 			while (this.isRepairLifecycleActive(repairLifecycleController)) {
 				if (!this.isRepairLifecycleActive(repairLifecycleController)) {
@@ -11329,8 +11402,10 @@ export class SharedLog<
 				pruneStaleJoinWarmupPeers();
 
 				if (pendingModes.size === 0) {
+					if (profileCounts) profileCounts.outcome = "completed";
 					return;
 				}
+				if (profileCounts) profileCounts.passes += 1;
 
 				const optimisticGidPeersByMode = new Map<
 					RepairDispatchMode,
@@ -11429,6 +11504,10 @@ export class SharedLog<
 						}
 						return;
 					}
+					if (profileCounts) {
+						profileCounts.repairCandidates += entries.size;
+						profileCounts.repairBatches += 1;
+					}
 					this.dispatchMaybeMissingEntries(
 						target,
 						entries,
@@ -11495,6 +11574,10 @@ export class SharedLog<
 					residentEntriesByHash &&
 					!this.hasCustomFindLeaders()
 				) {
+					if (profileCounts) {
+						profileCounts.nativePasses += 1;
+						profileCounts.inputEntries += residentEntriesByHash.size;
+					}
 					const repairDispatchPlan = pruneStaleJoinWarmupPeers()
 						? await this.planResidentRepairDispatchBatch(
 								{
@@ -11533,6 +11616,7 @@ export class SharedLog<
 							const entries = await iterator.next(
 								REPAIR_SWEEP_ENTRY_BATCH_SIZE,
 							);
+							if (profileCounts) profileCounts.inputEntries += entries.length;
 							if (!this.isRepairLifecycleActive(repairLifecycleController)) {
 								return;
 							}
@@ -11657,6 +11741,7 @@ export class SharedLog<
 				}
 			}
 		} catch (error: any) {
+			if (profileCounts) profileCounts.outcome = "error";
 			if (
 				this.isRepairLifecycleActive(repairLifecycleController) &&
 				!isNotStartedError(error)
@@ -11678,6 +11763,21 @@ export class SharedLog<
 					void this.runRepairSweep(repairLifecycleController);
 				}
 			}
+			if (profileCounts) {
+				emitAdvisorySyncProfileDuration(profile, startedAt, {
+					name: "sharedLog.placement.pass",
+					component: "shared-log",
+					entries: profileCounts.inputEntries,
+					count: profileCounts.repairCandidates,
+					details: {
+						phase: "repair-sweep",
+						outcome: profileCounts.outcome,
+						passes: profileCounts.passes,
+						nativePasses: profileCounts.nativePasses,
+						repairBatches: profileCounts.repairBatches,
+					},
+				});
+			}
 		}
 	}
 
@@ -21256,10 +21356,11 @@ export class SharedLog<
 			msg.heads.map((head) => head.hash),
 		);
 		if (syncProfile) {
-			emitSyncProfileDuration(syncProfile, rawExistingStartedAt, {
+			emitAdvisorySyncProfileDuration(syncProfile, rawExistingStartedAt, {
 				name: "sharedLog.rawReceive.existingHeads",
 				component: "shared-log",
 				entries: msg.heads.length,
+				count: rawExistingHashes.size,
 				messages: 1,
 			});
 		}
@@ -21693,10 +21794,11 @@ export class SharedLog<
 						? undefined
 						: await this.log.hasMany(headHashes);
 					if (syncProfile) {
-						emitSyncProfileDuration(syncProfile, existingStartedAt, {
+						emitAdvisorySyncProfileDuration(syncProfile, existingStartedAt, {
 							name: "sharedLog.receive.existingHeads",
 							component: "shared-log",
 							entries: heads.length,
+							count: existingHashes?.size,
 							messages: 1,
 							details: { rawMaterializedKnownMissing },
 						});
@@ -24943,6 +25045,27 @@ export class SharedLog<
 				recoveryTimer = undefined;
 				if (!continueWait()) return;
 				const recoveryTarget = confirmationRecoveryTarget;
+				if (confirmationController && recoveryTarget) {
+					if (
+						this._peerSessions.current(peerHash) !==
+							recoveryTarget.peerSession ||
+						this._peerSessions.receiveEpoch(peerHash) !==
+							recoveryTarget.receiveEpoch ||
+						this._peerSyncCapabilitySessions.get(peerHash) !==
+							recoveryTarget.capabilitySession ||
+						!this.uniqueReplicators.has(peerHash)
+					) {
+						// Events can be coalesced while application confirmation waits.
+						// Release the obsolete target on the recovery tick as well,
+						// including loss of replicator eligibility within one session.
+						// A temporary receive reservation keeps this same target alive;
+						// the final readiness inspection still checks every admission gate.
+						confirmationRecoveryTarget = undefined;
+						confirmationController.abort(
+							new AbortError("Persisted-receipt readiness target changed"),
+						);
+					}
+				}
 				this.nudgePersistedReceiptPeerReadiness(
 					key,
 					operationSignal,
@@ -29439,6 +29562,17 @@ export class SharedLog<
 			isOwnershipLifecycleCurrent() &&
 			[...warmupPeers].every(isCurrentJoinWarmupTarget);
 
+		const profile = this._logProperties?.sync?.profile;
+		const profileStartedAt = syncProfileStart(profile);
+		const profileCounts = profile
+			? {
+					examinedEntries: 0,
+					repairCandidates: 0,
+					repairBatches: 0,
+					pruneScan: false,
+					outcome: "stale",
+				}
+			: undefined;
 		try {
 			const uncheckedDeliver: Map<
 				string,
@@ -29462,6 +29596,10 @@ export class SharedLog<
 					: isWarmupTarget
 						? "join-warmup"
 						: "join-authoritative";
+				if (profileCounts) {
+					profileCounts.repairCandidates += entries.size;
+					profileCounts.repairBatches += 1;
+				}
 				this.dispatchMaybeMissingEntries(
 					target,
 					entries,
@@ -29516,6 +29654,7 @@ export class SharedLog<
 						forceFresh: forceFreshDelivery || useJoinWarmupFastPath,
 					},
 				)) {
+					if (profileCounts) profileCounts.examinedEntries += 1;
 					if (
 						!isOwnershipLifecycleCurrent() ||
 						(useJoinWarmupFastPath && !areJoinWarmupGenerationsCurrent())
@@ -29799,6 +29938,7 @@ export class SharedLog<
 					));
 
 			if (shouldRunLocalPruneScan) {
+				if (profileCounts) profileCounts.pruneScan = true;
 				throwIfOwnershipLifecycleInactive();
 				// Adaptive range changes and fixed zero-width updates can make already-indexed
 				// local heads prunable even when the incremental rebalance scan misses them
@@ -29818,6 +29958,7 @@ export class SharedLog<
 				}
 			}
 
+			if (profileCounts) profileCounts.outcome = "completed";
 			return changed;
 		} catch (error: any) {
 			if (!isOwnershipLifecycleCurrent()) {
@@ -29827,8 +29968,27 @@ export class SharedLog<
 				return false; // we are not started yet, so no changes
 			}
 
+			if (profileCounts) profileCounts.outcome = "error";
 			logger.error(error.toString());
 			throw error;
+		} finally {
+			if (profileCounts) {
+				emitAdvisorySyncProfileDuration(profile, profileStartedAt, {
+					name: "sharedLog.placement.pass",
+					component: "shared-log",
+					entries: profileCounts.examinedEntries,
+					count: profileCounts.repairCandidates,
+					details: {
+						phase: "range-change",
+						outcome: profileCounts.outcome,
+						changes: changes.length,
+						repairBatches: profileCounts.repairBatches,
+						pruneScan: profileCounts.pruneScan,
+						forceFreshDelivery,
+						joinWarmupFastPath: useJoinWarmupFastPath,
+					},
+				});
+			}
 		}
 	}
 
@@ -29887,6 +30047,15 @@ export class SharedLog<
 		ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
 		rebalanceParticipationDebounced = this.rebalanceParticipationDebounced,
 	) {
+		const profile = this._isAdaptiveReplicating
+			? this._logProperties?.sync?.profile
+			: undefined;
+		const profileStartedAt = syncProfileStart(profile);
+		const profileDetails:
+			| Record<string, string | number | boolean | undefined>
+			| undefined = profile
+			? { outcome: "stale", idleRemainingMs: 0 }
+			: undefined;
 		// Stage 3: the lifecycle owns all three identity terms. `lifecycle` may
 		// go stale later; its deps late-bind to the host, so the debouncer term
 		// still reads the current host field, and the role term can disagree
@@ -29926,6 +30095,14 @@ export class SharedLog<
 
 			if (this._isAdaptiveReplicating) {
 				if (this.shouldDelayAdaptiveRebalance()) {
+					if (profileDetails) {
+						profileDetails.outcome = "idle-deferred";
+						profileDetails.idleRemainingMs = Math.max(
+							0,
+							this.adaptiveRebalanceIdleMs -
+								(Date.now() - this._lastLocalAppendAt),
+						);
+					}
 					if (isCurrent()) {
 						void rebalanceParticipationDebounced?.call();
 					}
@@ -29935,11 +30112,17 @@ export class SharedLog<
 				const peers = this.replicationIndex;
 				const usedMemory = await this.getMemoryUsage();
 				if (!isCurrent()) return false;
+				if (profileDetails) {
+					profileDetails.storageUsedBytes = usedMemory;
+					profileDetails.storageObjectiveBytes =
+						this.replicationController.maxMemoryLimit;
+				}
 				this.scheduleReplicationStatusRefreshForStorage(usedMemory);
 				let dynamicRange = await this.getDynamicRange();
 				if (!isCurrent()) return false;
 
 				if (!dynamicRange) {
+					if (profileDetails) profileDetails.outcome = "not-permitted";
 					return; // not allowed to replicate
 				}
 
@@ -29966,13 +30149,24 @@ export class SharedLog<
 				const totalParticipation = await this.calculateTotalParticipation();
 				if (!isCurrent()) return false;
 
+				const cpuUsage = this.cpuUsage?.value();
+				const stepStartedAt = syncProfileStart(profile);
 				const newFactor = this.replicationController.step({
 					memoryUsage: usedMemory,
 					currentFactor: dynamicRange.widthNormalized,
 					totalFactor: totalParticipation, // TODO use this._totalParticipation when flakiness is fixed
 					peerCount: peersSize,
-					cpuUsage: this.cpuUsage?.value(),
+					cpuUsage,
 				});
+				if (profileDetails) {
+					profileDetails.preStepMs = stepStartedAt - profileStartedAt;
+					profileDetails.stepMs = syncProfileStart(profile) - stepStartedAt;
+					profileDetails.currentFactor = dynamicRange.widthNormalized;
+					profileDetails.proposedFactor = newFactor;
+					profileDetails.totalFactor = totalParticipation;
+					profileDetails.controllerPeerCount = peersSize;
+					profileDetails.cpuUsage = cpuUsage;
+				}
 
 				const absoluteDifference = Math.abs(
 					dynamicRange.widthNormalized - newFactor,
@@ -30009,9 +30203,11 @@ export class SharedLog<
 						(await this._isTrustedReplicator(this.node.identity.publicKey));
 					if (!isCurrent()) return false;
 					if (!canReplicate) {
+						if (profileDetails) profileDetails.outcome = "not-permitted";
 						return false;
 					}
 
+					const applyStartedAt = syncProfileStart(profile);
 					await this.startAnnounceReplicating(
 						[dynamicRange],
 						{
@@ -30022,6 +30218,10 @@ export class SharedLog<
 						ownershipLifecycleController,
 					);
 					if (!isCurrent()) return false;
+					if (profileDetails) {
+						profileDetails.outcome = "apply-settled";
+						profileDetails.applyMs = syncProfileStart(profile) - applyStartedAt;
+					}
 
 					/* await this._updateRole(newRole, onRoleChange); */
 					if (isCurrent()) {
@@ -30030,6 +30230,7 @@ export class SharedLog<
 
 					return true;
 				} else {
+					if (profileDetails) profileDetails.outcome = "unchanged";
 					if (isCurrent()) {
 						void rebalanceParticipationDebounced?.call();
 					}
@@ -30039,14 +30240,24 @@ export class SharedLog<
 			return false;
 		};
 
-		const resp = await fn().catch((error: any) => {
-			if (isNotStartedError(error) || isClosedStoreRace(error)) {
-				return false;
+		try {
+			return await fn().catch((error: any) => {
+				if (isNotStartedError(error) || isClosedStoreRace(error)) {
+					if (profileDetails) profileDetails.outcome = "stale";
+					return false;
+				}
+				if (profileDetails) profileDetails.outcome = "error";
+				throw error;
+			});
+		} finally {
+			if (profileDetails) {
+				emitAdvisorySyncProfileDuration(profile, profileStartedAt, {
+					name: "sharedLog.adaptive.rebalance",
+					component: "shared-log",
+					details: profileDetails,
+				});
 			}
-			throw error;
-		});
-
-		return resp;
+		}
 	}
 
 	private getDynamicRangeOffset(): NumberFromType<R> {

package/src/replication-info-v2-receive.ts

@@ -27,6 +27,7 @@ const DEFAULT_MAX_REQUEST_RETRY_MS = 30_000;
 const DEFAULT_REQUEST_MAX_ATTEMPTS = 7;
 const DEFAULT_REMOTE_FULL_REARM_ATTEMPT_TIMEOUT_MS = 2_000;
 const DEFAULT_REMOTE_FULL_REARM_COOLDOWN_MS = 5_000;
+const MAX_REMOTE_FULL_REARM_COOLDOWN_MS = 30_000;
 const MAX_REMOTE_FULL_REARM_OUTSTANDING_PER_SESSION = 2;
 const MAX_REMOTE_FULL_REARM_OUTSTANDING_GLOBAL = 64;
 const MAX_TIMER_MS = 2_147_483_647;
@@ -825,18 +826,28 @@ export class ReplicationInfoV2ReceiveCoordinator {
 		) {
 			return true;
 		}
+		// Repeated hints must leave time for a slow Full and its Applied response
+		// to settle. Reuse this exact generation's bounded attempt counter so a
+		// watchdog cannot continually replace the binding it is trying to confirm.
+		const nextAttemptAt =
+			now +
+			Math.min(
+				MAX_REMOTE_FULL_REARM_COOLDOWN_MS,
+				this.remoteFullRearmCooldownMs *
+					2 ** Math.min(rearm?.attemptsStarted ?? 0, MAX_BACKOFF_EXPONENT),
+			);
 		if (!rearm) {
 			rearm = {
 				peerHash: properties.peerHash,
 				receiveEpoch: properties.receiveEpoch,
 				receiverTransportSession,
-				nextAttemptAt: now + this.remoteFullRearmCooldownMs,
+				nextAttemptAt,
 				attemptsStarted: 0,
 				outstandingAttempts: 0,
 			};
 			this._remoteFullRearmBySession.set(properties.peerSession, rearm);
 		} else {
-			rearm.nextAttemptAt = now + this.remoteFullRearmCooldownMs;
+			rearm.nextAttemptAt = nextAttemptAt;
 		}
 		const attemptState = rearm;
 		const outstandingReservation = {};