@peerbit/shared-log 16.0.29 → 16.0.30

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/shared-log",
-  "version": "16.0.29",
+  "version": "16.0.30",
   "description": "Shared log",
   "sideEffects": false,
   "type": "module",
@@ -62,37 +62,37 @@
     "pidusage": "^4.0.1",
     "pino": "^9.4.0",
     "uint8arrays": "^5.1.0",
-    "@peerbit/blocks": "4.3.0",
-    "@peerbit/blocks-interface": "2.2.0",
     "@peerbit/any-store": "2.2.17",
-    "@peerbit/cache": "3.1.1",
+    "@peerbit/blocks": "4.3.1",
     "@peerbit/crypto": "3.1.6",
-    "@peerbit/log": "6.2.32",
+    "@peerbit/blocks-interface": "2.2.1",
+    "@peerbit/cache": "3.1.1",
     "@peerbit/indexer-sqlite3": "3.0.20",
-    "@peerbit/logger": "2.0.2",
-    "@peerbit/indexer-interface": "3.0.13",
-    "@peerbit/pubsub": "5.4.5",
-    "@peerbit/rpc": "6.1.27",
     "@peerbit/diagnostics": "0.0.1",
+    "@peerbit/log": "6.2.33",
+    "@peerbit/program": "6.0.60",
+    "@peerbit/pubsub": "5.4.6",
+    "@peerbit/pubsub-interface": "5.2.2",
+    "@peerbit/logger": "2.0.2",
     "@peerbit/riblt": "1.2.0",
+    "@peerbit/rpc": "6.1.28",
     "@peerbit/stream-interface": "6.0.16",
-    "@peerbit/pubsub-interface": "5.2.2",
-    "@peerbit/time": "3.0.1",
-    "@peerbit/program": "6.0.59"
+    "@peerbit/indexer-interface": "3.0.13",
+    "@peerbit/time": "3.0.1"
   },
   "optionalDependencies": {
-    "@peerbit/native-backbone": "0.2.14",
-    "@peerbit/shared-log-rust": "0.1.7"
+    "@peerbit/shared-log-rust": "0.1.7",
+    "@peerbit/native-backbone": "0.2.15"
   },
   "devDependencies": {
     "@types/libsodium-wrappers": "^0.7.14",
     "@types/pidusage": "^2.0.5",
     "uuid": "^11.1.1",
-    "@peerbit/any-store-rust": "0.1.7",
-    "@peerbit/indexer-simple": "1.2.17",
     "@peerbit/indexer-rust": "1.0.14",
-    "peerbit": "5.4.1",
-    "@peerbit/test-utils": "3.1.40"
+    "@peerbit/indexer-simple": "1.2.17",
+    "@peerbit/test-utils": "3.1.41",
+    "@peerbit/any-store-rust": "0.1.7",
+    "peerbit": "5.4.2"
   },
   "repository": {
     "type": "git",

package/src/index.ts

@@ -564,7 +564,7 @@ const emitAdvisorySyncProfileDuration = (
 	try {
 		emitSyncProfileDuration(profile, startedAt, event);
 	} catch {
-		// Diagnostics must not change open or provider-resolution correctness.
+		// Advisory diagnostics must not change replication or lifecycle behavior.
 	}
 };
 
@@ -4033,7 +4033,13 @@ export class SharedLog<
 				this._repairMetrics["join-warmup"].simpleFallbackPasses += 1;
 			},
 			sendEntriesSimple: (target, entries, options) =>
-				this.sendRepairEntriesWithTransport(target, entries, "simple", options),
+				this.sendRepairEntriesWithTransport(
+					target,
+					entries,
+					"simple",
+					options,
+					"join-warmup",
+				),
 			logError: (error) => logger.error(error),
 		});
 	}
@@ -10639,6 +10645,7 @@ export class SharedLog<
 			isStillCurrent?: () => boolean;
 			signal?: AbortSignal;
 		},
+		mode?: RepairDispatchMode,
 	) {
 		const isStillCurrent = options?.isStillCurrent ?? (() => true);
 		if (!isStillCurrent()) {
@@ -10646,53 +10653,87 @@ export class SharedLog<
 		}
 		const unknownEntries = new Map<string, RepairDispatchEntry<R>>();
 		const knownHashes: string[] = [];
-		for (const [hash, entry] of entries) {
-			if (
-				(options?.bypassRecentKnownPeers ||
-					!this.isEntryRecentlyKnownByPeer(
-						hash,
-						target,
-						RECENT_KNOWN_REPAIR_SUPPRESSION_MS,
-					)) &&
-				(options?.bypassKnownPeers || !this.isEntryKnownByPeer(hash, target))
-			) {
-				unknownEntries.set(hash, entry);
-			} else {
-				knownHashes.push(hash);
+		const profile = this._logProperties?.sync?.profile;
+		const startedAt = syncProfileStart(profile);
+		const inputEntries = profile ? entries.size : 0;
+		let selectedEntries = 0;
+		let lastObservedCurrent = true;
+		let outcome = "stale";
+		try {
+			for (const [hash, entry] of entries) {
+				if (
+					(options?.bypassRecentKnownPeers ||
+						!this.isEntryRecentlyKnownByPeer(
+							hash,
+							target,
+							RECENT_KNOWN_REPAIR_SUPPRESSION_MS,
+						)) &&
+					(options?.bypassKnownPeers || !this.isEntryKnownByPeer(hash, target))
+				) {
+					unknownEntries.set(hash, entry);
+				} else {
+					knownHashes.push(hash);
+				}
 			}
-		}
-		if (!isStillCurrent()) {
-			return;
-		}
-		this.clearRepairFrontierHashes(target, knownHashes);
-		if (unknownEntries.size === 0) {
-			return;
-		}
-		if (transport === "simple") {
-			// Fallback repair should not depend on the target completing the
-			// RequestMaybeSync -> ResponseMaybeSync round trip.
-			await this.pushRepairEntries(
-				target,
-				unknownEntries,
-				isStillCurrent,
-				options?.signal,
-			);
-			return;
-		}
-
-		const syncEntries = this._logProperties?.sync?.priority
-			? (this._coordinates.materializeRepairDispatchEntries(
+			// A custom synchronizer may mutate the Map once it receives it.
+			if (profile) selectedEntries = unknownEntries.size;
+			if (!isStillCurrent()) return;
+			this.clearRepairFrontierHashes(target, knownHashes);
+			if (unknownEntries.size === 0) {
+				outcome = "known-suppressed";
+				return;
+			}
+			if (transport === "simple") {
+				// Observe only checks the lower path already makes, without adding
+				// lifecycle decisions or wrapping the disabled-profiling path.
+				const dispatchIsStillCurrent = profile
+					? () => (lastObservedCurrent = isStillCurrent())
+					: isStillCurrent;
+				// Fallback repair does not wait for the maybe-sync round trip.
+				await this.pushRepairEntries(
+					target,
 					unknownEntries,
-				) as unknown as Map<string, SyncEntryCoordinates<R>>)
-			: (unknownEntries as Map<string, SyncEntryCoordinates<R>>);
-		if (!isStillCurrent()) {
-			return;
+					dispatchIsStillCurrent,
+					options?.signal,
+				);
+			} else {
+				const syncEntries = this._logProperties?.sync?.priority
+					? (this._coordinates.materializeRepairDispatchEntries(
+							unknownEntries,
+						) as unknown as Map<string, SyncEntryCoordinates<R>>)
+					: (unknownEntries as Map<string, SyncEntryCoordinates<R>>);
+				if (!isStillCurrent()) return;
+				await this.syncronizer.onMaybeMissingEntries({
+					entries: syncEntries,
+					targets: [target],
+					signal: options?.signal,
+				});
+			}
+			outcome = !lastObservedCurrent
+				? "stale"
+				: options?.signal?.aborted
+					? "cancelled"
+					: "dispatched";
+		} catch (error) {
+			outcome = "error";
+			throw error;
+		} finally {
+			if (profile) {
+				emitAdvisorySyncProfileDuration(profile, startedAt, {
+					name: "sharedLog.repair.dispatch",
+					component: "shared-log",
+					entries: inputEntries,
+					count: selectedEntries,
+					targets: 1,
+					details: {
+						mode,
+						transport,
+						outcome,
+						knownSuppressedEntries: knownHashes.length,
+					},
+				});
+			}
 		}
-		await this.syncronizer.onMaybeMissingEntries({
-			entries: syncEntries,
-			targets: [target],
-			signal: options?.signal,
-		});
 	}
 
 	private async sendMaybeMissingEntriesNow(
@@ -10776,6 +10817,7 @@ export class SharedLog<
 						this.isRepairLifecycleActive(repairLifecycleController),
 					signal: repairLifecycleController.signal,
 				},
+				options.mode,
 			),
 		).catch((error: any) => logger.error(error));
 	}
@@ -11166,6 +11208,7 @@ export class SharedLog<
 							this.isRepairLifecycleActive(repairLifecycleController),
 						signal: repairLifecycleController.signal,
 					},
+					options.mode,
 				),
 			).catch((error: any) => logger.error(error));
 		};
@@ -11309,6 +11352,18 @@ export class SharedLog<
 		repairLifecycleController: AbortController = this._instanceLifecycle
 			?.ownershipLifecycleController as AbortController,
 	) {
+		const profile = this._logProperties?.sync?.profile;
+		const startedAt = syncProfileStart(profile);
+		const profileCounts = profile
+			? {
+					passes: 0,
+					inputEntries: 0,
+					nativePasses: 0,
+					repairCandidates: 0,
+					repairBatches: 0,
+					outcome: "stale",
+				}
+			: undefined;
 		try {
 			while (this.isRepairLifecycleActive(repairLifecycleController)) {
 				if (!this.isRepairLifecycleActive(repairLifecycleController)) {
@@ -11347,8 +11402,10 @@ export class SharedLog<
 				pruneStaleJoinWarmupPeers();
 
 				if (pendingModes.size === 0) {
+					if (profileCounts) profileCounts.outcome = "completed";
 					return;
 				}
+				if (profileCounts) profileCounts.passes += 1;
 
 				const optimisticGidPeersByMode = new Map<
 					RepairDispatchMode,
@@ -11447,6 +11504,10 @@ export class SharedLog<
 						}
 						return;
 					}
+					if (profileCounts) {
+						profileCounts.repairCandidates += entries.size;
+						profileCounts.repairBatches += 1;
+					}
 					this.dispatchMaybeMissingEntries(
 						target,
 						entries,
@@ -11513,6 +11574,10 @@ export class SharedLog<
 					residentEntriesByHash &&
 					!this.hasCustomFindLeaders()
 				) {
+					if (profileCounts) {
+						profileCounts.nativePasses += 1;
+						profileCounts.inputEntries += residentEntriesByHash.size;
+					}
 					const repairDispatchPlan = pruneStaleJoinWarmupPeers()
 						? await this.planResidentRepairDispatchBatch(
 								{
@@ -11551,6 +11616,7 @@ export class SharedLog<
 							const entries = await iterator.next(
 								REPAIR_SWEEP_ENTRY_BATCH_SIZE,
 							);
+							if (profileCounts) profileCounts.inputEntries += entries.length;
 							if (!this.isRepairLifecycleActive(repairLifecycleController)) {
 								return;
 							}
@@ -11675,6 +11741,7 @@ export class SharedLog<
 				}
 			}
 		} catch (error: any) {
+			if (profileCounts) profileCounts.outcome = "error";
 			if (
 				this.isRepairLifecycleActive(repairLifecycleController) &&
 				!isNotStartedError(error)
@@ -11696,6 +11763,21 @@ export class SharedLog<
 					void this.runRepairSweep(repairLifecycleController);
 				}
 			}
+			if (profileCounts) {
+				emitAdvisorySyncProfileDuration(profile, startedAt, {
+					name: "sharedLog.placement.pass",
+					component: "shared-log",
+					entries: profileCounts.inputEntries,
+					count: profileCounts.repairCandidates,
+					details: {
+						phase: "repair-sweep",
+						outcome: profileCounts.outcome,
+						passes: profileCounts.passes,
+						nativePasses: profileCounts.nativePasses,
+						repairBatches: profileCounts.repairBatches,
+					},
+				});
+			}
 		}
 	}
 
@@ -21274,10 +21356,11 @@ export class SharedLog<
 			msg.heads.map((head) => head.hash),
 		);
 		if (syncProfile) {
-			emitSyncProfileDuration(syncProfile, rawExistingStartedAt, {
+			emitAdvisorySyncProfileDuration(syncProfile, rawExistingStartedAt, {
 				name: "sharedLog.rawReceive.existingHeads",
 				component: "shared-log",
 				entries: msg.heads.length,
+				count: rawExistingHashes.size,
 				messages: 1,
 			});
 		}
@@ -21711,10 +21794,11 @@ export class SharedLog<
 						? undefined
 						: await this.log.hasMany(headHashes);
 					if (syncProfile) {
-						emitSyncProfileDuration(syncProfile, existingStartedAt, {
+						emitAdvisorySyncProfileDuration(syncProfile, existingStartedAt, {
 							name: "sharedLog.receive.existingHeads",
 							component: "shared-log",
 							entries: heads.length,
+							count: existingHashes?.size,
 							messages: 1,
 							details: { rawMaterializedKnownMissing },
 						});
@@ -29478,6 +29562,17 @@ export class SharedLog<
 			isOwnershipLifecycleCurrent() &&
 			[...warmupPeers].every(isCurrentJoinWarmupTarget);
 
+		const profile = this._logProperties?.sync?.profile;
+		const profileStartedAt = syncProfileStart(profile);
+		const profileCounts = profile
+			? {
+					examinedEntries: 0,
+					repairCandidates: 0,
+					repairBatches: 0,
+					pruneScan: false,
+					outcome: "stale",
+				}
+			: undefined;
 		try {
 			const uncheckedDeliver: Map<
 				string,
@@ -29501,6 +29596,10 @@ export class SharedLog<
 					: isWarmupTarget
 						? "join-warmup"
 						: "join-authoritative";
+				if (profileCounts) {
+					profileCounts.repairCandidates += entries.size;
+					profileCounts.repairBatches += 1;
+				}
 				this.dispatchMaybeMissingEntries(
 					target,
 					entries,
@@ -29555,6 +29654,7 @@ export class SharedLog<
 						forceFresh: forceFreshDelivery || useJoinWarmupFastPath,
 					},
 				)) {
+					if (profileCounts) profileCounts.examinedEntries += 1;
 					if (
 						!isOwnershipLifecycleCurrent() ||
 						(useJoinWarmupFastPath && !areJoinWarmupGenerationsCurrent())
@@ -29838,6 +29938,7 @@ export class SharedLog<
 					));
 
 			if (shouldRunLocalPruneScan) {
+				if (profileCounts) profileCounts.pruneScan = true;
 				throwIfOwnershipLifecycleInactive();
 				// Adaptive range changes and fixed zero-width updates can make already-indexed
 				// local heads prunable even when the incremental rebalance scan misses them
@@ -29857,6 +29958,7 @@ export class SharedLog<
 				}
 			}
 
+			if (profileCounts) profileCounts.outcome = "completed";
 			return changed;
 		} catch (error: any) {
 			if (!isOwnershipLifecycleCurrent()) {
@@ -29866,8 +29968,27 @@ export class SharedLog<
 				return false; // we are not started yet, so no changes
 			}
 
+			if (profileCounts) profileCounts.outcome = "error";
 			logger.error(error.toString());
 			throw error;
+		} finally {
+			if (profileCounts) {
+				emitAdvisorySyncProfileDuration(profile, profileStartedAt, {
+					name: "sharedLog.placement.pass",
+					component: "shared-log",
+					entries: profileCounts.examinedEntries,
+					count: profileCounts.repairCandidates,
+					details: {
+						phase: "range-change",
+						outcome: profileCounts.outcome,
+						changes: changes.length,
+						repairBatches: profileCounts.repairBatches,
+						pruneScan: profileCounts.pruneScan,
+						forceFreshDelivery,
+						joinWarmupFastPath: useJoinWarmupFastPath,
+					},
+				});
+			}
 		}
 	}
 
@@ -29926,6 +30047,15 @@ export class SharedLog<
 		ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
 		rebalanceParticipationDebounced = this.rebalanceParticipationDebounced,
 	) {
+		const profile = this._isAdaptiveReplicating
+			? this._logProperties?.sync?.profile
+			: undefined;
+		const profileStartedAt = syncProfileStart(profile);
+		const profileDetails:
+			| Record<string, string | number | boolean | undefined>
+			| undefined = profile
+			? { outcome: "stale", idleRemainingMs: 0 }
+			: undefined;
 		// Stage 3: the lifecycle owns all three identity terms. `lifecycle` may
 		// go stale later; its deps late-bind to the host, so the debouncer term
 		// still reads the current host field, and the role term can disagree
@@ -29965,6 +30095,14 @@ export class SharedLog<
 
 			if (this._isAdaptiveReplicating) {
 				if (this.shouldDelayAdaptiveRebalance()) {
+					if (profileDetails) {
+						profileDetails.outcome = "idle-deferred";
+						profileDetails.idleRemainingMs = Math.max(
+							0,
+							this.adaptiveRebalanceIdleMs -
+								(Date.now() - this._lastLocalAppendAt),
+						);
+					}
 					if (isCurrent()) {
 						void rebalanceParticipationDebounced?.call();
 					}
@@ -29974,11 +30112,17 @@ export class SharedLog<
 				const peers = this.replicationIndex;
 				const usedMemory = await this.getMemoryUsage();
 				if (!isCurrent()) return false;
+				if (profileDetails) {
+					profileDetails.storageUsedBytes = usedMemory;
+					profileDetails.storageObjectiveBytes =
+						this.replicationController.maxMemoryLimit;
+				}
 				this.scheduleReplicationStatusRefreshForStorage(usedMemory);
 				let dynamicRange = await this.getDynamicRange();
 				if (!isCurrent()) return false;
 
 				if (!dynamicRange) {
+					if (profileDetails) profileDetails.outcome = "not-permitted";
 					return; // not allowed to replicate
 				}
 
@@ -30005,13 +30149,24 @@ export class SharedLog<
 				const totalParticipation = await this.calculateTotalParticipation();
 				if (!isCurrent()) return false;
 
+				const cpuUsage = this.cpuUsage?.value();
+				const stepStartedAt = syncProfileStart(profile);
 				const newFactor = this.replicationController.step({
 					memoryUsage: usedMemory,
 					currentFactor: dynamicRange.widthNormalized,
 					totalFactor: totalParticipation, // TODO use this._totalParticipation when flakiness is fixed
 					peerCount: peersSize,
-					cpuUsage: this.cpuUsage?.value(),
+					cpuUsage,
 				});
+				if (profileDetails) {
+					profileDetails.preStepMs = stepStartedAt - profileStartedAt;
+					profileDetails.stepMs = syncProfileStart(profile) - stepStartedAt;
+					profileDetails.currentFactor = dynamicRange.widthNormalized;
+					profileDetails.proposedFactor = newFactor;
+					profileDetails.totalFactor = totalParticipation;
+					profileDetails.controllerPeerCount = peersSize;
+					profileDetails.cpuUsage = cpuUsage;
+				}
 
 				const absoluteDifference = Math.abs(
 					dynamicRange.widthNormalized - newFactor,
@@ -30048,9 +30203,11 @@ export class SharedLog<
 						(await this._isTrustedReplicator(this.node.identity.publicKey));
 					if (!isCurrent()) return false;
 					if (!canReplicate) {
+						if (profileDetails) profileDetails.outcome = "not-permitted";
 						return false;
 					}
 
+					const applyStartedAt = syncProfileStart(profile);
 					await this.startAnnounceReplicating(
 						[dynamicRange],
 						{
@@ -30061,6 +30218,10 @@ export class SharedLog<
 						ownershipLifecycleController,
 					);
 					if (!isCurrent()) return false;
+					if (profileDetails) {
+						profileDetails.outcome = "apply-settled";
+						profileDetails.applyMs = syncProfileStart(profile) - applyStartedAt;
+					}
 
 					/* await this._updateRole(newRole, onRoleChange); */
 					if (isCurrent()) {
@@ -30069,6 +30230,7 @@ export class SharedLog<
 
 					return true;
 				} else {
+					if (profileDetails) profileDetails.outcome = "unchanged";
 					if (isCurrent()) {
 						void rebalanceParticipationDebounced?.call();
 					}
@@ -30078,14 +30240,24 @@ export class SharedLog<
 			return false;
 		};
 
-		const resp = await fn().catch((error: any) => {
-			if (isNotStartedError(error) || isClosedStoreRace(error)) {
-				return false;
+		try {
+			return await fn().catch((error: any) => {
+				if (isNotStartedError(error) || isClosedStoreRace(error)) {
+					if (profileDetails) profileDetails.outcome = "stale";
+					return false;
+				}
+				if (profileDetails) profileDetails.outcome = "error";
+				throw error;
+			});
+		} finally {
+			if (profileDetails) {
+				emitAdvisorySyncProfileDuration(profile, profileStartedAt, {
+					name: "sharedLog.adaptive.rebalance",
+					component: "shared-log",
+					details: profileDetails,
+				});
 			}
-			throw error;
-		});
-
-		return resp;
+		}
 	}
 
 	private getDynamicRangeOffset(): NumberFromType<R> {