@peerbit/shared-log 13.0.23 → 13.0.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/shared-log",
-  "version": "13.0.23",
+  "version": "13.0.24",
   "description": "Shared log",
   "sideEffects": false,
   "type": "module",
@@ -62,18 +62,18 @@
     "pino": "^9.4.0",
     "uint8arrays": "^5.1.0",
     "@peerbit/any-store": "2.2.9",
-    "@peerbit/blocks": "4.0.11",
-    "@peerbit/crypto": "3.1.1",
     "@peerbit/blocks-interface": "2.0.8",
     "@peerbit/cache": "3.0.0",
+    "@peerbit/blocks": "4.0.11",
+    "@peerbit/crypto": "3.1.1",
     "@peerbit/indexer-interface": "3.0.3",
     "@peerbit/indexer-sqlite3": "3.0.6",
     "@peerbit/log": "6.0.21",
     "@peerbit/logger": "2.0.1",
+    "@peerbit/program": "6.0.17",
     "@peerbit/pubsub": "5.1.6",
     "@peerbit/pubsub-interface": "5.1.1",
     "@peerbit/rpc": "6.0.21",
-    "@peerbit/program": "6.0.17",
     "@peerbit/stream-interface": "6.0.7",
     "@peerbit/time": "3.0.0",
     "@peerbit/riblt": "1.2.0"

package/src/index.ts

@@ -495,6 +495,7 @@ const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE = 0.01;
 const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT = 0.005;
 const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT = 0.001;
 const RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR = 1e-3;
+const TOPIC_SUBSCRIBERS_CACHE_TTL_MS = 250;
 const ADAPTIVE_REBALANCE_IDLE_INTERVAL_MULTIPLIER = 5;
 const ADAPTIVE_REBALANCE_MIN_IDLE_AFTER_LOCAL_APPEND_MS = 10_000;
 
@@ -752,6 +753,10 @@ export class SharedLog<
 	private _repairSweepRunning!: boolean;
 	private _repairSweepForceFreshPending!: boolean;
 	private _repairSweepAddedPeersPending!: Set<string>;
+	private _topicSubscribersCache!: Map<
+		string,
+		{ expiresAt: number; keys: PublicSignKey[] }
+	>;
 
 	// regular distribution checks
 	private distributeQueue?: PQueue;
@@ -1364,14 +1369,26 @@ export class SharedLog<
 	private async _getTopicSubscribers(
 		topic: string,
 	): Promise<PublicSignKey[] | undefined> {
+		const cached = this._topicSubscribersCache.get(topic);
+		if (cached && cached.expiresAt > Date.now()) {
+			return cached.keys.slice();
+		}
+
 		const maxPeers = 64;
+		const cache = (keys: PublicSignKey[]) => {
+			this._topicSubscribersCache.set(topic, {
+				expiresAt: Date.now() + TOPIC_SUBSCRIBERS_CACHE_TTL_MS,
+				keys,
+			});
+			return keys.slice();
+		};
 
 		// Prefer the bounded peer set we already know from the fanout overlay.
 		if (this._fanoutChannel && (topic === this.topic || topic === this.rpc.topic)) {
 			const hashes = this._fanoutChannel
 				.getPeerHashes({ includeSelf: false })
 				.slice(0, maxPeers);
-			if (hashes.length === 0) return [];
+			if (hashes.length === 0) return cache([]);
 
 			const keys = await Promise.all(
 				hashes.map((hash) => this._resolvePublicKeyFromHash(hash)),
@@ -1387,7 +1404,7 @@ export class SharedLog<
 				seen.add(hash);
 				uniqueKeys.push(key);
 			}
-			return uniqueKeys;
+			return cache(uniqueKeys);
 		}
 
 		const selfHash = this.node.identity.publicKey.hashcode();
@@ -1444,7 +1461,7 @@ export class SharedLog<
 			}
 		}
 
-		if (hashes.length === 0) return [];
+		if (hashes.length === 0) return cache([]);
 
 		const uniqueHashes: string[] = [];
 		const seen = new Set<string>();
@@ -1465,7 +1482,18 @@ export class SharedLog<
 			if (hash === selfHash) continue;
 			uniqueKeys.push(key);
 		}
-		return uniqueKeys;
+		return cache(uniqueKeys);
+	}
+
+	private invalidateTopicSubscribersCache(...topics: (string | undefined)[]) {
+		for (const topic of topics) {
+			if (!topic) continue;
+			this._topicSubscribersCache.delete(topic);
+		}
+	}
+
+	private invalidateSharedLogTopicSubscribersCache() {
+		this.invalidateTopicSubscribersCache(this.topic, this.rpc.topic);
 	}
 
 	// @deprecated
@@ -2934,6 +2962,7 @@ export class SharedLog<
 		this._repairSweepRunning = false;
 		this._repairSweepForceFreshPending = false;
 		this._repairSweepAddedPeersPending = new Set();
+		this._topicSubscribersCache = new Map();
 		this.coordinateToHash = new Cache<string>({ max: 1e6, ttl: 1e4 });
 		this.recentlyRebalanced = new Cache<string>({ max: 1e4, ttl: 1e5 });
 
@@ -3958,39 +3987,40 @@ export class SharedLog<
 		this.coordinateToHash.clear();
 		this.recentlyRebalanced.clear();
 		this.uniqueReplicators.clear();
-			this._closeController.abort();
+		this._topicSubscribersCache.clear();
+		this._closeController.abort();
 
-			clearInterval(this.interval);
-			this.stopReplicatorLivenessSweep();
+		clearInterval(this.interval);
+		this.stopReplicatorLivenessSweep();
 
-			this.node.services.pubsub.removeEventListener(
-				"subscribe",
-				this._onSubscriptionFn,
+		this.node.services.pubsub.removeEventListener(
+			"subscribe",
+			this._onSubscriptionFn,
 		);
 
 		this.node.services.pubsub.removeEventListener(
 			"unsubscribe",
 			this._onUnsubscriptionFn,
 		);
-			for (const timer of this._repairRetryTimers) {
-				clearTimeout(timer);
-			}
-			this._repairRetryTimers.clear();
-			this._recentRepairDispatch.clear();
-			this._repairSweepRunning = false;
-			this._repairSweepForceFreshPending = false;
-			this._repairSweepAddedPeersPending.clear();
+		for (const timer of this._repairRetryTimers) {
+			clearTimeout(timer);
+		}
+		this._repairRetryTimers.clear();
+		this._recentRepairDispatch.clear();
+		this._repairSweepRunning = false;
+		this._repairSweepForceFreshPending = false;
+		this._repairSweepAddedPeersPending.clear();
 
 		for (const [_k, v] of this._pendingDeletes) {
 			v.clear();
 			v.promise.resolve(); // TODO or reject?
 		}
-			for (const [_k, v] of this._pendingIHave) {
-				v.clear();
-			}
-			for (const [_k, v] of this._checkedPruneRetries) {
-				if (v.timer) clearTimeout(v.timer);
-			}
+		for (const [_k, v] of this._pendingIHave) {
+			v.clear();
+		}
+		for (const [_k, v] of this._checkedPruneRetries) {
+			if (v.timer) clearTimeout(v.timer);
+		}
 
 		await this.remoteBlocks.stop();
 		this._pendingDeletes.clear();
@@ -5390,6 +5420,7 @@ export class SharedLog<
 		entry: Entry<T> | EntryReplicated<R> | ShallowEntry,
 		options?: {
 			roleAge?: number;
+			candidates?: Iterable<string>;
 			onLeader?: (key: string) => void;
 			// persist even if not leader
 			persist?:
@@ -5433,6 +5464,7 @@ export class SharedLog<
 		},
 		options?: {
 			roleAge?: number;
+			candidates?: Iterable<string>;
 			onLeader?: (key: string) => void;
 			// persist even if not leader
 			persist?:
@@ -5458,6 +5490,7 @@ export class SharedLog<
 		cursors: NumberFromType<R>[],
 		options?: {
 			roleAge?: number;
+			candidates?: Iterable<string>;
 		},
 	): Promise<Map<string, { intersecting: boolean }>> {
 		const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge()); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
@@ -5467,44 +5500,48 @@ export class SharedLog<
 		// If it is still warming up (for example, only contains self), supplement with
 		// current subscribers until we have enough candidates for this decision.
 		let peerFilter: Set<string> | undefined = undefined;
-		const selfReplicating = await this.isReplicating();
-		if (this.uniqueReplicators.size > 0) {
-			peerFilter = new Set(this.uniqueReplicators);
-			if (selfReplicating) {
-				peerFilter.add(selfHash);
-			} else {
-				peerFilter.delete(selfHash);
-			}
+		if (options?.candidates) {
+			peerFilter = new Set(options.candidates);
+		} else {
+			const selfReplicating = await this.isReplicating();
+			if (this.uniqueReplicators.size > 0) {
+				peerFilter = new Set(this.uniqueReplicators);
+				if (selfReplicating) {
+					peerFilter.add(selfHash);
+				} else {
+					peerFilter.delete(selfHash);
+				}
 
-			try {
-				const subscribers = await this._getTopicSubscribers(this.topic);
-				if (subscribers && subscribers.length > 0) {
-					for (const subscriber of subscribers) {
-						peerFilter.add(subscriber.hashcode());
-					}
-					if (selfReplicating) {
-						peerFilter.add(selfHash);
-					} else {
-						peerFilter.delete(selfHash);
+				try {
+					const subscribers = await this._getTopicSubscribers(this.topic);
+					if (subscribers && subscribers.length > 0) {
+						for (const subscriber of subscribers) {
+							peerFilter.add(subscriber.hashcode());
+						}
+						if (selfReplicating) {
+							peerFilter.add(selfHash);
+						} else {
+							peerFilter.delete(selfHash);
+						}
 					}
+				} catch {
+					// Best-effort only; keep current peerFilter.
 				}
-			} catch {
-				// Best-effort only; keep current peerFilter.
-			}
-		} else {
-			try {
-				const subscribers =
-					(await this._getTopicSubscribers(this.topic)) ?? undefined;
-				if (subscribers && subscribers.length > 0) {
-					peerFilter = new Set(subscribers.map((key) => key.hashcode()));
-					if (selfReplicating) {
-						peerFilter.add(selfHash);
-					} else {
-						peerFilter.delete(selfHash);
+			} else {
+				try {
+					const subscribers =
+						(await this._getTopicSubscribers(this.topic)) ?? undefined;
+					if (subscribers && subscribers.length > 0) {
+						peerFilter = new Set(subscribers.map((key) => key.hashcode()));
+						if (selfReplicating) {
+							peerFilter.add(selfHash);
+						} else {
+							peerFilter.delete(selfHash);
+						}
 					}
+				} catch {
+					// Best-effort only; if pubsub isn't ready, do a full scan.
 				}
-			} catch {
-				// Best-effort only; if pubsub isn't ready, do a full scan.
 			}
 		}
 		return getSamples<R>(
@@ -6156,30 +6193,48 @@ export class SharedLog<
 			}
 		}
 
-		const changed = false;
-		const replacedPeers = new Set<string>();
-		for (const change of changes) {
-			if (change.type === "replaced" && change.range.hash !== selfHash) {
-				replacedPeers.add(change.range.hash);
-			}
-		}
-		const addedPeers = new Set<string>();
-		for (const change of changes) {
-			if (change.type === "added" || change.type === "replaced") {
-				const hash = change.range.hash;
-				if (hash !== selfHash) {
-					// Range updates can reassign entries to an existing peer shortly after it
-					// already received a subset. Avoid suppressing legitimate follow-up repair.
-					this._recentRepairDispatch.delete(hash);
+			const changed = false;
+			const addedPeers = new Set<string>();
+			const warmupPeers = new Set<string>();
+			const hasSelfWarmupChange = changes.some(
+				(change) =>
+					change.range.hash === selfHash &&
+					(change.type === "added" || change.type === "replaced"),
+			);
+			for (const change of changes) {
+				if (change.type === "added" || change.type === "replaced") {
+					const hash = change.range.hash;
+					if (hash !== selfHash) {
+						// Range updates can reassign entries to an existing peer shortly after it
+						// already received a subset. Avoid suppressing legitimate follow-up repair.
+						this._recentRepairDispatch.delete(hash);
+					}
 				}
-			}
-			if (change.type === "added") {
-				const hash = change.range.hash;
-				if (hash !== selfHash && !replacedPeers.has(hash)) {
-					addedPeers.add(hash);
+				if (change.type === "added") {
+					const hash = change.range.hash;
+					if (hash !== selfHash) {
+						addedPeers.add(hash);
+						warmupPeers.add(hash);
+					}
 				}
 			}
-		}
+		const hasAdaptiveStorageLimit =
+			this._isAdaptiveReplicating &&
+			this.replicationController?.maxMemoryLimit != null;
+		const useJoinWarmupFastPath =
+			!forceFreshDelivery &&
+			warmupPeers.size > 0 &&
+			!hasSelfWarmupChange &&
+			!hasAdaptiveStorageLimit;
+		const immediateRebalanceChanges = useJoinWarmupFastPath
+			? changes.filter(
+					(change) =>
+						!(
+							change.range.hash === selfHash &&
+							(change.type === "added" || change.type === "replaced")
+						),
+				)
+			: changes;
 
 		try {
 			const uncheckedDeliver: Map<
@@ -6191,15 +6246,15 @@ export class SharedLog<
 				if (!entries || entries.size === 0) {
 					return;
 				}
-				const isJoinWarmupTarget = addedPeers.has(target);
-				const bypassRecentDedupe = isJoinWarmupTarget || forceFreshDelivery;
-				this.dispatchMaybeMissingEntries(target, entries, {
-					bypassRecentDedupe,
-					retryScheduleMs: isJoinWarmupTarget
-						? JOIN_WARMUP_RETRY_SCHEDULE_MS
-						: undefined,
-					forceFreshDelivery,
-				});
+					const isWarmupTarget = warmupPeers.has(target);
+					const bypassRecentDedupe = isWarmupTarget || forceFreshDelivery;
+					this.dispatchMaybeMissingEntries(target, entries, {
+						bypassRecentDedupe,
+						retryScheduleMs: isWarmupTarget
+							? JOIN_WARMUP_RETRY_SCHEDULE_MS
+							: undefined,
+						forceFreshDelivery,
+					});
 				uncheckedDeliver.delete(target);
 			};
 			const queueUncheckedDeliver = (
@@ -6220,18 +6275,85 @@ export class SharedLog<
 				}
 			};
 
-			for await (const entryReplicated of toRebalance<R>(
-				changes,
-				this.entryCoordinatesIndex,
-				this.recentlyRebalanced,
-				{ forceFresh: forceFreshDelivery },
-			)) {
+				if (immediateRebalanceChanges.length > 0) {
+					for await (const entryReplicated of toRebalance<R>(
+						immediateRebalanceChanges,
+						this.entryCoordinatesIndex,
+						this.recentlyRebalanced,
+						{
+							forceFresh: forceFreshDelivery || useJoinWarmupFastPath,
+						},
+					)) {
 				if (this.closed) {
 					break;
 				}
 
-				let oldPeersSet: Set<string> | undefined;
-				if (!forceFreshDelivery) {
+					if (useJoinWarmupFastPath) {
+						let oldPeersSet: Set<string> | undefined;
+						const gid = entryReplicated.gid;
+						oldPeersSet = gidPeersHistorySnapshot.get(gid);
+						if (!gidPeersHistorySnapshot.has(gid)) {
+							const existing = this._gidPeersHistory.get(gid);
+							oldPeersSet = existing ? new Set(existing) : undefined;
+							gidPeersHistorySnapshot.set(gid, oldPeersSet);
+						}
+
+						for (const target of warmupPeers) {
+							queueUncheckedDeliver(target, entryReplicated);
+						}
+
+						const candidatePeers = new Set<string>([selfHash]);
+						for (const target of warmupPeers) {
+							candidatePeers.add(target);
+						}
+						if (oldPeersSet) {
+							for (const oldPeer of oldPeersSet) {
+								candidatePeers.add(oldPeer);
+							}
+						}
+
+						const currentPeers = await this.findLeaders(
+							entryReplicated.coordinates,
+							entryReplicated,
+							{
+								roleAge: 0,
+								candidates: candidatePeers,
+								persist: false,
+							},
+						);
+
+						if (oldPeersSet) {
+							for (const oldPeer of oldPeersSet) {
+								if (!currentPeers.has(oldPeer)) {
+									this.removePruneRequestSent(entryReplicated.hash);
+								}
+							}
+						}
+
+						this.addPeersToGidPeerHistory(
+							entryReplicated.gid,
+							currentPeers.keys(),
+							true,
+						);
+
+						if (!currentPeers.has(selfHash)) {
+							this.pruneDebouncedFnAddIfNotKeeping({
+								key: entryReplicated.hash,
+								value: { entry: entryReplicated, leaders: currentPeers },
+							});
+
+							this.responseToPruneDebouncedFn.delete(entryReplicated.hash);
+						} else {
+							this.pruneDebouncedFn.delete(entryReplicated.hash);
+							await this._pendingDeletes
+								.get(entryReplicated.hash)
+								?.reject(new Error("Failed to delete, is leader again"));
+							this.removePruneRequestSent(entryReplicated.hash);
+						}
+						continue;
+					}
+
+					let oldPeersSet: Set<string> | undefined;
 					const gid = entryReplicated.gid;
 					oldPeersSet = gidPeersHistorySnapshot.get(gid);
 					if (!gidPeersHistorySnapshot.has(gid)) {
@@ -6239,18 +6361,18 @@ export class SharedLog<
 						oldPeersSet = existing ? new Set(existing) : undefined;
 						gidPeersHistorySnapshot.set(gid, oldPeersSet);
 					}
-				}
-				let isLeader = false;
 
-				let currentPeers = await this.findLeaders(
-					entryReplicated.coordinates,
-					entryReplicated,
-					{
-						// we do this to make sure new replicators get data even though they are not mature so they can figure out if they want to replicate more or less
-						// TODO make this smarter because if a new replicator is not mature and want to replicate too much data the syncing overhead can be bad
-						roleAge: 0,
-					},
-				);
+					let isLeader = false;
+					const currentPeers = await this.findLeaders(
+						entryReplicated.coordinates,
+						entryReplicated,
+						{
+							// We do this to make sure new replicators get data even though
+							// they are not mature so they can figure out if they want to
+							// replicate more or less.
+							roleAge: 0,
+						},
+					);
 
 					for (const [currentPeer] of currentPeers) {
 						if (currentPeer === this.node.identity.publicKey.hashcode()) {
@@ -6263,41 +6385,63 @@ export class SharedLog<
 						}
 					}
 
-				if (oldPeersSet) {
-					for (const oldPeer of oldPeersSet) {
-						if (!currentPeers.has(oldPeer)) {
-							this.removePruneRequestSent(entryReplicated.hash);
+					if (oldPeersSet) {
+						for (const oldPeer of oldPeersSet) {
+							if (!currentPeers.has(oldPeer)) {
+								this.removePruneRequestSent(entryReplicated.hash);
+							}
 						}
 					}
-				}
 
-				this.addPeersToGidPeerHistory(
-					entryReplicated.gid,
-					currentPeers.keys(),
-					true,
-				);
+					this.addPeersToGidPeerHistory(
+						entryReplicated.gid,
+						currentPeers.keys(),
+						true,
+					);
 
-				if (!isLeader) {
-					this.pruneDebouncedFnAddIfNotKeeping({
-						key: entryReplicated.hash,
-						value: { entry: entryReplicated, leaders: currentPeers },
-					});
+					if (!isLeader) {
+						this.pruneDebouncedFnAddIfNotKeeping({
+							key: entryReplicated.hash,
+							value: { entry: entryReplicated, leaders: currentPeers },
+						});
 
-					this.responseToPruneDebouncedFn.delete(entryReplicated.hash); // don't allow others to prune because of expecting me to replicating this entry
-				} else {
-					this.pruneDebouncedFn.delete(entryReplicated.hash);
-					await this._pendingDeletes
-						.get(entryReplicated.hash)
-						?.reject(new Error("Failed to delete, is leader again"));
-					this.removePruneRequestSent(entryReplicated.hash);
+						this.responseToPruneDebouncedFn.delete(entryReplicated.hash); // don't allow others to prune because of expecting me to replicating this entry
+					} else {
+						this.pruneDebouncedFn.delete(entryReplicated.hash);
+						await this._pendingDeletes
+							.get(entryReplicated.hash)
+							?.reject(new Error("Failed to delete, is leader again"));
+						this.removePruneRequestSent(entryReplicated.hash);
+					}
+				}
 				}
-			}
 
-			if (forceFreshDelivery || addedPeers.size > 0) {
-				// Schedule a coalesced background sweep for churn/join windows instead of
-				// scanning the whole index synchronously on each replication change.
-				this.scheduleRepairSweep({ forceFreshDelivery, addedPeers });
-			}
+				if (forceFreshDelivery) {
+					// Removed/shrunk ranges still need the authoritative background pass.
+					this.scheduleRepairSweep({ forceFreshDelivery, addedPeers });
+				} else if (useJoinWarmupFastPath) {
+					// Pure join warmup uses the cheap immediate maybe-missing dispatch above,
+					// then defers the authoritative sweep so it does not compete with the
+					// write burst itself.
+					const peers = new Set(addedPeers);
+					const timer = setTimeout(() => {
+						this._repairRetryTimers.delete(timer);
+						if (this.closed) {
+							return;
+						}
+						this.scheduleRepairSweep({
+							forceFreshDelivery: false,
+							addedPeers: peers,
+						});
+					}, 250);
+					timer.unref?.();
+					this._repairRetryTimers.add(timer);
+				} else if (addedPeers.size > 0) {
+					this.scheduleRepairSweep({
+						forceFreshDelivery: false,
+						addedPeers,
+					});
+				}
 
 			for (const target of [...uncheckedDeliver.keys()]) {
 				flushUncheckedDeliverTarget(target);
@@ -6336,6 +6480,7 @@ export class SharedLog<
 		if (!prev || prev < now) {
 			this.latestReplicationInfoMessage.set(fromHash, now);
 		}
+		this.invalidateSharedLogTopicSubscribersCache();
 
 		return this.handleSubscriptionChange(
 			evt.detail.from,
@@ -6356,6 +6501,7 @@ export class SharedLog<
 
 		this.remoteBlocks.onReachable(evt.detail.from);
 		this._replicationInfoBlockedPeers.delete(evt.detail.from.hashcode());
+		this.invalidateSharedLogTopicSubscribersCache();
 
 		await this.handleSubscriptionChange(
 			evt.detail.from,