@peerbit/shared-log 13.1.3 → 13.1.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peerbit/shared-log",
-  "version": "13.1.3",
+  "version": "13.1.5",
   "description": "Shared log",
   "sideEffects": false,
   "type": "module",
@@ -62,27 +62,27 @@
     "pino": "^9.4.0",
     "uint8arrays": "^5.1.0",
     "@peerbit/any-store": "2.2.9",
+    "@peerbit/blocks-interface": "2.0.9",
+    "@peerbit/blocks": "4.1.2",
     "@peerbit/cache": "3.0.0",
-    "@peerbit/blocks-interface": "2.0.8",
-    "@peerbit/indexer-interface": "3.0.3",
     "@peerbit/crypto": "3.1.1",
-    "@peerbit/blocks": "4.1.1",
+    "@peerbit/indexer-interface": "3.0.3",
     "@peerbit/indexer-sqlite3": "3.0.6",
+    "@peerbit/log": "6.0.26",
     "@peerbit/logger": "2.0.1",
-    "@peerbit/program": "6.0.19",
+    "@peerbit/pubsub": "5.2.2",
     "@peerbit/pubsub-interface": "5.1.1",
-    "@peerbit/pubsub": "5.2.0",
-    "@peerbit/rpc": "6.0.23",
+    "@peerbit/program": "6.0.21",
     "@peerbit/riblt": "1.2.0",
+    "@peerbit/rpc": "6.0.25",
     "@peerbit/stream-interface": "6.0.7",
-    "@peerbit/log": "6.0.24",
     "@peerbit/time": "3.0.0"
   },
   "devDependencies": {
     "@types/libsodium-wrappers": "^0.7.14",
     "@types/pidusage": "^2.0.5",
     "uuid": "^10.0.0",
-    "@peerbit/test-utils": "3.0.23"
+    "@peerbit/test-utils": "3.0.25"
   },
   "repository": {
     "type": "git",

package/src/index.ts

@@ -1580,67 +1580,54 @@ export class SharedLog<
 			return keys.slice();
 		};
 
-		// Prefer the bounded peer set we already know from the fanout overlay.
-		if (this._fanoutChannel && (topic === this.topic || topic === this.rpc.topic)) {
-			const hashes = this._fanoutChannel
-				.getPeerHashes({ includeSelf: false })
-				.slice(0, maxPeers);
-			if (hashes.length === 0) return cache([]);
-
-			const keys = await Promise.all(
-				hashes.map((hash) => this._resolvePublicKeyFromHash(hash)),
-			);
-			const uniqueKeys: PublicSignKey[] = [];
-			const seen = new Set<string>();
-			const selfHash = this.node.identity.publicKey.hashcode();
-			for (const key of keys) {
-				if (!key) continue;
-				const hash = key.hashcode();
-				if (hash === selfHash) continue;
-				if (seen.has(hash)) continue;
-				seen.add(hash);
-				uniqueKeys.push(key);
-			}
-			return cache(uniqueKeys);
-		}
-
 		const selfHash = this.node.identity.publicKey.hashcode();
-		const hashes: string[] = [];
+		const hashes = new Set<string>();
+		const keysByHash = new Map<string, PublicSignKey>();
+		const addHash = (hash: string | undefined) => {
+			if (!hash || hash === selfHash || keysByHash.has(hash)) {
+				return;
+			}
+			hashes.add(hash);
+		};
+		const addKey = (key: PublicSignKey | undefined) => {
+			if (!key) {
+				return;
+			}
+			const hash = key.hashcode();
+			if (hash === selfHash) {
+				return;
+			}
+			hashes.delete(hash);
+			keysByHash.set(hash, key);
+		};
 
-		// Best-effort provider discovery (bounded). This requires bootstrap trackers.
-		try {
-			const fanoutService = getSharedLogFanoutService(this.node.services);
-			if (fanoutService?.queryProviders) {
-				const ns = `shared-log|${this.topic}`;
-				const seed = hashToSeed32(topic);
-				const providers: string[] = await fanoutService.queryProviders(ns, {
-					want: maxPeers,
-					seed,
-				});
-				for (const h of providers ?? []) {
-					if (!h || h === selfHash) continue;
-					hashes.push(h);
-					if (hashes.length >= maxPeers) break;
-				}
+		// Fanout is a useful hint, but it can lag direct pubsub connectivity. Keep
+		// collecting other local views instead of treating an empty fanout snapshot as
+		// authoritative absence.
+		if (this._fanoutChannel && (topic === this.topic || topic === this.rpc.topic)) {
+			for (const hash of this._fanoutChannel.getPeerHashes({
+				includeSelf: false,
+			})) {
+				addHash(hash);
+				if (hashes.size + keysByHash.size >= maxPeers) break;
 			}
-		} catch {
-			// Best-effort only.
 		}
 
-		// Next, use already-connected peer streams (bounded and cheap).
-		const peerMap: Map<string, unknown> | undefined = (this.node.services.pubsub as any)
-			?.peers;
-		if (peerMap?.keys) {
-			for (const h of peerMap.keys()) {
-				if (!h || h === selfHash) continue;
-				hashes.push(h);
-				if (hashes.length >= maxPeers) break;
+		// Already-connected peer streams are cheap and are the strongest local signal
+		// when fanout/provider membership is stale.
+		const peerMap: Map<string, { publicKey?: PublicSignKey }> | undefined = (this.node
+			.services.pubsub as any)?.peers;
+		if (peerMap?.entries) {
+			for (const [hash, peer] of peerMap.entries()) {
+				addKey(peer?.publicKey);
+				addHash(hash);
+				if (hashes.size + keysByHash.size >= maxPeers) break;
 			}
 		}
 
-		// Finally, fall back to libp2p connections (e.g. bootstrap peers) without requiring
-		// any global topic membership view.
-		if (hashes.length < maxPeers) {
+		// Libp2p connections cover bootstrap/direct peers even before a higher-level
+		// topic subscriber snapshot has converged.
+		if (hashes.size + keysByHash.size < maxPeers) {
 			const connectionManager = (this.node.services.pubsub as any)?.components
 				?.connectionManager;
 			const connections = connectionManager?.getConnections?.() ?? [];
@@ -1648,38 +1635,48 @@ export class SharedLog<
 				const peerId = conn?.remotePeer;
 				if (!peerId) continue;
 				try {
-					const h = getPublicKeyFromPeerId(peerId).hashcode();
-					if (!h || h === selfHash) continue;
-					hashes.push(h);
-					if (hashes.length >= maxPeers) break;
+					addKey(getPublicKeyFromPeerId(peerId));
+					if (hashes.size + keysByHash.size >= maxPeers) break;
 				} catch {
 					// Best-effort only.
 				}
 			}
 		}
 
-		if (hashes.length === 0) return cache([]);
-
-		const uniqueHashes: string[] = [];
-		const seen = new Set<string>();
-		for (const h of hashes) {
-			if (seen.has(h)) continue;
-			seen.add(h);
-			uniqueHashes.push(h);
-			if (uniqueHashes.length >= maxPeers) break;
+		// Best-effort provider discovery (bounded). This requires bootstrap trackers.
+		if (hashes.size + keysByHash.size < maxPeers) {
+			try {
+				const fanoutService = getSharedLogFanoutService(this.node.services);
+				if (fanoutService?.queryProviders) {
+					const ns = `shared-log|${this.topic}`;
+					const seed = hashToSeed32(topic);
+					const providers: string[] = await fanoutService.queryProviders(ns, {
+						want: maxPeers - keysByHash.size - hashes.size,
+						seed,
+					});
+					for (const hash of providers ?? []) {
+						addHash(hash);
+						if (hashes.size + keysByHash.size >= maxPeers) break;
+					}
+				}
+			} catch {
+				// Best-effort only.
+			}
 		}
 
+		if (hashes.size === 0 && keysByHash.size === 0) return cache([]);
+
+		const unresolvedHashes = [...hashes].slice(
+			0,
+			Math.max(0, maxPeers - keysByHash.size),
+		);
 		const keys = await Promise.all(
-			uniqueHashes.map((hash) => this._resolvePublicKeyFromHash(hash)),
+			unresolvedHashes.map((hash) => this._resolvePublicKeyFromHash(hash)),
 		);
-		const uniqueKeys: PublicSignKey[] = [];
 		for (const key of keys) {
-			if (!key) continue;
-			const hash = key.hashcode();
-			if (hash === selfHash) continue;
-			uniqueKeys.push(key);
+			addKey(key);
 		}
-		return cache(uniqueKeys);
+		return cache([...keysByHash.values()].slice(0, maxPeers));
 	}
 
 	private invalidateTopicSubscribersCache(...topics: (string | undefined)[]) {
@@ -6556,10 +6553,11 @@ export class SharedLog<
 		// If it is still warming up (for example, only contains self), supplement with
 		// current subscribers until we have enough candidates for this decision.
 		let peerFilter: Set<string> | undefined = undefined;
+		let selfReplicating = false;
 		if (options?.candidates) {
 			peerFilter = new Set(options.candidates);
 		} else {
-			const selfReplicating = await this.isReplicating();
+			selfReplicating = await this.isReplicating();
 			if (this.uniqueReplicators.size > 0) {
 				peerFilter = new Set(this.uniqueReplicators);
 				if (selfReplicating) {
@@ -6601,6 +6599,17 @@ export class SharedLog<
 			}
 		}
 
+		if (!options?.candidates) {
+			// Reachability snapshots can briefly under-report peers. Do not let that
+			// turn a known mature indexed range into a false self-only full replica.
+			peerFilter = await this.includeIndexedLeaderCandidatesWhenUnderfilled(
+				peerFilter,
+				roleAge,
+				cursors.length,
+				selfReplicating,
+			);
+		}
+
 		if (!options?.candidates) {
 			const fullReplicaLeaders = await this.findFullReplicaLeaders(
 				cursors.length,
@@ -6624,6 +6633,47 @@ export class SharedLog<
 		);
 	}
 
+	private async includeIndexedLeaderCandidatesWhenUnderfilled(
+		peerFilter: Set<string> | undefined,
+		roleAge: number,
+		replicas: number,
+		selfReplicating: boolean,
+	): Promise<Set<string> | undefined> {
+		if (!peerFilter || peerFilter.size > replicas) {
+			return peerFilter;
+		}
+
+		const selfHash = this.node.identity.publicKey.hashcode();
+		const now = Date.now();
+		const iterator = this.replicationIndex.iterate(
+			{},
+			{ shape: { hash: true, timestamp: true }, reference: true },
+		);
+
+		try {
+			for (;;) {
+				const batch = await iterator.next(64);
+				if (batch.length === 0) {
+					break;
+				}
+				for (const result of batch) {
+					const range = result.value;
+					if (range.hash === selfHash && !selfReplicating) {
+						continue;
+					}
+					if (!isMatured(range, now, roleAge)) {
+						continue;
+					}
+					peerFilter.add(range.hash);
+				}
+			}
+		} finally {
+			await iterator.close();
+		}
+
+		return peerFilter;
+	}
+
 	private async findFullReplicaLeaders(
 		replicas: number,
 		roleAge: number,