@optimystic/db-p2p 0.14.1 → 0.16.3

package/src/libp2p-key-network.ts

@@ -1,4 +1,4 @@
-import type { AbortOptions, Libp2p, PeerId, Stream } from "@libp2p/interface";
+import type { AbortOptions, Connection, Libp2p, PeerId, Stream } from "@libp2p/interface";
 import { toString as u8ToString } from 'uint8arrays'
 import type { ClusterPeers, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork } from "@optimystic/db-core";
 import { peerIdFromString } from '@libp2p/peer-id'
@@ -31,12 +31,33 @@ export const FIND_COORDINATOR_ERROR_CODES = {
 	 */
 	SELF_COORDINATION_EXHAUSTED: 'SELF_COORDINATION_EXHAUSTED',
 	/** No peer (including self) is an eligible coordinator. */
-	NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE'
+	NO_COORDINATOR_AVAILABLE: 'NO_COORDINATOR_AVAILABLE',
+	/**
+	 * The candidate set was non-empty but every non-self candidate serves a
+	 * DIFFERENT network's protocol (or none of this network's). Distinct from
+	 * NO_COORDINATOR_AVAILABLE so a Sereus-style trace points at the real cause —
+	 * "peer(s) do not serve this network's protocol" — instead of a generic
+	 * "all candidates excluded" / super-majority failure.
+	 */
+	NO_NETWORK_COORDINATOR: 'NO_NETWORK_COORDINATOR'
 } as const;
 
 export type FindCoordinatorErrorCode =
 	typeof FIND_COORDINATOR_ERROR_CODES[keyof typeof FIND_COORDINATOR_ERROR_CODES];
 
+/**
+ * Network-membership classification of a peer relative to THIS node's network,
+ * derived from the peer's libp2p peerStore protocol list:
+ * - `serves`   — advertises this network's namespaced `cluster`/`repo` protocol.
+ * - `foreign`  — has a non-empty protocol list but none for this network → another network.
+ * - `unknown`  — protocol list empty / peer absent → identify not yet completed. This is
+ *                both a fresh same-network peer (will flip to `serves`) AND a cross-network
+ *                peer (whose network-namespaced identify can NEVER complete, so it stays
+ *                `unknown` forever) — indistinguishable at a single instant, separated over
+ *                the retry/stabilization window.
+ */
+export type NetworkMembership = 'serves' | 'foreign' | 'unknown';
+
 export class FindCoordinatorError extends Error {
 	readonly code: FindCoordinatorErrorCode;
 	constructor(code: FindCoordinatorErrorCode, message: string) {
@@ -94,7 +115,16 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 		selfCoordinationConfig?: SelfCoordinationConfig,
 		networkMode?: NetworkMode,
 		persistence?: NetworkStatePersistence,
-		private readonly reputation?: IPeerReputation
+		private readonly reputation?: IPeerReputation,
+		/**
+		 * Network-namespaced protocol prefix (`/optimystic/<networkName>`). When
+		 * provided, coordinator/cohort selection is scoped to peers that serve THIS
+		 * network's `cluster`/`repo` protocol, so a peer that only belongs to another
+		 * network sharing the same physical nodes/bootstraps is never chosen. When
+		 * ABSENT, the membership filter is disabled (today's exact behavior) — required
+		 * for backward compatibility because most call sites don't know the network name.
+		 */
+		private readonly protocolPrefix?: string
 	) {
 		this.selfCoordinationConfig = {
 			gracePeriodMs: selfCoordinationConfig?.gracePeriodMs ?? 30_000,
@@ -291,17 +321,39 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 		return undefined
 	}
 
+	/**
+	 * True for a circuit-relay ("limited") connection. libp2p stamps a relayed
+	 * connection with `limits` (per-circuit data/duration caps); we additionally
+	 * sniff the multiaddr for `/p2p-circuit` as a fallback for transports/versions
+	 * that don't populate `limits`.
+	 */
+	private isLimitedConnection(c: Connection): boolean {
+		if ((c as { limits?: unknown }).limits != null) return true
+		const addr = c.remoteAddr?.toString?.()
+		return addr != null && addr.includes('/p2p-circuit')
+	}
+
 	connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
 		const conns = this.libp2p.getConnections?.(peerId) ?? []
 		// Filter to only-open connections so a closing/closed entry that libp2p
 		// hasn't yet evicted from its index doesn't get picked up here.
-		const open = conns.find(c => c?.status === 'open' && typeof c?.newStream === 'function')
-		if (open) {
+		const open = conns.filter(c => c?.status === 'open' && typeof c?.newStream === 'function')
+		// Prefer a DIRECT connection over a limited (circuit-relay) one for the RPC.
+		// A relayed/limited connection can be reset by the relay once a per-circuit
+		// cap or reservation lapses (@libp2p/circuit-relay-v2), surfacing to the
+		// coordinator as a StreamResetError that fails consensus. After DCUtR upgrades
+		// a relayed link to direct, both connections briefly coexist — picking the
+		// direct one avoids riding the soon-to-be-reset circuit. We only fall back to
+		// the limited connection (with runOnLimitedConnection) when it is the only open
+		// path — the steady state for browsers and NATed peers before any upgrade.
+		const chosen = open.find(c => !this.isLimitedConnection(c)) ?? open[0]
+		if (chosen) {
 			// runOnLimitedConnection: true is required to open a stream over a
 			// circuit-relay (limited) connection — the steady-state path for
 			// browsers and NATed peers. Without it, the warm relay connection
-			// from a prior dialProtocol cannot be reused on subsequent RPCs.
-			return open.newStream([protocol], {
+			// from a prior dialProtocol cannot be reused on subsequent RPCs. It is
+			// a harmless no-op on the preferred direct connection.
+			return chosen.newStream([protocol], {
 				signal: options?.signal,
 				runOnLimitedConnection: true,
 				negotiateFully: false
@@ -332,6 +384,12 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 		const t0 = Date.now();
 		const excludedSet = new Set<string>((_options?.excludedPeers ?? []).map(p => p.toString()))
 		const keyStr = this.toCacheKey(key).substring(0, 12);
+		// Tracks whether the network-membership filter excluded an UNCONFIRMED candidate
+		// — `foreign` (another network) OR `unknown` (not yet confirmed to serve this
+		// network) — during any attempt. If selection ultimately fails with self
+		// unavailable, this lets us surface NO_NETWORK_COORDINATOR (the real cause)
+		// instead of the generic NO_COORDINATOR_AVAILABLE.
+		let droppedUnconfirmedAnyAttempt = false;
 
 		this.log('findCoordinator:start key=%s excluded=%o', keyStr, Array.from(excludedSet).map(s => s.substring(0, 12)))
 
@@ -367,7 +425,17 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 					.sort((a, b) => (this.reputation?.getScore(a) ?? 0) - (this.reputation?.getScore(b) ?? 0))
 				this.log('findCoordinator:fret-connected key=%s count=%d peers=%o', keyStr, connectedFretIds.length, connectedFretIds.map(s => s.substring(0, 12)))
 
-				const pick = connectedFretIds[0]
+				// Network-membership scoping (no-op when protocolPrefix is unset): only a peer
+				// CONFIRMED to serve this network ('serves') is eligible — both `foreign`
+				// (another network) and `unknown` (not yet identified) peers are excluded
+				// from selection. A cross-network peer is permanently 'unknown' (its
+				// namespaced identify never completes), so it is never gambled on; over the
+				// 3×500ms retry window a genuine same-network peer flips to 'serves' on a
+				// re-read of the peerStore and is selected normally on that attempt. Self
+				// always classifies as 'serves' and stays eligible.
+				const { ranked, droppedUnconfirmed } = await this.filterByMembership(connectedFretIds)
+				if (droppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
+				const pick = ranked[0]
 				if (pick) {
 					const pid = peerIdFromString(pick)
 					this.recordCoordinator(key, pid)
@@ -378,15 +446,23 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 				this.log('findCoordinator getNeighborIdsForKey failed - %o', err)
 			}
 
-			// fallback: prefer any existing connected peer that's not excluded or banned
-			const connectedPick = connected
+			// fallback: prefer any existing connected peer that's not excluded or banned,
+			// scoped to this network's serving peers (a `foreign` or not-yet-confirmed
+			// `unknown` peer is never picked). Note this candidate set is built from
+			// connected REMOTE peers and never includes self, so when no serving peer is
+			// present selection falls through to the last-resort self-coordination block.
+			const connectedCandidates = connected
 				.filter(p => !excludedSet.has(p.toString()) && !(this.reputation?.isBanned(p.toString())))
 				.sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))
-				[0]
+				.map(p => p.toString())
+			const { ranked: connRanked, droppedUnconfirmed: connDroppedUnconfirmed } = await this.filterByMembership(connectedCandidates)
+			if (connDroppedUnconfirmed) droppedUnconfirmedAnyAttempt = true
+			const connectedPick = connRanked[0]
 			if (connectedPick) {
-				this.recordCoordinator(key, connectedPick)
+				const pid = peerIdFromString(connectedPick)
+				this.recordCoordinator(key, pid)
 				this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'connected-fallback')
-				return connectedPick
+				return pid
 			}
 
 			// If no connections and not the last attempt, wait and retry
@@ -424,6 +500,20 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 			return self
 		}
 
+		// Self is excluded and selection found no eligible peer. If the membership filter is
+		// the reason the candidate set emptied (the only other peers are `foreign` — serving
+		// a DIFFERENT network — or `unknown` — not yet confirmed to serve this network),
+		// surface a distinct, accurate cause instead of the generic codes below.
+		if (droppedUnconfirmedAnyAttempt) {
+			this.log('findCoordinator:no-network-coordinator key=%s prefix=%s self=%s',
+				keyStr, this.protocolPrefix ?? '?', self.toString().substring(0, 12))
+			throw new FindCoordinatorError(
+				FIND_COORDINATOR_ERROR_CODES.NO_NETWORK_COORDINATOR,
+				`No coordinator available for key on network ${this.protocolPrefix ?? '?'}: ` +
+				`the remaining candidate peer(s) are foreign or not-yet-confirmed to serve this network's cluster/repo protocol.`
+			);
+		}
+
 		// Self is excluded. On a solo/bootstrap node (HWM<=1 and no other connected/FRET peers),
 		// this means the caller already tried self and the retry has nowhere to go — surface a
 		// distinct error so retry logic stops and the original first-attempt cause is preserved.
@@ -458,7 +548,7 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 	private parseMultiaddrs(addrs: string[]): string[] {
 		const out: string[] = []
 		for (const a of addrs) {
-			try { multiaddr(a); out.push(a) } catch (err) { console.warn('invalid multiaddr from connection', a, err) }
+			try { multiaddr(a); out.push(a) } catch (err) { this.log('WARN: invalid multiaddr from connection %s %o', a, err) }
 		}
 		return out
 	}
@@ -467,13 +557,70 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 		const t0 = Date.now();
 		const fret = this.getFret()
 		const coord = await hashKey(key)
-		const cohort = fret.assembleCohort(coord, this.clusterSize)
+		// When membership scoping is active, over-fetch a wider proximity band so the
+		// nearest peers that SERVE this network are in the candidate pool even if cross-
+		// network peers sit nearer the key (see membershipOverfetch).
+		const wants = this.protocolPrefix != null ? this.membershipOverfetch() : this.clusterSize
+		const cohort = fret.assembleCohort(coord, wants)
 		const keyStr = this.toCacheKey(key).substring(0, 12);
 		this.log('findCluster:start key=%s', keyStr);
 
 		// Include self in the cohort
 		const selfId = this.libp2p.peerId.toString()
-		const ids = Array.from(new Set([...cohort, selfId]))
+		let ids = Array.from(new Set([...cohort, selfId]))
+
+		// Network-membership scoping (no-op when protocolPrefix is unset): a cohort
+		// member that serves a DIFFERENT network's protocol can never negotiate THIS
+		// network's cluster/repo dial, so it guarantees a super-majority failure rather
+		// than contributing a promise. Drop such 'foreign' members; build the cohort from
+		// positively-'serves' members only and NEVER admit a not-yet-identified ('unknown')
+		// member. A permanently cross-network peer and a freshly-discovered same-network
+		// peer mid-identify are indistinguishable while 'unknown' (both have an empty
+		// peerStore protocol list), so admitting an 'unknown' on the strength of a viability
+		// floor risks pulling a cross-network contaminant into the cohort — its repo dial
+		// then negotiates a different network's protocol and the whole write fails. A fresh
+		// same-network peer is not starved: it flips to 'serves' once identify completes and
+		// is re-included on the caller's retry, and in the meantime a self-only cohort still
+		// completes the write under allowClusterDownsize (the default).
+		// Scoped path only: one peerStore read per cohort member yields both protocols
+		// (for membership classification here) and addresses (reused at backfill below),
+		// so a finally-selected member isn't fetched from the peerStore twice. Left
+		// undefined on the unscoped path, which never classifies membership.
+		let peerStoreRecords: Record<string, { protocols: string[]; addrs: string[] }> | undefined
+		if (this.protocolPrefix != null) {
+			// `cohort` is the over-fetched nearest-first band. Classify each non-self
+			// member, preserving proximity order within each tier.
+			const nonSelf = cohort.filter(id => id !== selfId)
+			peerStoreRecords = await this.getPeerStoreRecordsByPeer(nonSelf)
+			const serves: string[] = []
+			const unknown: string[] = []
+			let foreignDropped = 0
+			for (const id of nonSelf) {
+				const m = this.membershipOf(id, peerStoreRecords[id]?.protocols)
+				if (m === 'serves') serves.push(id)
+				else if (m === 'unknown') unknown.push(id)
+				else foreignDropped++
+			}
+			// Take the nearest `clusterSize - 1` SERVING peers. Self is ALWAYS added below and
+			// counts toward `clusterSize` (matching the unscoped path, where `assembleCohort`
+			// returns the nearest `clusterSize` peers INCLUDING self when self is near the key —
+			// the coordinator case), so reserving a slot for self keeps a healthy same-network
+			// cohort at exactly `clusterSize` members rather than `clusterSize + 1`. Over-sizing
+			// would inflate the super-majority promise count (ceil(peerCount * threshold)) above
+			// what the configured `clusterSize` intends and hurt write availability. 'unknown'
+			// members are never backfilled: an 'unknown' peer may be a permanently cross-network
+			// contaminant whose repo dial cannot negotiate this network's protocol, and a fresh
+			// same-network peer mid-identify is indistinguishable from it. We therefore admit
+			// only positively-'serves' peers; when self is the sole serving member the cohort is
+			// self-only, which completes the write under allowClusterDownsize (the default) and
+			// re-includes any legitimate peer as 'serves' on the caller's retry once identify
+			// completes. `unknown.length` is still computed above for the diagnostic log line.
+			const nonSelfTarget = Math.max(0, this.clusterSize - 1)
+			const others = serves.slice(0, nonSelfTarget)
+			ids = Array.from(new Set([selfId, ...others]))
+			this.log('findCluster:membership key=%s serves=%d unknown=%d foreignDropped=%d kept=%d',
+				keyStr, serves.length, unknown.length, foreignDropped, ids.length)
+		}
 
 		const connectedByPeer = this.getConnectedAddrsByPeer()
 		const connectedPeerIds = Object.keys(connectedByPeer)
@@ -481,8 +628,17 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 		// Backfill addresses from the peerStore for cohort members we don't have
 		// a live connection to. The cohort is keyspace-determined and can include
 		// peers we know-of but haven't dialed yet; without this backfill those
-		// would be silently dropped.
-		const peerStoreAddrs = await this.getPeerStoreAddrsByPeer(ids.filter(id => id !== selfId))
+		// would be silently dropped. On the scoped path reuse the addresses already
+		// read into `peerStoreRecords` above (no second store.get per member); on the
+		// unscoped path (no record map) do the single peerStore read as before.
+		const backfillIds = ids.filter(id => id !== selfId)
+		const peerStoreAddrs = peerStoreRecords
+			? Object.fromEntries(
+				backfillIds
+					.map(id => [id, peerStoreRecords![id]?.addrs ?? []] as const)
+					.filter(([, addrs]) => addrs.length > 0)
+			)
+			: await this.getPeerStoreAddrsByPeer(backfillIds)
 
 		this.log('findCluster key=%s fretCohort=%d connected=%d', keyStr, cohort.length, connectedPeerIds.length)
 		if (verbose) this.log('findCluster:detail key=%s cohortPeers=%o connectedPeers=%o', keyStr, ids, connectedPeerIds)
@@ -541,4 +697,111 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 		}))
 		return out
 	}
+
+	/**
+	 * Single-pass peerStore read returning BOTH protocols and addresses per peer from one
+	 * `store.get` call. Used on the membership-scoped `findCluster` hot path, where the
+	 * cohort needs protocols (to classify membership) AND addresses (to backfill dial
+	 * targets) for the same peers — reading them together avoids a second `store.get` per
+	 * finally-selected member. Same error handling as {@link getPeerStoreProtocolsByPeer}
+	 * and {@link getPeerStoreAddrsByPeer}: a missing peer or peerStore failure is left
+	 * absent from the map (caller treats absent protocols as 'unknown', absent addrs as none).
+	 */
+	private async getPeerStoreRecordsByPeer(ids: string[]): Promise<Record<string, { protocols: string[]; addrs: string[] }>> {
+		const out: Record<string, { protocols: string[]; addrs: string[] }> = {}
+		const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[]; addresses?: Array<{ multiaddr: { toString(): string } }> }> } }).peerStore
+		if (!store?.get) return out
+		await Promise.all(ids.map(async (idStr) => {
+			try {
+				const pid = peerIdFromString(idStr)
+				const peer = await store.get!(pid)
+				const addrs = (peer?.addresses ?? []).map(a => a.multiaddr.toString())
+				out[idStr] = { protocols: peer?.protocols ?? [], addrs }
+			} catch {
+				// Unknown peer or peerStore failure — leave out of the map.
+			}
+		}))
+		return out
+	}
+
+	/**
+	 * Prefetch each peer's advertised protocol list from the libp2p peerStore.
+	 * Returns a map from peer-id string to its protocols (empty array when the peer
+	 * is absent or has not yet been identified). Mirrors {@link getPeerStoreAddrsByPeer};
+	 * errors are swallowed so a peerStore hiccup degrades to "unknown" rather than throwing.
+	 */
+	private async getPeerStoreProtocolsByPeer(ids: string[]): Promise<Record<string, string[]>> {
+		const out: Record<string, string[]> = {}
+		const store = (this.libp2p as { peerStore?: { get?: (id: PeerId) => Promise<{ protocols?: string[] }> } }).peerStore
+		if (!store?.get) return out
+		await Promise.all(ids.map(async (idStr) => {
+			try {
+				const pid = peerIdFromString(idStr)
+				const peer = await store.get!(pid)
+				out[idStr] = peer?.protocols ?? []
+			} catch {
+				// Unknown peer or peerStore failure — leave out (treated as 'unknown').
+			}
+		}))
+		return out
+	}
+
+	/**
+	 * Over-fetch width for network-membership scoping. A cross-network peer can sit
+	 * NEARER the key than a legitimate same-network peer and displace it from the
+	 * nearest-`clusterSize` window, so when scoping is active we ask FRET for a wider
+	 * proximity band and then keep the nearest peers that actually serve this network.
+	 * (A ring polluted by more cross-network peers than this band is the domain of the
+	 * separate FRET-side eviction follow-up; this band covers realistic co-location.)
+	 */
+	private membershipOverfetch(): number {
+		return Math.max(this.clusterSize * 4, this.clusterSize + 16)
+	}
+
+	/**
+	 * Classify a peer's network membership from its advertised protocols. Self always
+	 * `serves` (it trivially serves its own network). When no `protocolPrefix` is
+	 * configured the filter is disabled and EVERY peer is reported `serves`, so all
+	 * callers behave exactly as before this scoping was added.
+	 */
+	private membershipOf(idStr: string, protocols: string[] | undefined): NetworkMembership {
+		if (this.protocolPrefix == null) return 'serves'
+		if (idStr === this.libp2p.peerId.toString()) return 'serves'
+		if (protocols == null || protocols.length === 0) return 'unknown'
+		if (protocols.includes(`${this.protocolPrefix}/cluster/1.0.0`)
+			|| protocols.includes(`${this.protocolPrefix}/repo/1.0.0`)) return 'serves'
+		return 'foreign'
+	}
+
+	/**
+	 * Scope a reputation-ordered candidate id list to this network for COORDINATOR
+	 * selection: keep ONLY peers confirmed to serve this network (`serves`, which always
+	 * includes self), dropping both `foreign` peers (serving another network) and
+	 * `unknown` peers (peerStore protocol list empty — not yet confirmed). Incoming
+	 * (reputation) order is preserved among the surviving `serves` peers. A no-op
+	 * (returns the input unchanged, no drops) when `protocolPrefix` is unset or the list
+	 * is empty — the membership-disabled path is therefore untouched.
+	 *
+	 * `droppedUnconfirmed` reports whether any candidate was excluded because it was not
+	 * confirmed to serve this network — `foreign` OR `unknown` under scoping — so the
+	 * caller can surface a distinct "no network coordinator" failure rather than a generic
+	 * one. An `unknown` peer is not gambled on as coordinator: a permanent cross-network
+	 * contaminant and a fresh same-network peer mid-identify are indistinguishable at an
+	 * instant, but the filter re-reads the peerStore on every retry attempt, so a genuine
+	 * same-network peer that completes `identify` within the retry window flips to `serves`
+	 * and is selected normally on that attempt.
+	 */
+	private async filterByMembership(ids: string[]): Promise<{ ranked: string[]; droppedUnconfirmed: boolean }> {
+		if (this.protocolPrefix == null || ids.length === 0) return { ranked: ids, droppedUnconfirmed: false }
+		const selfStr = this.libp2p.peerId.toString()
+		const protocolsByPeer = await this.getPeerStoreProtocolsByPeer(ids.filter(id => id !== selfStr))
+		const serves: string[] = []
+		let droppedUnconfirmed = false
+		for (const id of ids) {
+			const m = this.membershipOf(id, protocolsByPeer[id])
+			if (m === 'serves') serves.push(id)
+			else droppedUnconfirmed = true
+		}
+		return { ranked: serves, droppedUnconfirmed }
+	}
 }