@optimystic/db-p2p 0.24.1 → 0.25.0

package/src/libp2p-key-network.ts

@@ -1,12 +1,13 @@
-import type { AbortOptions, Connection, Libp2p, PeerId, Stream } from "@libp2p/interface";
+import type { AbortOptions, Libp2p, PeerId, Stream } from "@libp2p/interface";
 import { toString as u8ToString } from 'uint8arrays'
 import type { ClusterPeers, CoordinatorIntent, FindCoordinatorOptions, IKeyNetwork, IPeerNetwork } from "@optimystic/db-core";
 import { peerIdFromString } from '@libp2p/peer-id'
 import type { FretService, SerializedTable } from 'p2p-fret'
 import { hashKey } from 'p2p-fret'
 import { createLogger, verbose } from './logger.js'
-import { mergePeerAddresses, validMultiaddrStrings } from './peer-address-book.js'
+import { classifySelfDialability, mergePeerAddresses, publishableConnectionAddr, unionPublishableAddrs, type AddressLog } from './peer-address-book.js'
 import type { IPeerReputation } from './reputation/types.js'
+import { openProtocolStream } from './network/open-protocol-stream.js'
 
 interface WithFretService { services?: { fret?: FretService } }
 
@@ -70,6 +71,39 @@ export class FindCoordinatorError extends Error {
 	}
 }
 
+/**
+ * `.code` on {@link SelfRelayOnlyAddressesError}. A stable string so it survives the
+ * `ClusterErrorEnvelope` round trip (`toClusterErrorEnvelope` carries `name` and `code`) and shows
+ * up as `code=SELF_RELAY_ONLY_ADDRESSES` on `ProtocolClient`'s `dial:fail` line instead of `none`.
+ */
+export const SELF_RELAY_ONLY_ERROR_CODE = 'SELF_RELAY_ONLY_ADDRESSES';
+
+/**
+ * Thrown by {@link Libp2pKeyPeerNetwork.connect} when we hold addresses for a peer but EVERY one
+ * of them reaches it by relaying through this node.
+ *
+ * This is the steady state a relay reaches for its own reservation holders: the address such a
+ * client advertises is `/<our transport addr>/p2p/<our peer id>/p2p-circuit`, which is correct and
+ * useful to every node except us. Dialing it asks us to relay to the client through ourselves, so
+ * it can only fail — with an error text (`NoValidAddressesError`, or an `AggregateError` of
+ * `Can not dial self`, depending on whether the circuit transport is registered as a dialer here)
+ * that is indistinguishable from "nobody ever taught us an address". Retrying cannot help: once the
+ * client's connection drops, only the client can re-initiate. So we fail fast and distinctly,
+ * letting the caller's existing exclude-and-continue logic move to another cohort member instead of
+ * burning a dial timeout.
+ */
+export class SelfRelayOnlyAddressesError extends Error {
+	readonly code = SELF_RELAY_ONLY_ERROR_CODE;
+	constructor(peer: string, protocol: string, addrCount: number) {
+		super(
+			`Peer ${peer} is reachable only through a circuit on THIS node: all ${addrCount} address(es) ` +
+			`we hold route back through us, so no dial for ${protocol} can succeed. ` +
+			`Only that peer can re-establish the connection.`
+		);
+		this.name = 'SelfRelayOnlyAddressesError';
+	}
+}
+
 /** Snapshot format this build writes and is willing to read. Bump on any incompatible change. */
 export const PERSISTED_STATE_VERSION = 2 as const;
 
@@ -248,6 +282,12 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 	private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
 	private static readonly MAX_CACHE_ENTRIES = 1000
 	private readonly log: ReturnType<typeof createLogger>
+	/**
+	 * This instance's logger, in the shape `peer-address-book.ts` accepts. Declared once so the
+	 * five address predicates that take a sink all report under the same peer-id-suffixed
+	 * namespace, rather than five separately-written adapters drifting apart.
+	 */
+	private readonly addressLog: AddressLog = (fmt, ...args) => this.log(fmt, ...args)
 
 	private toCacheKey(key: Uint8Array): string { return u8ToString(key, 'base64url') }
 
@@ -390,6 +430,15 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 	 * nor banned by reputation. Shared by all three places `findCoordinator` narrows a candidate
 	 * list — the FRET tier, the connected-peer fallback, and the retry-futility input — so the
 	 * futility test can never disagree with the tiers about who is pickable.
+	 *
+	 * NOTE: eligibility here is deliberately blind to self-dialability, so on a relay
+	 * `findCoordinator` can still pick one of our own reservation holders and only then fail fast
+	 * with `SelfRelayOnlyAddressesError`. The caller's exclude-and-continue walk
+	 * (`db-core/utility/batch-coordinator.ts`) absorbs it, and each such pick now costs an
+	 * instant refusal rather than a burned dial timeout, so it is a selection round-trip, not a
+	 * stall — and the verdict is a live peerStore read that a stale eligibility filter would have
+	 * to guess at. If a relay serving many reservation holders is ever measured spending real time
+	 * walking through them, feed the verdict into the tiers instead of discovering it at dial.
 	 */
 	private isSelectable(id: string, excluded: Set<string>): boolean {
 		return !excluded.has(id) && !(this.reputation?.isBanned(id));
@@ -532,7 +581,7 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 	 * protocol client dials through.
 	 */
 	public recordPeerAddresses(peerId: PeerId, multiaddrs: string[]): void {
-		mergePeerAddresses(this.libp2p, peerId, multiaddrs, (fmt, ...args) => this.log(fmt, ...args))
+		mergePeerAddresses(this.libp2p, peerId, multiaddrs, this.addressLog)
 	}
 
 	private getCachedCoordinator(key: Uint8Array): PeerId | undefined {
@@ -544,49 +593,60 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 	}
 
 	/**
-	 * True for a circuit-relay ("limited") connection. libp2p stamps a relayed
-	 * connection with `limits` (per-circuit data/duration caps); we additionally
-	 * sniff the multiaddr for `/p2p-circuit` as a fallback for transports/versions
-	 * that don't populate `limits`.
+	 * Open a stream to `peerId` on `protocol` — reusing a live connection when we hold one, and
+	 * otherwise dialing.
+	 *
+	 * Connection selection (prefer a direct connection over a resettable circuit-relay one, skip
+	 * entries libp2p has not yet evicted, opt in to limited connections) lives in
+	 * {@link openProtocolStream}, the single place in this package that opens a protocol stream.
+	 *
+	 * `negotiateFully: false` is safe here and saves a round trip: this is request/response and the
+	 * caller always reads a reply, so an unsupported-protocol failure deferred to the first read is
+	 * still observed. The caller's `AbortSignal` is forwarded so a per-peer dial deadline (enforced
+	 * upstream by `ProtocolClient.processMessage`) can actually cancel a stuck dial — without it,
+	 * libp2p falls back to its built-in connection-manager `dialTimeout` and the caller's tighter
+	 * deadline is decorative.
+	 *
+	 * The cold path pays one `peerStore.get` before dialing, to separate two failures libp2p
+	 * reports identically: "nobody ever taught us an address" and "every address we hold routes
+	 * back through us" (see {@link SelfRelayOnlyAddressesError}). Only the second is diagnosed
+	 * here; the first still dials, so an unknown peer produces libp2p's own `NoValidAddressesError`
+	 * exactly as before. It rides `beforeDial`, which never runs on the reuse path — the warm path
+	 * is deliberately kept clear of that read, since a live connection is the case this method
+	 * exists to make cheap.
 	 */
-	private isLimitedConnection(c: Connection): boolean {
-		if ((c as { limits?: unknown }).limits != null) return true
-		const addr = c.remoteAddr?.toString?.()
-		return addr != null && addr.includes('/p2p-circuit')
+	async connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
+		return await openProtocolStream(this.libp2p, peerId, protocol, {
+			signal: options?.signal,
+			negotiateFully: false,
+			beforeDial: () => this.assertNotSelfRelayOnly(peerId, protocol, options)
+		})
 	}
 
-	connect(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<Stream> {
-		const conns = this.libp2p.getConnections?.(peerId) ?? []
-		// Filter to only-open connections so a closing/closed entry that libp2p
-		// hasn't yet evicted from its index doesn't get picked up here.
-		const open = conns.filter(c => c?.status === 'open' && typeof c?.newStream === 'function')
-		// Prefer a DIRECT connection over a limited (circuit-relay) one for the RPC.
-		// A relayed/limited connection can be reset by the relay once a per-circuit
-		// cap or reservation lapses (@libp2p/circuit-relay-v2), surfacing to the
-		// coordinator as a StreamResetError that fails consensus. After DCUtR upgrades
-		// a relayed link to direct, both connections briefly coexist — picking the
-		// direct one avoids riding the soon-to-be-reset circuit. We only fall back to
-		// the limited connection (with runOnLimitedConnection) when it is the only open
-		// path — the steady state for browsers and NATed peers before any upgrade.
-		const chosen = open.find(c => !this.isLimitedConnection(c)) ?? open[0]
-		if (chosen) {
-			// runOnLimitedConnection: true is required to open a stream over a
-			// circuit-relay (limited) connection — the steady-state path for
-			// browsers and NATed peers. Without it, the warm relay connection
-			// from a prior dialProtocol cannot be reused on subsequent RPCs. It is
-			// a harmless no-op on the preferred direct connection.
-			return chosen.newStream([protocol], {
-				signal: options?.signal,
-				runOnLimitedConnection: true,
-				negotiateFully: false
-			})
-		}
-		// Forward the caller's AbortSignal so a per-peer dial deadline (enforced
-		// upstream by ProtocolClient.processMessage) can actually cancel a stuck
-		// dial — without this, libp2p falls back to its built-in dial timeout
-		// (default ~30s) and the caller's tighter deadline is decorative.
-		const dialOptions = { runOnLimitedConnection: true, negotiateFully: false, signal: options?.signal } as const
-		return this.libp2p.dialProtocol(peerId, [protocol], dialOptions)
+	/**
+	 * Throw {@link SelfRelayOnlyAddressesError} when every address we hold for `peerId` routes
+	 * through us, so the caller skips a dial that cannot succeed.
+	 *
+	 * Holding NOTHING is left alone on purpose — that dial still happens and still fails with
+	 * libp2p's `NoValidAddressesError`, because "we were never told an address" is a genuinely
+	 * different condition with a genuinely different remedy (someone teaches us one; see
+	 * `recordPeerAddresses`).
+	 */
+	private async assertNotSelfRelayOnly(peerId: PeerId, protocol: string, options?: AbortOptions): Promise<void> {
+		// NOTE: this costs one `peerStore.get` on EVERY cold dial, not only on relays, and libp2p's
+		// own dial queue reads the same record moments later — so a cold dial pays the peerStore
+		// twice. Unmeasured, and negligible against a dial's own cost; if cold-dial latency or
+		// peerStore contention ever shows up in a profile, hoist the verdict into the dial path
+		// rather than reading ahead of it.
+		const idStr = peerId.toString()
+		const held = (await this.getPeerStoreAddrsByPeer([idStr]))[idStr] ?? []
+		// A caller that cancelled while we were reading the peerStore is owed ITS reason, not a
+		// verdict we computed from a snapshot it no longer cares about.
+		options?.signal?.throwIfAborted()
+		if (classifySelfDialability(held, this.libp2p.peerId.toString(), this.addressLog) !== 'self-relay-only') return
+		this.log('connect:self-relay-only peer=%s protocol=%s addrs=%d',
+			idStr.substring(0, 12), protocol, held.length)
+		throw new SelfRelayOnlyAddressesError(idStr, protocol, held.length)
 	}
 
 	private getFret(): FretService {
@@ -836,21 +896,27 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 		);
 	}
 
+	/**
+	 * Connection-derived addresses for the cluster record, keyed by peer id.
+	 *
+	 * Only OUTBOUND connections contribute — see {@link publishableConnectionAddr} for why an
+	 * inbound connection's `remoteAddr` is an ephemeral source socket that no third party can
+	 * reach. This is the only place `findCluster` derives addresses from connections, so the
+	 * rule has exactly one site.
+	 */
 	private getConnectedAddrsByPeer(): Record<string, string[]> {
 		const conns = this.libp2p.getConnections()
 		const byPeer: Record<string, string[]> = {}
 		for (const c of conns) {
+			const addr = publishableConnectionAddr(c, this.addressLog)
+			if (addr === undefined) continue
 			const id = c.remotePeer.toString()
-			const addr = c.remoteAddr?.toString?.()
-			if (addr) (byPeer[id] ??= []).push(addr)
+			const forPeer = byPeer[id] ??= []
+			forPeer.push(addr)
 		}
 		return byPeer
 	}
 
-	private parseMultiaddrs(addrs: string[]): string[] {
-		return validMultiaddrStrings(addrs, (fmt, ...args) => this.log(fmt, ...args))
-	}
-
 	async findCluster(key: Uint8Array): Promise<ClusterPeers> {
 		const t0 = Date.now();
 		const fret = this.getFret()
@@ -947,6 +1013,13 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 		// reported symptom: clean membership logs on every node while every dial to the
 		// addressless member died instantly and the write never completed.
 		const addressless: string[] = []
+		// The OTHER way a member can be undialable by us: we hold addresses, but every one of
+		// them is a circuit through this node — the steady state for our own reservation holders.
+		// Counted separately because the two have different remedies (be taught an address, vs.
+		// wait for the client to re-dial us) and libp2p's dial error cannot tell them apart.
+		// These addresses are still PUBLISHED: a cohort sibling reaching the member through our
+		// relay is the working path, and dropping them would break it.
+		const selfRelayOnly: string[] = []
 
 		for (const idStr of ids) {
 			if (idStr === selfId) {
@@ -956,12 +1029,13 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 			}
 			const connectedStrings = connectedByPeer[idStr] ?? []
 			const peerStoreStrings = peerStoreAddrs[idStr] ?? []
-			// De-duplicate while preserving connected-first ordering. The
-			// connected multiaddr is the one libp2p just used to reach this peer
-			// and is the most reliable; peerStore addrs are the fallback for
-			// cohort members we know-of but aren't currently connected to.
-			const merged = Array.from(new Set([...connectedStrings, ...peerStoreStrings]))
-			const parsed = this.parseMultiaddrs(merged)
+			// The union rule — connected-first, de-duplicated, validated — lives in
+			// `unionPublishableAddrs` so a cluster record and a redirect payload cannot describe
+			// the same peer differently. The sync form is used here (rather than
+			// `publishableAddrsForPeer`) because both halves are already in hand: the peerStore
+			// read above is batched across the whole cohort, and on the membership-scoped path it
+			// is the same `store.get` that fetched protocols.
+			const parsed = unionPublishableAddrs(connectedStrings, peerStoreStrings, this.addressLog)
 			const remotePeerId = peerIdFromString(idStr)
 			const raw = remotePeerId.publicKey?.raw ?? new Uint8Array()
 			// Note: parsed may be empty for a cohort member we have neither a
@@ -970,7 +1044,9 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 			// retry/exclude logic takes over — we intentionally do NOT drop
 			// addressless members here, because shrinking the cohort below
 			// `clusterSize` puts consensus supermajority out of reach.
-			if (parsed.length === 0) addressless.push(idStr.substring(0, 12))
+			const dialability = classifySelfDialability(parsed, selfId, this.addressLog)
+			if (dialability === 'none') addressless.push(idStr.substring(0, 12))
+			else if (dialability === 'self-relay-only') selfRelayOnly.push(idStr.substring(0, 12))
 			peers[idStr] = { multiaddrs: parsed, publicKey: u8ToString(raw, 'base64url') }
 		}
 
@@ -983,8 +1059,16 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 				keyStr, addressless.length, Object.keys(peers).length, addressless)
 		}
 
-		this.log('findCluster:done key=%s ms=%d peers=%d addressless=%d',
-			keyStr, Date.now() - t0, Object.keys(peers).length, addressless.length)
+		// Same reasoning, second condition: a non-zero count here means we DO hold addresses for
+		// these members and still cannot dial them, because the only route we know runs through
+		// our own relay. No amount of retrying changes that (see `SelfRelayOnlyAddressesError`).
+		if (selfRelayOnly.length > 0) {
+			this.log('findCluster:self-relay-only-members key=%s count=%d of=%d peers=%o',
+				keyStr, selfRelayOnly.length, Object.keys(peers).length, selfRelayOnly)
+		}
+
+		this.log('findCluster:done key=%s ms=%d peers=%d addressless=%d selfRelayOnly=%d',
+			keyStr, Date.now() - t0, Object.keys(peers).length, addressless.length, selfRelayOnly.length)
 		return peers
 	}