@optimystic/db-p2p 0.24.2 → 0.25.0

package/src/libp2p-node-base.ts

@@ -5,7 +5,6 @@ import { identify, identifyPush } from '@libp2p/identify';
 import { ping } from '@libp2p/ping';
 import { dcutr } from '@libp2p/dcutr';
 import { autoNAT } from '@libp2p/autonat';
-import { gossipsub } from '@chainsafe/libp2p-gossipsub';
 import { bootstrap } from '@libp2p/bootstrap';
 import { circuitRelayServer, type CircuitRelayServerInit } from '@libp2p/circuit-relay-v2';
 import { peerIdFromString } from '@libp2p/peer-id';
@@ -14,10 +13,13 @@ import type { ConnectionGater, PrivateKey } from '@libp2p/interface';
 import { clusterService } from './cluster/service.js';
 import { blockTransferService } from './cluster/block-transfer-service.js';
 import { repoService } from './repo/service.js';
-import { StorageRepo, withBlockCommitLatch } from './storage/storage-repo.js';
+import { StorageRepo } from './storage/storage-repo.js';
 import { BlockStorage } from './storage/block-storage.js';
 import { MemoryRawStorage } from './storage/memory-storage.js';
+import { withReadCache, type ResolvedReadCache } from './storage/with-read-cache.js';
 import type { IRawStorage } from './storage/i-raw-storage.js';
+import { latestClaimFromArchive, servableProof, type ArchiveServingRepo } from './storage/block-archive.js';
+import { createServedRepoProxy } from './repo/served-repo-proxy.js';
 import { seedOwnedBlocksFromStorage } from './owned-block-seed.js';
 import { clusterMember, type ReconcileBlockCallback, type CommitCertificateSink, type DeriveExpectedClusterCallback } from './cluster/cluster-repo.js';
 import { createReconcileBlock } from './cluster/reconcile-block.js';
@@ -26,7 +28,7 @@ import { assertClusterSizeCoupling } from './cluster/cluster-size-coupling.js';
 import { createCommitCertStore, makeClusterCommitCertExtractor, type CommitCertStore } from './cluster/commit-cert.js';
 import { coordinatorRepo } from './repo/coordinator-repo.js';
 import { Libp2pKeyPeerNetwork, type NetworkMode, type NetworkStatePersistence } from './libp2p-key-network.js';
-import { mergePeerAddresses, publishableConnectionAddr, type AddressLog } from './peer-address-book.js';
+import { mergePeerAddresses, publishableAddrsForPeer, type AddressLog } from './peer-address-book.js';
 import type { OptimysticNode, OptimysticNodeAttachments } from './optimystic-node.js';
 import { ClusterClient } from './cluster/client.js';
 import type { IRepo, ICluster, ITransactionValidator, BlockId, IBlockChangeNotifier } from '@optimystic/db-core';
@@ -135,7 +137,18 @@ const reactivityWiringLog = createLogger('reactivity-node-wiring');
  */
 const wiringLog = createLogger('node-wiring');
 
-/** Factory function or instance for creating raw storage */
+/**
+ * Factory function or instance for creating raw storage. The node puts the write-through read
+ * cache in front of the resolved instance (`withReadCache`, unless it is a `MemoryRawStorage` or
+ * already cached) under a lease, and releases THAT LEASE when it stops; the instance you supplied
+ * is never disposed. The cache is shared per backing store: two concurrently running nodes handed
+ * one uncached instance — or two instances that report the same `getStoreIdentity()`, such as two
+ * `FileRawStorage` over one directory — read and write through ONE cache, which is cleared and
+ * unregistered only when the last of them stops. Sequential reuse (a restart over the same
+ * instance) starts cold once the previous node's lease has released. What remains unguarded is
+ * the cross-process case (Invariant 5 in `packages/db-p2p/docs/storage.md`). A host that builds
+ * its own `CachedRawStorage` and hands it in keeps owning it; the node never releases it.
+ */
 export type RawStorageProvider = IRawStorage | (() => IRawStorage);
 
 /**
@@ -180,7 +193,7 @@ export type NodeOptions = ClusterPolicyOptions & {
 	 * `{ reservations: { applyDefaultLimit: false } }` to lift the cap.
 	 */
 	relayServerInit?: CircuitRelayServerInit;
-	/** Storage provider - either an IRawStorage instance or a factory function. Defaults to MemoryRawStorage if not provided. */
+	/** Storage provider - either an IRawStorage instance or a factory function. Defaults to MemoryRawStorage if not provided. See {@link RawStorageProvider} for the ownership rule. */
 	storage?: RawStorageProvider;
 	/** Override libp2p listen multiaddrs. */
 	listenAddrs?: string[];
@@ -225,11 +238,15 @@ export type NodeOptions = ClusterPolicyOptions & {
 
 	/**
 	 * Rebalance reaction tuning. Drives the RebalanceMonitor + BlockTransferCoordinator pull-gained/
-	 * push-lost path when arachnode/FRET are available (the only place fretAdapter + restoration
-	 * coordinator exist). Absent -> enabled with defaults (see RebalanceMonitorConfig). Set
-	 * { enabled: false } to disable the rebalance reaction on this node. When arachnode is disabled
-	 * or FRET is absent the rebalance path stays inert regardless of this flag (rebalance is a
-	 * resilience optimization, not a correctness requirement).
+	 * push-lost/replicate-grown path when arachnode/FRET are available (the only place fretAdapter +
+	 * restoration coordinator exist). The grown arm is what pushes a block this node keeps to peers
+	 * that newly became co-responsible for it — the founder case: anything committed while the
+	 * deployment was one node gets its second copy only through this path (bounded per pass by
+	 * `growthBlockBudget`). Absent -> enabled with defaults (see RebalanceMonitorConfig). Set
+	 * { enabled: false } to disable the rebalance reaction (including the grown arm) on this node.
+	 * When arachnode is disabled or FRET is absent the rebalance path stays inert regardless of this
+	 * flag (rebalance is a resilience optimization — except for singly-held blocks, where the grown
+	 * arm is currently the only mechanism that ever creates a second copy).
 	 */
 	rebalance?: Partial<RebalanceMonitorConfig> & { enabled?: boolean };
 
@@ -242,6 +259,21 @@ export type NodeOptions = ClusterPolicyOptions & {
 	/** Dispute protocol configuration */
 	dispute?: Partial<DisputeConfig>;
 
+	/**
+	 * Block-transfer (churn re-replication) receiver tuning.
+	 *
+	 * `requirePushCertificate` (default `true`) refuses a pushed block that carries no verifying
+	 * cohort commit proof — see `BlockTransferServiceInit.requirePushCertificate` for why that is
+	 * the default. It is exposed here because the migration it exists for is a DEPLOYMENT decision
+	 * and the escape hatch lives on the RECEIVER: a node still holding pre-proof blocks cannot
+	 * certify them (the signatures no longer exist), so under the strict default those blocks never
+	 * gain a new holder — which also means `BlockTransferCoordinator.confirmReplicated` never
+	 * confirms them, so a rebalance never releases them and a ring shift's confirm phase aborts on
+	 * them (`docs/arachnode-ring-handoff.md` § Phase B). Clearing the backlog means running the
+	 * receivers with this `false` until every such block has been rewritten under current code.
+	 */
+	blockTransfer?: { requirePushCertificate?: boolean };
+
 	/** Optional persistent store for 2PC transaction state (enables crash recovery) */
 	transactionStateStore?: ITransactionStateStore;
 
@@ -327,11 +359,30 @@ export type NodeOptions = ClusterPolicyOptions & {
 	connectionGater?: ConnectionGater;
 };
 
-function resolveStorage(provider: RawStorageProvider | undefined): IRawStorage {
+/**
+ * Resolve the node's raw storage and put the write-through read cache in front of it. This is
+ * the single place the network node resolves its `IRawStorage`, so it is the single place the
+ * cache is wired (`withReadCache` states the exclusions: memory storage and already-cached
+ * storage pass through unchanged). The default is a bare `MemoryRawStorage`, deliberately not
+ * routed through the helper — nothing to cache.
+ *
+ * `lease` is this node's claim on the (possibly shared) cache, and the ONLY thing its stop path
+ * may release — a host that supplied its own `CachedRawStorage` keeps owning it (see
+ * {@link ResolvedReadCache}).
+ */
+function resolveStorage(provider: RawStorageProvider | undefined, networkName: string): ResolvedReadCache {
 	if (!provider) {
-		return new MemoryRawStorage();
+		return { storage: new MemoryRawStorage(), lease: undefined };
 	}
-	return typeof provider === 'function' ? provider() : provider;
+	const storage = typeof provider === 'function' ? provider() : provider;
+	// NOTE: the label is the network name, so N nodes on one network in one process over N DISTINCT
+	// stores produce N identically-labelled rows in `SharedCachePool.stats()` (the pool keys on a
+	// monotonic store id, not the label); N nodes over ONE store share a single row, labelled by
+	// whichever node wrapped first. Harmless while the label is only read by a human eyeballing
+	// occupancy; if pool stats ever need to attribute bytes to a SPECIFIC node, fold the peer id
+	// in — it is not known here, so that would mean labelling after node construction rather than
+	// at resolve time.
+	return withReadCache(storage, `node:${networkName}`);
 }
 
 /**
@@ -374,7 +425,7 @@ export async function createLibp2pNodeBase(
 		transports: Libp2pTransports;
 	}
 ): Promise<OptimysticNode> {
-	const rawStorage = resolveStorage(options.storage);
+	const { storage: rawStorage, lease } = resolveStorage(options.storage, options.networkName);
 
 	// Create placeholder restore callback (will be replaced after node starts)
 	let restoreCallback: RestoreCallback = async (_blockId, _rev?) => {
@@ -386,15 +437,6 @@ export async function createLibp2pNodeBase(
 		new BlockStorage(blockId, rawStorage, restoreCallback)
 	);
 
-	// Per-block commit-latch runner, ready to thread into the invalidation-apply sink (`onInvalidate`)
-	// passed to `clusterMember(...)` and into each cascade `CollectionEnv`, the instant either is wired
-	// here. Sharing the `StorageRepo.commit:<blockId>` latch makes a compensating saveReplica/saveDeletion
-	// RMW of `meta.latest` mutually exclusive with a concurrent commit on the same block. It is unused
-	// today only because no `onInvalidate`/cascade driver is wired in the live node (see review handoff);
-	// it is bound here so that wiring is a one-liner and cannot reach for a divergent latch key.
-	const blockCommitLatch = withBlockCommitLatch;
-	void blockCommitLatch;
-
 	let clusterImpl: ICluster | undefined;
 	let coordinatedRepo: IRepo | undefined;
 	// The running node, bound immediately after `createLibp2p` below. Service factories that need
@@ -415,24 +457,11 @@ export async function createLibp2pNodeBase(
 		}
 	};
 
-	const repoProxy: IRepo = {
-		async get(blockGets, options) {
-			const target = coordinatedRepo ?? storageRepo;
-			return await target.get(blockGets, options);
-		},
-		async pend(request, options) {
-			const target = coordinatedRepo ?? storageRepo;
-			return await target.pend(request, options);
-		},
-		async cancel(trxRef, options) {
-			const target = coordinatedRepo ?? storageRepo;
-			return await target.cancel(trxRef, options);
-		},
-		async commit(request, options) {
-			const target = coordinatedRepo ?? storageRepo;
-			return await target.commit(request, options);
-		}
-	};
+	// Built by a named factory, not an inline literal: see `createServedRepoProxy` for why (an
+	// inline object here is unreachable from every test that does not boot a libp2p node, which is
+	// how it served every repair archive without its commit proof). `coordinatedRepo` is read per
+	// call because it is assigned further down, after the cluster is assembled.
+	const repoProxy: ArchiveServingRepo = createServedRepoProxy(storageRepo, () => coordinatedRepo);
 
 	// The ONE authorization slice, spread verbatim into all four database-protocol service inits
 	// below. Building it once (rather than repeating two option reads per service) is what makes
@@ -547,10 +576,17 @@ export async function createLibp2pNodeBase(
 			// is acceptable — they neither throw nor break the build in that case.
 			dcutr: dcutr(),
 			autoNAT: autoNAT(),
-			pubsub: gossipsub({
-				allowPublishToZeroTopicPeers: true,
-				heartbeatInterval: 7000
-			}),
+			// NOTE: no `pubsub` service. `@chainsafe/libp2p-gossipsub` (currently at 14.1.2, the newest
+			// published version) is built against `@libp2p/interface@^2` — its `Stream` still expects
+			// `.sink`/`.source`. Under libp2p 3 `Stream extends MessageStream` (event target +
+			// AsyncIterable), which has neither, so `it-pipe`'s `isDuplex` check on the outbound stream
+			// fails and `OutboundStream`'s constructor throws `fns.shift(...) is not a function`
+			// synchronously — before gossipsub's own `.catch` can attach. Gossipsub logs and swallows
+			// it, so the service reports as running while never sending a subscription to any peer:
+			// `publish` always resolves with zero recipients. Verified end-to-end against this repo's
+			// installed dependency tree (gotchoices/Optimystic#9). Nothing in this repo uses `pubsub`
+			// today — re-add only against a gossipsub release built for libp2p 3.
+
 			// Circuit relay server - enables this node to relay connections for other peers
 			...(options.relay ? { relay: circuitRelayServer(options.relayServerInit) } : {}),
 
@@ -571,17 +607,13 @@ export async function createLibp2pNodeBase(
 					peerId: components.peerId,
 					// Fallback addr resolver for redirect targets whose multiaddrs are not
 					// already embedded in record.peers. A redirect payload is handed to a THIRD
-					// party, so it obeys the same rule the cluster record does — see
-					// `publishableConnectionAddr`: only an outbound connection's remoteAddr is an
-					// address anyone else can reach.
-					getConnectionAddrs: (peerId: any) => {
-						const conns = liveNode?.getConnections?.(peerId) ?? [];
-						const addrs: string[] = [];
-						for (const c of conns) {
-							const addr = publishableConnectionAddr(c, addressLog);
-							if (addr !== undefined) addrs.push(addr);
-						}
-						return addrs;
+					// party, so it answers with `publishableAddrsForPeer` — the same rule, and the
+					// same function, `findCluster` uses to fill a cluster record: the publishable
+					// half of our live connections plus the peer's own advertised addresses.
+					getConnectionAddrs: async (peerId: any) => {
+						if (!liveNode) return [];
+						const conns = liveNode.getConnections?.(peerId) ?? [];
+						return await publishableAddrsForPeer(liveNode, conns, peerId, addressLog);
 					},
 					// Inbound cluster records carry each cohort member's multiaddrs. libp2p only
 					// propagates addresses between directly-connected peers, so for a cohort chosen
@@ -635,11 +667,20 @@ export async function createLibp2pNodeBase(
 			blockTransfer: (components: any) => {
 				const serviceFactory = blockTransferService({
 					protocolPrefix: `/optimystic/${options.networkName}`,
+					// Absent → the strict default inside the service; see `blockTransfer` on the options.
+					...(options.blockTransfer?.requirePushCertificate !== undefined
+						? { requirePushCertificate: options.blockTransfer.requirePushCertificate }
+						: {}),
 					...inboundAuthorization
 				});
 				return serviceFactory({
 					registrar: components.registrar,
 					repo: storageRepo,
+					// Verifying a pushed block's commit proof needs the cohort's super-majority fraction.
+					// Read from the SAME resolved `consensusConfig` the member and coordinator read (whose
+					// coupling `assertSuperMajorityCoupling` below already asserts) — a third copy resolving
+					// its own default would defeat that.
+					superMajorityThreshold: consensusConfig.superMajorityThreshold,
 					// So this service's authorization denials reach the same error sink as the other three.
 					logger: components.logger
 				});
@@ -694,6 +735,26 @@ export async function createLibp2pNodeBase(
 	// soon as the protocol handler goes live in start().
 	liveNode = node;
 
+	// Release this node's lease on the raw-storage read cache when the node stops. Installed FIRST
+	// — before start() and before every other stop wrapper — so it runs LAST in the wrapper chain,
+	// after every monitor and service that may still read storage during its own stop. Only wired
+	// when the seam handed out a lease: a MemoryRawStorage passes through unwrapped, and a
+	// host-supplied `CachedRawStorage` stays the host's to dispose. Releasing a lease is safe even
+	// when other nodes share the cache — it is cleared and unregistered only when the last lease
+	// goes, so stopping one node never blinds the others. A skipped release leaks only cold
+	// entries the pool evicts under pressure; the point of the polite release is honest pool
+	// occupancy on a long-lived provider node.
+	if (lease) {
+		const previousStop = node.stop.bind(node);
+		node.stop = async () => {
+			try {
+				await previousStop();
+			} finally {
+				await lease.release();
+			}
+		};
+	}
+
 	// Inject the REAL libp2p node into the services that need it, before start(). These are
 	// load-bearing and the node has NOT started yet, so any throw fails fast and rejects node
 	// creation (nothing started leaks) — far better than the service silently falling back to the
@@ -777,10 +838,21 @@ export async function createLibp2pNodeBase(
 		const reconcileBlock: ReconcileBlockCallback = createReconcileBlock({
 			selfPeerId: node.peerId.toString(),
 			fetchArchive: fetchArchiveFromPeer,
-			saveReplicatedBlock: (blockId, block, source) => storageRepo.saveReplicatedBlock(blockId, block, source),
+			// The 4th parameter is a proof reconcile verified against these exact bytes; it is
+			// persisted so the repaired replica serves it onward (see StorageRepo.saveReplicatedBlock).
+			saveReplicatedBlock: (blockId, block, source, verifiedProof) =>
+				storageRepo.saveReplicatedBlock(blockId, block, source, verifiedProof),
 			simpleMajorityThreshold: consensusConfig.simpleMajorityThreshold,
+			superMajorityThreshold: consensusConfig.superMajorityThreshold,
 			repairCorroborationClusterSize: consensusConfig.repairCorroborationClusterSize,
 			reputation
+			// `anchoring` (proof layer-2, `ProofAnchoring`) is intentionally NOT wired here yet — nor
+			// is the coordinator's `proofAnchoring` below: a real implementation re-derives the
+			// block's cohort from `keyNetwork.findCluster` (the same source `deriveExpectedCluster`
+			// uses) and needs a churn-tolerance window so historic cohort rotation does not read as an
+			// anomaly flood. Until then certification runs on layer-1 cryptography and LOGS the
+			// unanchored residual. See `feat-cluster-membership-threshold-cert-anchoring`, and the
+			// matching `recomputeArbitratorSet` note at the clusterMember construction below.
 		});
 
 		// Member-side membership derivation for the admission gate: independently re-derive this block's
@@ -855,12 +927,18 @@ export async function createLibp2pNodeBase(
 		);
 
 		// Create callback for querying cluster peers for their latest block revision. Three-way
-		// contract (see ClusterLatestCallback): an ActionRev is the peer's claim, a resolved
+		// contract (see ClusterLatestCallback): a CertifiedActionRev is the peer's claim, a resolved
 		// `undefined` is the peer answering "I hold nothing", and a REJECTION is silence — the
 		// coordinator counts it as "did not answer" and refuses to report an authoritative absent
 		// over it. Transport errors must therefore propagate, not collapse into `undefined` (that
 		// collapse let a slow two-node cohort report a missing block as authoritatively absent —
 		// ticket cluster-read-consult-cannot-report-unreachable).
+		//
+		// The claim carries the cohort's commit proof for the claimed revision when the answering
+		// peer retained one. It is attached UNVERIFIED — a peer chooses what to send. Both repair
+		// paths verify it before weighing it (the shared layer in `cluster/certified-claims.ts`):
+		// the coordinator's certifyClaim pass in `queryClusterForLatest`, and reconcile's
+		// certification pass in `cluster/reconcile-block.ts`.
 		const clusterLatestCallback: ClusterLatestCallback = async (peerId, blockId, context?) => {
 			// Self-read short-circuit: dialling self via SyncClient is a round trip
 			// with no remote on the other end, and on nodes without listen addresses
@@ -871,7 +949,13 @@ export async function createLibp2pNodeBase(
 			if (peerId.equals(node.peerId)) {
 				try {
 					const result = await storageRepo.get({ blockIds: [blockId], context });
-					return result[blockId]?.state?.latest;
+					const latest = result[blockId]?.state?.latest;
+					if (!latest) return undefined;
+					// The SAME lookup a peer serving an archive uses, so a self answer and a remote
+					// answer attach proofs by one rule (including the mis-pairing guard). It never
+					// throws — a proof fault degrades to "no proof", never to a lost claim.
+					const proof = await servableProof(storageRepo, blockId, latest);
+					return proof ? { ...latest, proof } : latest;
 				} catch {
 					return undefined;
 				}
@@ -881,14 +965,11 @@ export async function createLibp2pNodeBase(
 			// per-peer deadline also bounds a hung request — slowness needs no race here.
 			const response = await syncClient.requestBlock({ blockId, rev: undefined });
 			if (response.success && response.archive) {
-				const revisions = Object.keys(response.archive.revisions).map(Number);
-				if (revisions.length > 0) {
-					const maxRev = Math.max(...revisions);
-					const revisionData = response.archive.revisions[maxRev];
-					if (revisionData?.action) {
-						return { actionId: revisionData.action.actionId, rev: maxRev };
-					}
-				}
+				// Projection lives with the archive shape (`latestClaimFromArchive`) rather than
+				// re-read inline here — it reads the claim and its proof out of ONE revision entry,
+				// so a serving peer cannot pair a proof with a revision it does not certify.
+				const claim = latestClaimFromArchive(response.archive);
+				if (claim) return claim;
 			}
 			// The peer DID answer, without data: `success:false` is the sync service's "Block not
 			// found in local storage", and an archive with no usable revisions holds nothing either
@@ -1065,7 +1146,11 @@ export async function createLibp2pNodeBase(
 				// Setup restoration coordinator with FRET adapter
 				const restorationCoordinatorV2 = new RestorationCoordinator(
 					fretAdapter,
-					{ connect: (pid, protocol) => node.dialProtocol(pid as Parameters<typeof node.dialProtocol>[0], [protocol]) },
+					// The node's own IPeerNetwork, not an inline `dialProtocol` lambda: the lambda opened
+					// streams without `runOnLimitedConnection`, so a block holder reachable only through a
+					// relay looked like a peer that simply did not have the block, and it dropped the
+					// caller's AbortSignal, so `SyncClient`'s per-peer dial deadline never bounded a dial.
+					keyNetwork,
 					`/optimystic/${options.networkName}`,
 					node.peerId.toString()
 				);
@@ -1135,6 +1220,18 @@ export async function createLibp2pNodeBase(
 						// async loop. Adding/deleting a Set entry during iteration does not throw in JS — entries
 						// are visited best-effort — which is acceptable for a resilience mechanism, so we
 						// document it here rather than add locking.
+						//
+						// The event's `grown` arm (blocks this node KEEPS whose cohort acquired new peers — the
+						// founder/cohort-growth case): handleRebalanceEvent pushes each grown block to the newly
+						// co-responsible peers and returns a per-block GrowthOutcome in `result.growth`, which is
+						// fed back into the monitor here. The monitor records a peer as seen ONLY off that
+						// feedback (a confirmed replica, or a block that otherwise reached its floor), so a failed
+						// push is re-detected on the next check instead of being silently dropped — bounded by
+						// growthMaxAttempts, after which the monitor abandons the peer for that block (visible in
+						// getGrowthDiagnostics). Nothing is released or untracked off the grown arm — the node
+						// keeps serving the block either way. On the catch path NOTHING is recorded: the monitor's
+						// state stays un-advanced and the next check retries, the correct outcome for a reaction
+						// that threw.
 						rebalanceMonitor.onRebalance((event) => {
 							for (const blockId of event.gained) ownedBlocks.add(blockId);
 							coordinator.handleRebalanceEvent(event).then((result) => {
@@ -1142,7 +1239,25 @@ export async function createLibp2pNodeBase(
 									rebalanceMonitor.untrackBlock(blockId); // also evicts from the shared ownedBlocks set
 									gcEligible.add(blockId);                 // confirmed replicated → safe to sweep
 								}
+								for (const [blockId, outcome] of result.growth) {
+									rebalanceMonitor.recordGrowthOutcome(blockId, outcome);
+								}
+								if (result.underReplicated.length > 0) {
+									const growthDiag = rebalanceMonitor.getGrowthDiagnostics();
+									log?.('cohort-growth: %d of %d grown blocks not confirmed on new peers this pass ' +
+										'(awaiting-confirmation=%d given-up-pairs=%d)',
+										result.underReplicated.length, event.grown.size,
+										growthDiag.blocksAwaitingConfirmation, growthDiag.abandonedPairs);
+								}
 							}).catch((err) => {
+								// NOTE: accepted tradeoff — recording nothing here means growthMaxAttempts (which
+								// only counts RECORDED incomplete outcomes) never bounds a reaction that throws
+								// every time, so the re-check timer retries it at growthRecheckIntervalMs forever,
+								// logging each failure. Kept deliberately: per-peer errors are already caught
+								// inside the coordinator, so a throw out of handleRebalanceEvent is a coding bug,
+								// and a loud unbounded retry is the right way to surface one — silently abandoning
+								// the block would hide it and leave the block singly held. Revisit if a legitimate
+								// recoverable condition is ever allowed to throw out of the reaction.
 								log?.('rebalance reaction failed: %o', err);
 							});
 						});
@@ -1503,13 +1618,13 @@ export async function createLibp2pNodeBase(
 
 			// Inbound notify frames → forwarder host (subscriber role delivers in-process; forwarder role fans out).
 			// NOTE: the four `register*Handler` helpers below (notify / pushStateGossip / recover /
-			// matchmaking query) all call `node.handle(...)` fire-and-forget (`void`), so a rejected
+			// matchmaking query) all call `registerProtocolHandler(...)` fire-and-forget (`void`), so a rejected
 			// registration escapes the post-start rollback `catch` as an UNHANDLED rejection instead of
 			// failing node creation. Harmless today — every protocol id here is a fixed constant registered
 			// exactly once, so the only realistic rejection is a duplicate, and that needs a caller to pass
 			// overlapping custom `cohortTopic.host.protocols`. If any of these ids ever becomes
 			// caller-configurable, or a helper grows a registration that can genuinely fail, make them await
-			// their `node.handle` so the failure reaches the rollback.
+			// their `registerProtocolHandler` so the failure reaches the rollback.
 			registerNotifyHandler(node, reactivityProtocols.notify, notify);
 			offInboundNotify = notify.onNotification((from, n): void => { void forwarderHost.onInbound(from, n); });
 

package/src/network/open-protocol-stream.ts

@@ -0,0 +1,87 @@
+/**
+ * The single place in `db-p2p` that opens a libp2p protocol stream.
+ *
+ * libp2p refuses to open a protocol stream over a *limited* (circuit-relay) connection unless the
+ * caller opts in with `runOnLimitedConnection: true`. Omitting it produces no compile error and no
+ * obvious failure — only "that peer never answers" — so every peer reachable only through a relay
+ * (the steady state for browsers, phones, and NATed peers) silently drops out. The flag is
+ * deliberately NOT an option here: a new call site gets relay support without its author needing to
+ * know the flag exists, and `test/dial-options-single-site.spec.ts` fails the build if a second
+ * source file calls `dialProtocol` / `newStream` directly.
+ *
+ * NOTE: accepted tradeoff — this duplicates FRET's `rpc/protocols.ts#openRpcStream` (exported since
+ * `p2p-fret@1.0.0-beta.3`) rather than delegating to it. FRET's helper pins `negotiateFully: false`
+ * with no way to opt out, and `cohort-topic/stream-util.ts` deliberately does not set that option
+ * (see the NOTE at its call sites); adopting FRET's would silently reverse a decision a human
+ * already made. It also has no hook for the pre-dial check `libp2p-key-network.ts#connect` runs.
+ * Revisit — and delete this module in favour of `openRpcStream` — if FRET ever parameterizes
+ * `negotiateFully`.
+ *
+ * Imports here are type-only on purpose: this module is reachable from the react-native entry
+ * (`src/rn.ts`) via `libp2p-key-network.ts`, which must not pull node-specific code.
+ */
+
+import type { Libp2p } from "libp2p";
+import type { Connection, PeerId, Stream } from "@libp2p/interface";
+
+export interface OpenProtocolStreamOptions {
+	/** Forwarded to libp2p so a caller's deadline cancels both the connection reuse and the dial. */
+	signal?: AbortSignal;
+	/**
+	 * Omit for libp2p's default (full multistream-select negotiation at stream-open).
+	 * Pass `false` to save the round trip, accepting that an unsupported-protocol failure is
+	 * deferred to the first read — only safe when the caller always reads a reply.
+	 */
+	negotiateFully?: boolean;
+	/**
+	 * Runs immediately before a FRESH dial, and never on the connection-reuse path.
+	 * Throwing aborts the open. This is the seam for checks that are only meaningful when no
+	 * connection exists yet (see `libp2p-key-network.ts#assertNotSelfRelayOnly`).
+	 */
+	beforeDial?: () => Promise<void> | void;
+}
+
+/**
+ * True for a circuit-relay ("limited") connection: libp2p stamps one with per-circuit `limits`
+ * (data/duration caps); sniffing `/p2p-circuit` in the remote multiaddr covers transports and
+ * versions that leave `limits` unpopulated.
+ */
+export function isLimitedConnection(c: Connection): boolean {
+	if (c.limits != null) return true;
+	return c.remoteAddr?.toString?.().includes("/p2p-circuit") ?? false;
+}
+
+/**
+ * Open `protocol` to `peer`, reusing a healthy existing connection when there is one.
+ *
+ * Skips connections libp2p has not yet evicted from its index but that are no longer open, and
+ * prefers a direct connection over a relayed one — a relayed connection can be reset once the
+ * relay's per-circuit cap or reservation lapses, and after DCUtR upgrades a link to direct both
+ * briefly coexist. Falls back to the relayed connection when it is the only open path.
+ */
+export async function openProtocolStream(
+	node: Libp2p,
+	peer: PeerId,
+	protocol: string,
+	options?: OpenProtocolStreamOptions,
+): Promise<Stream> {
+	// Before touching connections: a caller that has already given up is owed its own reason,
+	// not whatever libp2p would report several layers down.
+	options?.signal?.throwIfAborted();
+
+	const conns = node.getConnections?.(peer) ?? [];
+	const open = conns.filter(c => c?.status === "open" && typeof c?.newStream === "function");
+	const chosen = open.find(c => !isLimitedConnection(c)) ?? open[0];
+
+	// Keys are OMITTED rather than set to `undefined` when the caller did not supply them, so
+	// libp2p applies its own defaults instead of seeing an explicit `undefined`.
+	const streamOptions = {
+		runOnLimitedConnection: true,
+		...(options?.negotiateFully !== undefined ? { negotiateFully: options.negotiateFully } : {}),
+		...(options?.signal !== undefined ? { signal: options.signal } : {}),
+	};
+
+	if (chosen) return await chosen.newStream([protocol], streamOptions);
+	await options?.beforeDial?.();
+	return await node.dialProtocol(peer, [protocol], streamOptions);
+}

package/src/network/register-protocol-handler.ts

@@ -0,0 +1,85 @@
+/**
+ * The single place in `db-p2p` that registers a libp2p protocol handler.
+ *
+ * The mirror image of `./open-protocol-stream.ts`. Opening a stream over a *limited* (circuit-relay)
+ * connection needs BOTH sides to opt in, and libp2p checks each side against its own options:
+ *
+ * - the dialer's `newStream` / `dialProtocol` options (`libp2p/src/connection.ts`, outbound path), and
+ * - the options the *answering* peer passed when it registered the handler
+ *   (`libp2p/src/connection.ts`, incoming path — it reads back
+ *   `registrar.getHandler(protocol).options.runOnLimitedConnection`).
+ *
+ * Miss either one and libp2p throws `LimitedConnectionError`. On the answering side it does so
+ * *after* multistream-select has already acknowledged the protocol, so the dialer's stream open
+ * appears to succeed and the stream is then reset with no reply — indistinguishable, from the
+ * dialer's seat, from a peer that simply does not hold the data. Every peer reachable only through
+ * a capped relay (phones, browsers, machines behind a home router) silently drops out.
+ *
+ * `openProtocolStream` fixed the dialling half; this fixes the answering half. As there, the flag is
+ * deliberately NOT an option: a new protocol gets relay support without its author needing to know
+ * the flag exists, and `test/dial-options-single-site.spec.ts` fails the build if a second source
+ * file calls `.handle(...)` directly.
+ *
+ * Accepting the stream is necessary but not sufficient. A stock relay also caps the *whole* relayed
+ * connection — `applyDefaultLimit: true` stamps `Limit { data: 128 KiB, duration: 2 min }` on every
+ * reservation — so a relay-only peer now answers, but the circuit is reset once either cap is hit,
+ * which one cohort frame (`DEFAULT_STREAM_MAX_BYTES`, 512 KiB) can do on its own. Lifting that is
+ * the relay operator's knob, not this one: see `NodeOptions.relayServerInit` in `libp2p-node-base.ts`.
+ *
+ * Imports here are type-only on purpose: this module is reachable from the react-native entry
+ * (`src/rn.ts`), which must not pull node-specific code.
+ */
+
+import type { StreamHandler, StreamHandlerOptions, StreamMiddleware } from "@libp2p/interface";
+
+/**
+ * Anything that can register a libp2p protocol handler.
+ *
+ * Both shapes in this package satisfy it: a `Libp2p` node (`node.handle(...)`) and the `registrar`
+ * component the service classes are constructed with (`components.registrar.handle(...)`).
+ */
+export interface ProtocolRegistrar {
+	handle(protocol: string, handler: StreamHandler, options?: StreamHandlerOptions): Promise<void>;
+}
+
+/**
+ * The per-protocol settings that legitimately vary. `runOnLimitedConnection` is absent on purpose —
+ * it is a constant of this helper, not a caller's choice.
+ */
+export interface RegisterProtocolHandlerOptions {
+	/** Concurrent inbound streams allowed per connection. Omit for libp2p's default (32). */
+	maxInboundStreams?: number;
+	/** Concurrent outbound streams allowed per connection. Omit for libp2p's default (64). */
+	maxOutboundStreams?: number;
+	/** Stream middleware, run around the handler. Omit for none. */
+	middleware?: StreamMiddleware[];
+	/** Replace an existing registration for this protocol instead of rejecting. */
+	force?: true;
+	/** Forwarded to libp2p so a caller's deadline cancels the registration. */
+	signal?: AbortSignal;
+}
+
+/**
+ * Register `handler` for `protocol`, accepting streams over limited (relay) connections.
+ *
+ * Rejects if the registrar rejects — a duplicate protocol id, or an aborted signal. Callers that
+ * register fire-and-forget (`void registerProtocolHandler(...)`) keep that behaviour; see the NOTE
+ * in `libp2p-node-base.ts` about the reactivity handlers.
+ */
+export async function registerProtocolHandler(
+	target: ProtocolRegistrar,
+	protocol: string,
+	handler: StreamHandler,
+	options?: RegisterProtocolHandlerOptions,
+): Promise<void> {
+	// Keys are OMITTED rather than set to `undefined` when the caller did not supply them, so
+	// libp2p applies its own defaults instead of seeing an explicit `undefined`.
+	await target.handle(protocol, handler, {
+		runOnLimitedConnection: true,
+		...(options?.maxInboundStreams !== undefined ? { maxInboundStreams: options.maxInboundStreams } : {}),
+		...(options?.maxOutboundStreams !== undefined ? { maxOutboundStreams: options.maxOutboundStreams } : {}),
+		...(options?.middleware !== undefined ? { middleware: options.middleware } : {}),
+		...(options?.force !== undefined ? { force: options.force } : {}),
+		...(options?.signal !== undefined ? { signal: options.signal } : {}),
+	});
+}