@optimystic/db-p2p 0.16.3 → 0.18.0

package/src/cluster/service.ts

@@ -7,6 +7,7 @@ import { encodePeers, type RedirectPayload } from '../repo/redirect.js';
 import { toClusterErrorEnvelope } from './cluster-error.js';
 import { MAX_CONTROL_MESSAGE_BYTES } from '../protocol-limits.js';
 import type { Uint8ArrayList } from 'uint8arraylist';
+import { createInboundStreamAuthorization, type InboundStreamAuthorization, type InboundStreamAuthorizationInit } from '../inbound-authorization.js';
 
 interface BaseComponents {
 	logger: { forComponent: (name: string) => Logger },
@@ -31,7 +32,7 @@ export interface ClusterServiceComponents extends BaseComponents {
 	getConnectionAddrs?: (peerId: PeerId) => string[]
 }
 
-export interface ClusterServiceInit {
+export interface ClusterServiceInit extends InboundStreamAuthorizationInit {
 	protocol?: string,
 	protocolPrefix?: string,
 	maxInboundStreams?: number,
@@ -65,6 +66,8 @@ export class ClusterService implements Startable {
 	private running: boolean;
 	/** Responsibility K - small-mesh bypass threshold for redirect decisions */
 	private readonly responsibilityK: number;
+	/** Optional embedder authorization gate; `undefined` (the default) means no check runs. */
+	private readonly authorization: InboundStreamAuthorization | undefined;
 
 	constructor(components: ClusterServiceComponents, init: ClusterServiceInit = {}) {
 		this.components = components;
@@ -75,6 +78,7 @@ export class ClusterService implements Startable {
 		this.cluster = components.cluster;
 		this.running = false;
 		this.responsibilityK = init.responsibilityK ?? 1;
+		this.authorization = createInboundStreamAuthorization(init, this.protocol, (msg, ...args) => this.log.error(msg, ...args));
 	}
 
 	readonly [Symbol.toStringTag] = '@libp2p/cluster';
@@ -180,8 +184,8 @@ export class ClusterService implements Startable {
 		throw new Error(`Unknown operation: ${message.operation}`);
 	}
 
-	private handleIncomingStream(stream: Stream, connection: Connection): void {
-		const peerId = connection.remotePeer;
+	private handleIncomingStream(stream: Stream, connection?: Connection): void {
+		const peerId = connection?.remotePeer;
 
 		const processStream = async function* (this: ClusterService, source: AsyncIterable<Uint8ArrayList>) {
 			for await (const msg of source) {
@@ -215,6 +219,9 @@ export class ClusterService implements Startable {
 
 		void (async () => {
 			try {
+				// Authorization runs before ANY decoding or execution. Guarded on the field so a
+				// node without a predicate keeps the original path untouched.
+				if (this.authorization && await this.authorization.deny(stream, peerId?.toString())) return;
 				const responses = pipe(
 					stream,
 					(source) => lpDecode(source, { maxDataLength: MAX_CONTROL_MESSAGE_BYTES }),

package/src/cluster/spread-on-churn.ts

@@ -221,6 +221,14 @@ export class SpreadOnChurnMonitor implements Startable {
 			// Read block data from local storage
 			const result = await this.deps.repo.get({ blockIds: [blockId] })
 			const blockResult = result[blockId]
+			if (blockResult?.unavailable !== undefined) {
+				// The repo could not work out whether it still holds this block (unmaterializable
+				// history / failed restore). Untracking on that answer would silently drop the block
+				// from the spread set on a guess, and only a later re-commit would put it back — so
+				// keep it tracked and let the next sweep (or a heal) settle it.
+				log('unavailable block=%s reason=%s (keeping tracked)', blockId, blockResult.unavailable)
+				continue
+			}
 			if (!blockResult?.block) {
 				// The block has left local storage. No deletion event exists today to evict it
 				// from the tracked set, so prune here. Deleting the current element of a Set mid

package/src/inbound-authorization.ts

@@ -0,0 +1,190 @@
+/**
+ * Optional, embedder-supplied authorization for inbound protocol streams.
+ *
+ * The `repo`, `cluster`, `sync` and `block-transfer` services otherwise go straight from
+ * "stream opened" to "decode and execute", so any peer that can open a connection can issue
+ * database operations. An application that owns a *private* database (only nodes it admitted
+ * may read or write) needs a seam ahead of decoding; this is that seam.
+ *
+ * Contract:
+ *
+ * - **Absent predicate → today's behavior exactly.** Every service holds
+ *   `InboundStreamAuthorization | undefined`; when it is `undefined` the handler never
+ *   awaits anything extra and no code on this path runs.
+ * - **Supplied predicate → fail closed.** Only a literal `true` allows the stream. `false`,
+ *   a throw, a rejection, a timeout, or an unidentifiable remote peer all deny, and denial
+ *   aborts the stream *before* any frame is decoded or any operation executed.
+ * - **Called once per inbound stream**, not once per operation — which is equivalent here
+ *   because all four protocols are strictly one request per stream (each handler's generator
+ *   `return`s after the first response, so a second queued frame is never read).
+ * - **Peer id encoding**: the predicate receives `connection.remotePeer.toString()` — the
+ *   libp2p base58btc/CIDv1 peer-id string (e.g. `12D3KooW…`), the same form
+ *   `PeerId.toString()` produces everywhere else in this codebase. Compare against that,
+ *   never against a multiaddr, a public-key hash, or a base64 encoding.
+ * - **What a caller observes**: a stream reset. Denial is deliberately *not* reported on the
+ *   wire — telling an unauthorized peer "you are not a member" confirms membership state to
+ *   exactly the party the embedder decided not to trust, and the four protocols have four
+ *   different response shapes with no common error frame. The denial is instead loud on the
+ *   *denying* node: it is logged with the peer id, protocol and reason, and the stream is
+ *   aborted with an {@link UnauthorizedInboundStreamError} carrying
+ *   {@link INBOUND_STREAM_UNAUTHORIZED_CODE}, so local diagnostics can tell a denial apart
+ *   from a transport fault.
+ * - **Cost**: the predicate sits in the hot path of every inbound stream, ahead of the work
+ *   that stream would do. Embedders are expected to make it cheap — an in-memory set lookup —
+ *   and to memoize anything that would otherwise hit storage or the network per stream.
+ *
+ * NOTE: denial is stateless and unthrottled — a denied peer may reopen streams as fast as
+ * libp2p's per-connection `maxInboundStreams` allows, and nothing here records the denial. That
+ * is fine while the predicate is an in-memory lookup. If a denied peer ever shows up as load, the
+ * fix is upstream of this module, not inside it: feed denials into `PeerReputationService`, or
+ * refuse the peer at the connection level with `NodeOptions.connectionGater`.
+ */
+
+/**
+ * Predicate deciding whether `remotePeerId` may open `protocol` on this node.
+ *
+ * @param remotePeerId the dialing peer's `PeerId.toString()` (base58btc / CIDv1, `12D3KooW…`)
+ * @param protocol the full protocol id the stream was opened on, e.g.
+ *   `/optimystic/<network>/repo/1.0.0`
+ * @returns `true` to allow. Anything else — `false`, a throw, or a rejection — denies.
+ */
+export type AuthorizeInboundStream = (remotePeerId: string, protocol: string) => Promise<boolean> | boolean;
+
+/** Stable error code carried by {@link UnauthorizedInboundStreamError}. */
+export const INBOUND_STREAM_UNAUTHORIZED_CODE = 'ERR_INBOUND_STREAM_UNAUTHORIZED';
+
+/**
+ * The reason an inbound stream was aborted by the authorization gate. Distinct from a
+ * transport fault so the denying node's logs and any local abort-reason inspection can tell
+ * the two apart; the remote only ever sees the stream reset (see the module doc).
+ */
+export class UnauthorizedInboundStreamError extends Error {
+	readonly code = INBOUND_STREAM_UNAUTHORIZED_CODE;
+	constructor(remotePeerId: string, protocol: string, reason: string) {
+		super(`inbound stream denied: peer=${remotePeerId} protocol=${protocol} reason=${reason}`);
+		this.name = 'UnauthorizedInboundStreamError';
+	}
+}
+
+/**
+ * How long the predicate may take before the stream is denied. A hanging predicate would
+ * otherwise pin an inbound stream slot indefinitely; timing out is the fail-closed reading of
+ * "we could not establish that this peer is allowed".
+ */
+export const DEFAULT_INBOUND_AUTHORIZATION_TIMEOUT_MS = 5_000;
+
+/** The slice of a service's init that configures this gate. Mixed into all four service inits. */
+export interface InboundStreamAuthorizationInit {
+	/**
+	 * Optional predicate consulted once per inbound stream, before any decoding or execution.
+	 * Absent → no check at all. See {@link AuthorizeInboundStream}.
+	 */
+	authorizeInboundStream?: AuthorizeInboundStream;
+	/**
+	 * Deadline for {@link InboundStreamAuthorizationInit.authorizeInboundStream}; expiry denies
+	 * the stream. Default {@link DEFAULT_INBOUND_AUTHORIZATION_TIMEOUT_MS}.
+	 */
+	authorizeInboundStreamTimeoutMs?: number;
+}
+
+/** The part of a libp2p `Stream` this gate needs: the ability to tear it down. */
+interface AbortableStream {
+	abort: (err: Error) => void;
+}
+
+/** Log sink shape shared by the component loggers and the `debug` loggers the services use. */
+export type AuthorizationLog = (message: string, ...args: unknown[]) => void;
+
+/** Marker resolved by the deadline race when the predicate has not settled in time. */
+const TIMED_OUT = Symbol('inbound-authorization-timeout');
+
+/**
+ * A configured authorization gate for one service's protocol. Constructed only when the
+ * embedder supplied a predicate, so a service holding `undefined` runs the original code path.
+ */
+export class InboundStreamAuthorization {
+	constructor(
+		private readonly authorize: AuthorizeInboundStream,
+		private readonly protocol: string,
+		private readonly timeoutMs: number,
+		private readonly log: AuthorizationLog
+	) { }
+
+	/**
+	 * Consult the predicate for one inbound stream and abort the stream if it is not allowed.
+	 *
+	 * @returns `true` when the stream was denied — the caller must return immediately without
+	 *   decoding anything — and `false` when it may proceed.
+	 */
+	async deny(stream: AbortableStream, remotePeerId: string | undefined): Promise<boolean> {
+		// No identifiable remote → the predicate cannot be asked, so we cannot establish
+		// authorization. Fail closed rather than fall through to execution.
+		if (remotePeerId === undefined) {
+			return this.abort(stream, '<unidentified>', 'no remote peer id on the inbound connection');
+		}
+		const verdict = await this.decide(remotePeerId);
+		return verdict.allowed ? false : this.abort(stream, remotePeerId, verdict.reason);
+	}
+
+	/** Run the predicate under its deadline, converting every failure mode into a denial. */
+	private async decide(remotePeerId: string): Promise<{ allowed: true } | { allowed: false, reason: string }> {
+		let timer: ReturnType<typeof setTimeout> | undefined;
+		try {
+			const verdict = this.authorize(remotePeerId, this.protocol);
+			// A synchronous predicate needs no timer at all.
+			if (typeof verdict === 'boolean') {
+				return verdict ? { allowed: true } : { allowed: false, reason: 'predicate returned false' };
+			}
+			const decision = await Promise.race([
+				verdict,
+				new Promise<typeof TIMED_OUT>(resolve => { timer = setTimeout(() => resolve(TIMED_OUT), this.timeoutMs); })
+			]);
+			if (decision === TIMED_OUT) {
+				return { allowed: false, reason: `predicate did not settle within ${this.timeoutMs}ms` };
+			}
+			return decision === true ? { allowed: true } : { allowed: false, reason: 'predicate returned false' };
+		} catch (err) {
+			// A throwing predicate is a bug in the embedder, not permission to proceed: log it
+			// (never swallow) and deny.
+			this.log('authorization predicate threw for peer=%s protocol=%s - %o', remotePeerId, this.protocol, err);
+			return { allowed: false, reason: `predicate threw: ${err instanceof Error ? err.message : String(err)}` };
+		} finally {
+			if (timer !== undefined) clearTimeout(timer);
+		}
+	}
+
+	/** Log the denial and tear the stream down. Always returns `true` (denied). */
+	private abort(stream: AbortableStream, remotePeerId: string, reason: string): true {
+		const error = new UnauthorizedInboundStreamError(remotePeerId, this.protocol, reason);
+		this.log('inbound stream denied peer=%s protocol=%s reason=%s', remotePeerId, this.protocol, reason);
+		try {
+			stream.abort(error);
+		} catch (err) {
+			// The stream may already be torn down; the denial itself still stands.
+			this.log('aborting a denied stream failed peer=%s protocol=%s - %o', remotePeerId, this.protocol, err);
+		}
+		return true;
+	}
+}
+
+/**
+ * Build the gate for a service, or `undefined` when the embedder supplied no predicate.
+ *
+ * Returning `undefined` (rather than an always-allow gate) is what makes the absent-predicate
+ * case genuinely free: the call site guards on the field and never awaits.
+ */
+export function createInboundStreamAuthorization(
+	init: InboundStreamAuthorizationInit,
+	protocol: string,
+	log: AuthorizationLog
+): InboundStreamAuthorization | undefined {
+	if (init.authorizeInboundStream === undefined) {
+		return undefined;
+	}
+	return new InboundStreamAuthorization(
+		init.authorizeInboundStream,
+		protocol,
+		init.authorizeInboundStreamTimeoutMs ?? DEFAULT_INBOUND_AUTHORIZATION_TIMEOUT_MS,
+		log
+	);
+}

package/src/index.ts

@@ -6,6 +6,7 @@ export * from "./cluster/rebalance-monitor.js";
 export * from "./cluster/spread-on-churn.js";
 export * from "./cluster/block-transfer.js";
 export * from "./cluster/block-transfer-service.js";
+export * from "./inbound-authorization.js";
 export * from "./protocol-client.js";
 export * from "./repo/client.js";
 export * from "./repo/cluster-coordinator.js";

package/src/libp2p-key-network.ts

@@ -299,7 +299,25 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 		return { allow: true, reason: 'extended-isolation', warn: true };
 	}
 
+	/**
+	 * Memoize the coordinator for a key. A pick of SELF is deliberately ignored — the
+	 * cache is consulted ahead of every selection tier, so a self entry would keep the
+	 * key routed at our own (possibly stale) replica for the full TTL long after a
+	 * better-placed peer became reachable, and would return self without re-consulting
+	 * {@link shouldAllowSelfCoordination}, letting a partitioned node silently serve its
+	 * own data. Self needs no memoizing anyway: every tier that can select it re-derives
+	 * it from a local lookup with no dial and no retry sleep.
+	 *
+	 * The gate lives here rather than at each call site because most writers are OUTSIDE
+	 * this class — `recordCoordinator` is public and is fed self-valued picks by
+	 * `NetworkTransactor` (it writes back whatever `findCoordinator` returned, including
+	 * self) and by `RepoClient`/`ClusterClient` on redirect responses.
+	 */
 	public recordCoordinator(key: Uint8Array, peerId: PeerId, ttlMs = 30 * 60 * 1000): void {
+		if (peerId.toString() === this.libp2p.peerId.toString()) {
+			this.log('coordinator-cache:self-write-ignored key=%s', this.toCacheKey(key).substring(0, 12))
+			return
+		}
 		const k = this.toCacheKey(key)
 		const now = Date.now()
 		for (const [ck, entry] of this.coordinatorCache) {
@@ -417,11 +435,39 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 				this.log('findCoordinator:fret-neighbors key=%s candidates=%d', keyStr, ids.length)
 				if (verbose) this.log('findCoordinator:fret-candidates key=%s ids=%o connected=%o', keyStr, ids, Array.from(connectedSet))
 
-				// Filter to only connected FRET neighbors, excluding banned peers
+				// Filter to only connected FRET neighbors, excluding banned peers. Self is
+				// never "connected" to itself, so it is admitted by the explicit self clause
+				// below — but ONLY when the self-coordination guard allows it, otherwise a
+				// node whose FRET neighborhood contains self (essentially always on a small or
+				// forming network) would bypass the guard and the last-resort tier's
+				// SELF_COORDINATION_BLOCKED would never fire. On refusal self is merely DROPPED
+				// from the candidate list, so the connected-peer fallback below still gets its
+				// chance at a good remote peer; only if that also comes up empty does the
+				// last-resort tier raise the accurate error.
+				const selfStr = this.libp2p.peerId.toString()
+				let selfAllowedThisAttempt: boolean | undefined
+				// Memoized per ATTEMPT, and evaluated lazily so an all-remote neighborhood never
+				// pays detectPartition() / getNetworkSizeEstimate(). Re-evaluated on each attempt
+				// because a connection can land during the 500ms inter-attempt sleep and
+				// legitimately flip the answer — as filterByMembership re-reads the peerStore.
+				// NOTE: on a small network self is a neighbor of nearly every key, so this runs
+				// per findCoordinator call and self-coordinated keys are never cached to absorb
+				// it. Fine while detectPartition()/getNetworkSizeEstimate() stay local FRET
+				// table reads; if either ever grows a probe or other network round-trip, cache
+				// the decision with a short TTL on the instance instead of per attempt.
+				const isSelfAdmissible = (): boolean => {
+					if (selfAllowedThisAttempt === undefined) {
+						const decision = this.shouldAllowSelfCoordination()
+						selfAllowedThisAttempt = decision.allow
+						if (!decision.allow) {
+							this.log('findCoordinator:fret-self-dropped key=%s reason=%s attempt=%d', keyStr, decision.reason, attempt)
+						}
+					}
+					return selfAllowedThisAttempt
+				}
 				const connectedFretIds = ids
-					.filter(id => (connectedSet.has(id) || id === this.libp2p.peerId.toString())
-						&& !excludedSet.has(id)
-						&& !(this.reputation?.isBanned(id)))
+					.filter(id => !excludedSet.has(id) && !(this.reputation?.isBanned(id)))
+					.filter(id => connectedSet.has(id) || (id === selfStr && isSelfAdmissible()))
 					.sort((a, b) => (this.reputation?.getScore(a) ?? 0) - (this.reputation?.getScore(b) ?? 0))
 				this.log('findCoordinator:fret-connected key=%s count=%d peers=%o', keyStr, connectedFretIds.length, connectedFretIds.map(s => s.substring(0, 12)))
 
@@ -438,6 +484,8 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 				const pick = ranked[0]
 				if (pick) {
 					const pid = peerIdFromString(pick)
+					// A self pick is a no-op here — recordCoordinator ignores self-valued
+					// writes (see its doc comment), matching the last-resort self tier below.
 					this.recordCoordinator(key, pid)
 					this.log('findCoordinator:done key=%s ms=%d source=%s', keyStr, Date.now() - t0, 'fret')
 					return pid
@@ -451,6 +499,8 @@ export class Libp2pKeyPeerNetwork implements IKeyNetwork, IPeerNetwork {
 			// `unknown` peer is never picked). Note this candidate set is built from
 			// connected REMOTE peers and never includes self, so when no serving peer is
 			// present selection falls through to the last-resort self-coordination block.
+			// Being remote-only, this tier needs no self-coordination guard check, unlike the
+			// FRET tier above.
 			const connectedCandidates = connected
 				.filter(p => !excludedSet.has(p.toString()) && !(this.reputation?.isBanned(p.toString())))
 				.sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))