@optimystic/db-p2p 0.16.3 → 0.18.0

package/src/libp2p-node-base.ts

@@ -20,12 +20,13 @@ import { MemoryRawStorage } from './storage/memory-storage.js';
 import type { IRawStorage } from './storage/i-raw-storage.js';
 import { seedOwnedBlocksFromStorage } from './owned-block-seed.js';
 import { clusterMember, type ReconcileBlockCallback, type CommitCertificateSink, type DeriveExpectedClusterCallback } from './cluster/cluster-repo.js';
-import { selectQuorumRev, selectQuorumBlock, canonicalBlockHash, type RevClaim, type BlockHashCandidate } from './cluster/quorum-restore.js';
+import { createReconcileBlock } from './cluster/reconcile-block.js';
+import { resolveClusterPolicy, type ClusterPolicyOptions } from './cluster/cluster-policy.js';
 import { createCommitCertStore, makeClusterCommitCertExtractor, type CommitCertStore } from './cluster/commit-cert.js';
 import { coordinatorRepo } from './repo/coordinator-repo.js';
 import { Libp2pKeyPeerNetwork, type NetworkMode, type NetworkStatePersistence } from './libp2p-key-network.js';
 import { ClusterClient } from './cluster/client.js';
-import type { IRepo, ICluster, ITransactionValidator, BlockId, IBlock, IBlockChangeNotifier } from '@optimystic/db-core';
+import type { IRepo, ICluster, ITransactionValidator, BlockId, IBlockChangeNotifier } from '@optimystic/db-core';
 import type { ITransactionStateStore } from './cluster/i-transaction-state-store.js';
 import { networkManagerService, type NetworkManagerService } from './network/network-manager-service.js';
 import type { SpreadOnChurnConfig, SpreadOnChurnMonitor } from './cluster/spread-on-churn.js';
@@ -70,7 +71,6 @@ import {
 	reactivityNodePolicy,
 	createTierAddressing,
 	createRingHash,
-	DEFAULT_SUPER_MAJORITY_THRESHOLD,
 	Tier,
 	b64urlToBytes,
 	bytesToB64url,
@@ -85,7 +85,7 @@ import { assertSuperMajorityCoupling } from './cluster/supermajority-coupling.js
 import { createLogger } from './logger.js';
 import { PeerReputationService } from './reputation/peer-reputation.js';
 import type { IPeerReputation } from './reputation/types.js';
-import { PenaltyReason } from './reputation/types.js';
+import type { AuthorizeInboundStream, InboundStreamAuthorizationInit } from './inbound-authorization.js';
 import { DisputeService } from './dispute/dispute-service.js';
 import { DisputeClient } from './dispute/client.js';
 import { sampleArbitrators } from './dispute/arbitrator-selection.js';
@@ -135,7 +135,12 @@ const wiringLog = createLogger('node-wiring');
 /** Factory function or instance for creating raw storage */
 export type RawStorageProvider = IRawStorage | (() => IRawStorage);
 
-export type NodeOptions = {
+/**
+ * `ClusterPolicyOptions` is intersected in, not restated: `resolveClusterPolicy` consumes those
+ * fields structurally, so a second copy of the shape here would let a newly added knob compile and
+ * be silently ignored. See `cluster/cluster-policy.ts` for what each one resolves to.
+ */
+export type NodeOptions = ClusterPolicyOptions & {
 	/**
 	 * Network port. Only used by the default `listenAddrs` fallback.
 	 * For non-TCP transports (e.g. WebSockets), set `listenAddrs` explicitly.
@@ -174,15 +179,20 @@ export type NodeOptions = {
 	relayServerInit?: CircuitRelayServerInit;
 	/** Storage provider - either an IRawStorage instance or a factory function. Defaults to MemoryRawStorage if not provided. */
 	storage?: RawStorageProvider;
-	clusterSize?: number; // desired cluster size per key
-	clusterPolicy?: {
-		allowDownsize?: boolean;
-		sizeTolerance?: number; // acceptable relative difference (e.g. 0.5 = +/-50%)
-		superMajorityThreshold?: number; // fraction of peers needed for super-majority (default: DEFAULT_SUPER_MAJORITY_THRESHOLD = 0.75)
-	};
-
 	/** Override libp2p listen multiaddrs. */
 	listenAddrs?: string[];
+	/**
+	 * Multiaddrs to advertise INSTEAD OF the listen addrs. For a node behind a NAT / reverse proxy /
+	 * DNS front that binds one address but is reachable at another. When non-empty these REPLACE the
+	 * advertised set entirely — observed/relayed addresses and {@link NodeOptions.appendAnnounceAddrs}
+	 * are all dropped from it. An empty array means "unset" (libp2p's own semantics).
+	 */
+	announceAddrs?: string[];
+	/**
+	 * Multiaddrs to advertise IN ADDITION TO the listen addrs. Ignored while
+	 * {@link NodeOptions.announceAddrs} is non-empty.
+	 */
+	appendAnnounceAddrs?: string[];
 	/** Override libp2p transports. */
 	transports?: Libp2pTransports;
 
@@ -275,6 +285,36 @@ export type NodeOptions = {
 	 */
 	privateKey?: PrivateKey;
 
+	/**
+	 * Optional predicate deciding whether a remote peer may open one of the four Optimystic
+	 * database protocols on this node (`repo`, `cluster`, `sync`, `block-transfer`). It is
+	 * consulted once per inbound stream, before any frame is decoded or any operation executed.
+	 *
+	 * This is deliberately ONE node-level option threaded to all four services rather than four
+	 * per-service options: "is this peer allowed to talk to my database?" is a property of the
+	 * node, not of the protocol, and four independently-settable options make it easy to secure
+	 * three surfaces and silently miss the fourth. (Each service still accepts the same option in
+	 * its own init, so the services stay independently testable and usable outside this factory.)
+	 *
+	 * Absent → no check at all, and today's behavior exactly. Supplied → fail-closed: `false`, a
+	 * throw, a rejection, or a timeout all deny and abort the stream. `remotePeerId` is the
+	 * dialing peer's `PeerId.toString()`. See {@link AuthorizeInboundStream} and
+	 * `docs/internals.md` § Inbound Stream Authorization.
+	 *
+	 * NOTE: this covers the four database protocols only. The dispute, reactivity, matchmaking,
+	 * cohort-topic and libp2p built-in (identify/ping/…) protocols this node also registers are
+	 * NOT gated by it. To refuse a peer at the connection level instead — every protocol at once,
+	 * including identify — use {@link NodeOptions.connectionGater}.
+	 */
+	authorizeInboundStream?: AuthorizeInboundStream;
+
+	/**
+	 * Deadline for {@link NodeOptions.authorizeInboundStream}; expiry denies the stream (a hanging
+	 * predicate would otherwise pin an inbound stream slot). Defaults to
+	 * `DEFAULT_INBOUND_AUTHORIZATION_TIMEOUT_MS` (5s). Ignored when no predicate is supplied.
+	 */
+	authorizeInboundStreamTimeoutMs?: number;
+
 	/**
 	 * Optional libp2p connection gater. The libp2p browser default denies
 	 * dialing insecure WebSockets and private/loopback addresses; callers
@@ -383,6 +423,19 @@ export async function createLibp2pNodeBase(
 		}
 	};
 
+	// The ONE authorization slice, spread verbatim into all four database-protocol service inits
+	// below. Building it once (rather than repeating two option reads per service) is what makes
+	// "secured three surfaces, missed the fourth" impossible: adding a fifth protocol service is a
+	// spread of this object, and dropping it from one is visible at the call site.
+	// Absent `authorizeInboundStream` → every service constructs its gate as `undefined` and the
+	// inbound path is byte-for-byte what it was before this option existed.
+	const inboundAuthorization: InboundStreamAuthorizationInit = {
+		...(options.authorizeInboundStream ? { authorizeInboundStream: options.authorizeInboundStream } : {}),
+		...(options.authorizeInboundStreamTimeoutMs !== undefined
+			? { authorizeInboundStreamTimeoutMs: options.authorizeInboundStreamTimeoutMs }
+			: {})
+	};
+
 	const nodePrivateKey = options.privateKey ?? await generateKeyPair('Ed25519');
 
 	const listenAddrs = options.listenAddrs ?? defaults.listenAddrs;
@@ -408,8 +461,13 @@ export async function createLibp2pNodeBase(
 	const libp2pOptions: Libp2pInit = {
 		start: false,
 		privateKey: nodePrivateKey,
+		// NOTE: libp2p's `AddressManagerInit` also carries `noAnnounce` and `announceFilter`; neither is
+		// exposed on `NodeOptions`. Add them here the same way if a deployment ever needs to suppress a
+		// specific advertised address rather than replace the whole set.
 		addresses: {
-			listen: listenAddrs
+			listen: listenAddrs,
+			...(options.announceAddrs ? { announce: options.announceAddrs } : {}),
+			...(options.appendAnnounceAddrs ? { appendAnnounce: options.appendAnnounceAddrs } : {})
 		},
 		connectionManager: {
 			// `autoDial`, `minConnections`, and `dialQueue` were stale libp2p option keys silently
@@ -479,7 +537,8 @@ export async function createLibp2pNodeBase(
 			cluster: (components: any) => {
 				const serviceFactory = clusterService({
 					protocolPrefix: `/optimystic/${options.networkName}`,
-					responsibilityK: options.responsibilityK ?? 1
+					responsibilityK: options.responsibilityK ?? 1,
+					...inboundAuthorization
 				});
 				return serviceFactory({
 					logger: components.logger,
@@ -505,7 +564,8 @@ export async function createLibp2pNodeBase(
 			repo: (components: any) => {
 				const serviceFactory = repoService({
 					protocolPrefix: `/optimystic/${options.networkName}`,
-					responsibilityK: options.responsibilityK ?? 1
+					responsibilityK: options.responsibilityK ?? 1,
+					...inboundAuthorization
 				});
 				// RepoService.checkRedirect needs the running node (network manager for the
 				// responsible-set computation, self id for the membership check, connection
@@ -525,7 +585,8 @@ export async function createLibp2pNodeBase(
 
 			sync: (components: any) => {
 				const serviceFactory = syncService({
-					protocolPrefix: `/optimystic/${options.networkName}`
+					protocolPrefix: `/optimystic/${options.networkName}`,
+					...inboundAuthorization
 				});
 				return serviceFactory({
 					logger: components.logger,
@@ -539,11 +600,14 @@ export async function createLibp2pNodeBase(
 			// node's own storage, not be re-routed through the cluster-coordinated repo.
 			blockTransfer: (components: any) => {
 				const serviceFactory = blockTransferService({
-					protocolPrefix: `/optimystic/${options.networkName}`
+					protocolPrefix: `/optimystic/${options.networkName}`,
+					...inboundAuthorization
 				});
 				return serviceFactory({
 					registrar: components.registrar,
-					repo: storageRepo
+					repo: storageRepo,
+					// So this service's authorization denials reach the same error sink as the other three.
+					logger: components.logger
 				});
 			},
 
@@ -636,18 +700,11 @@ export async function createLibp2pNodeBase(
 	const partitionDetector = new PartitionDetector();
 	const fretSvc = (node as any).services?.fret as FretService | undefined;
 
-	const consensusConfig = {
-		superMajorityThreshold: options.clusterPolicy?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
-		simpleMajorityThreshold: 0.51,
-		minAbsoluteClusterSize: 2,
-		allowClusterDownsize: options.clusterPolicy?.allowDownsize ?? true,
-		clusterSizeTolerance: options.clusterPolicy?.sizeTolerance ?? 0.5,
-		partitionDetectionWindow: 60000,
-		// Configured full cluster size — the member's own reference for "full size" in the membership
-		// admission gate (a below-full-size declared set under low FRET confidence is refused as a possible
-		// self-shrink). Matches the size threaded into the coordinator below.
-		clusterSize: options.clusterSize ?? 10
-	};
+	// Every cluster-policy default lives in `cluster/cluster-policy.ts` — including WHY the admission
+	// gate and the repair corroboration floor resolve the one operator field
+	// (`clusterPolicy.assumedClusterSize`) to different values when it is absent. Extracted so those
+	// defaults can be asserted without booting a node.
+	const consensusConfig = resolveClusterPolicy(options);
 
 	// Fetch a block archive from one cohort peer over the sync protocol, bounded by a
 	// per-peer timeout so an unreachable peer can't stall reconciliation. Mirrors the
@@ -674,69 +731,23 @@ export async function createLibp2pNodeBase(
 		}
 	};
 
-	// Active reconciliation for a block this member committed without the matching pend
-	// (cohort drift). Queries the commit cohort (self already excluded) for the block,
-	// picks the highest revision that is at least the committed rev, and persists it via
-	// the churn-replication funnel so the block is no longer under-replicated.
-	const reconcileBlock: ReconcileBlockCallback = async (blockId, committed, cohortPeerIds) => {
-		const targets = cohortPeerIds.filter(id => id !== node.peerId.toString());
-		if (targets.length === 0) return;
-
-		const fetched = await Promise.all(
-			targets.map(async peerIdStr => ({ peerIdStr, archive: await fetchArchiveFromPeer(peerIdStr, blockId) }))
-		);
-
-		// Each cohort archive contributes one (rev, actionId) claim from its max
-		// revision (>= the rev we committed). Pick the target rev by quorum
-		// corroboration rather than raw Math.max — a lone peer inflating its rev
-		// cannot steer reconciliation. Keep the serving peer + block per candidate
-		// so we can then verify content agreement.
-		// NOTE: this quorum is corroboration-of-a-claim, NOT Sybil-resistant cohort
-		// membership — deferred to backlog `debt-read-repair-commit-cert-verification`.
-		const candidates: { peerIdStr: string; rev: number; actionId: string; block?: IBlock }[] = [];
-		for (const { peerIdStr, archive } of fetched) {
-			if (!archive) continue;
-			const revs = Object.keys(archive.revisions).map(Number);
-			if (revs.length === 0) continue;
-			const maxRev = Math.max(...revs);
-			if (maxRev < committed.rev) continue;
-			const data = archive.revisions[maxRev];
-			if (!data?.action) continue;
-			candidates.push({ peerIdStr, rev: maxRev, actionId: data.action.actionId, block: data.block });
-		}
-
-		const revClaims: RevClaim[] = candidates.map(c => ({ peerId: c.peerIdStr, rev: c.rev, actionId: c.actionId }));
-		const selected = selectQuorumRev(revClaims, consensusConfig.simpleMajorityThreshold);
-		if (!selected) return; // no rev corroborated by a quorum → leave block, churn/rebalance retries later
-
-		// Content agreement: among archives corroborating the chosen (rev, actionId)
-		// and actually carrying the block, the content must be byte-identical across
-		// a quorum. A cohort member serving content that hashes differently is rejected.
-		// NOTE: selectQuorumBlock recomputes its quorum over only the block-CARRYING
-		// corroborators, not the full rev-responder set. If most peers corroborate the
-		// rev but few carry block bytes (e.g. mid-prune), the content quorum can shrink
-		// to 2. Harmless with honest peers; if a colluding pair ever becomes the only
-		// block-servers for an agreed rev, that is the Sybil regime already deferred to
-		// backlog `debt-read-repair-commit-cert-verification`.
-		const corroborating = candidates.filter(c => c.rev === selected.rev && c.actionId === selected.actionId && c.block);
-		const hashCandidates: BlockHashCandidate[] = await Promise.all(
-			corroborating.map(async c => ({ peerId: c.peerIdStr, hash: await canonicalBlockHash(c.block!), block: c.block! }))
-		);
-		const agreed = selectQuorumBlock(hashCandidates, consensusConfig.simpleMajorityThreshold);
-		if (!agreed) return; // no content quorum → skip persist
-
-		// Best-effort: penalize cohort members that served content contradicting the
-		// agreed hash for the same committed (rev, actionId). Never let this throw.
-		try {
-			for (const c of hashCandidates) {
-				if (c.hash !== agreed.hash) {
-					reputation.reportPeer(c.peerId, PenaltyReason.InvalidRestoration, `reconcile:${blockId}`);
-				}
-			}
-		} catch { /* reputation write must never block restoration */ }
-
-		await storageRepo.saveReplicatedBlock(blockId, agreed.block, { actionId: selected.actionId, rev: selected.rev });
-	};
+	// Active reconciliation for a block this member committed without a materializable base
+	// (cohort drift, or a refused `missing-base-revision` commit). See `reconcile-block.ts` for
+	// the corroboration rules — in particular why both quorums are capped by how many peers
+	// could answer at all, which is what lets a genuinely two-node cohort heal.
+	// NOTE: this and the CoordinatorRepo below must cap against the SAME
+	// repairCorroborationClusterSize, or the two restoration paths disagree about how much trust a
+	// lone peer gets. Safe today because both read the one `resolveClusterPolicy` result above; if
+	// either ever resolves its own value, add a fail-fast coupling check like
+	// `assertSuperMajorityCoupling` rather than relying on proximity.
+	const reconcileBlock: ReconcileBlockCallback = createReconcileBlock({
+		selfPeerId: node.peerId.toString(),
+		fetchArchive: fetchArchiveFromPeer,
+		saveReplicatedBlock: (blockId, block, source) => storageRepo.saveReplicatedBlock(blockId, block, source),
+		simpleMajorityThreshold: consensusConfig.simpleMajorityThreshold,
+		repairCorroborationClusterSize: consensusConfig.repairCorroborationClusterSize,
+		reputation
+	});
 
 	// Member-side membership derivation for the admission gate: independently re-derive this block's
 	// responsible cluster from the SAME source the coordinator uses (IKeyNetwork.findCluster), plus FRET's
@@ -828,7 +839,13 @@ export async function createLibp2pNodeBase(
 		storageRepo,
 		localCluster: clusterImpl,
 		localPeerId: node.peerId,
-		clusterLatestCallback
+		clusterLatestCallback,
+		// Read-driven acquisition shares the commit path's reconcile callback verbatim: same bounded
+		// archive fetch, same (rev, actionId) and content quorums, same monotonic saveReplicatedBlock
+		// funnel. `clusterLatestCallback` alone can only tell the reader WHICH revision the cohort
+		// holds; this is what moves the bytes. Only reached once a corroborated revision exists, so a
+		// genuinely absent block still costs no archive fetch.
+		acquireBlockFromCohort: reconcileBlock
 	});
 
 	// Fail-fast coupling: the cluster member (what accepts a super-majority as sufficient) and the

package/src/repo/cluster-coordinator.ts

@@ -1,7 +1,6 @@
 import { peerIdFromString } from "@libp2p/peer-id";
-import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig } from "@optimystic/db-core";
+import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster } from "@optimystic/db-core";
 import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, membershipDigest } from "@optimystic/db-core";
-import { ClusterClient } from "../cluster/client.js";
 import { Pending } from "@optimystic/db-core";
 import type { PeerId } from "@libp2p/interface";
 import { createLogger, verbose } from '../logger.js'
@@ -13,6 +12,26 @@ import type { ITransactionStateStore } from "../cluster/i-transaction-state-stor
 
 const log = createLogger('cluster')
 
+/**
+ * Consensus refused a transaction: enough members voted reject that super-majority became
+ * impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
+ * "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
+ * rejection reasons — those are free-form text that is part of each member's signed vote payload
+ * (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
+ * `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
+ * loss (confirmed against local storage) or a genuine validation fault.
+ */
+export class ValidatorRejectionError extends Error {
+	constructor(
+		message: string,
+		/** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
+		readonly rejectReasons: Record<string, string>
+	) {
+		super(message);
+		this.name = 'ValidatorRejectionError';
+	}
+}
+
 /** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
 export type TimerCancel = () => void;
 
@@ -76,7 +95,8 @@ export class ClusterCoordinator {
 
 	constructor(
 		private readonly keyNetwork: IKeyNetwork,
-		private readonly createClusterClient: (peerId: PeerId) => ClusterClient,
+		/** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
+		private readonly createClusterClient: (peerId: PeerId) => ICluster,
 		private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
 		private readonly localCluster?: {
 			update: (record: ClusterRecord) => Promise<ClusterRecord>;
@@ -322,9 +342,11 @@ export class ClusterCoordinator {
 		// If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
 		const maxAllowedRejections = peerCount - superMajority;
 		if (rejectionCount > maxAllowedRejections) {
-			const rejectReasons = Object.entries(promises)
+			const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
 				.filter(([_, sig]) => sig.type === 'reject')
-				.map(([peerId, sig]) => `${peerId}: ${sig.rejectReason ?? 'unknown'}`)
+				.map(([peerId, sig]) => [peerId, sig.rejectReason ?? 'unknown']));
+			const rejectReasons = Object.entries(rejectReasonsByPeer)
+				.map(([peerId, reason]) => `${peerId}: ${reason}`)
 				.join('; ');
 			log('cluster-tx:rejected-by-validators', {
 				messageHash: record.messageHash,
@@ -334,7 +356,9 @@ export class ClusterCoordinator {
 				reasons: rejectReasons
 			});
 			this.updateTransactionRecord(promised.record, 'rejected-by-validators');
-			throw new Error(`Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`);
+			throw new ValidatorRejectionError(
+				`Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
+				rejectReasonsByPeer);
 		}
 
 		if (peerCount > 1 && approvalCount < superMajority) {