@optimystic/db-p2p 0.17.0 → 0.19.0

package/src/libp2p-node-base.ts

@@ -21,6 +21,7 @@ import type { IRawStorage } from './storage/i-raw-storage.js';
 import { seedOwnedBlocksFromStorage } from './owned-block-seed.js';
 import { clusterMember, type ReconcileBlockCallback, type CommitCertificateSink, type DeriveExpectedClusterCallback } from './cluster/cluster-repo.js';
 import { createReconcileBlock } from './cluster/reconcile-block.js';
+import { resolveClusterPolicy, type ClusterPolicyOptions } from './cluster/cluster-policy.js';
 import { createCommitCertStore, makeClusterCommitCertExtractor, type CommitCertStore } from './cluster/commit-cert.js';
 import { coordinatorRepo } from './repo/coordinator-repo.js';
 import { Libp2pKeyPeerNetwork, type NetworkMode, type NetworkStatePersistence } from './libp2p-key-network.js';
@@ -70,7 +71,6 @@ import {
 	reactivityNodePolicy,
 	createTierAddressing,
 	createRingHash,
-	DEFAULT_SUPER_MAJORITY_THRESHOLD,
 	Tier,
 	b64urlToBytes,
 	bytesToB64url,
@@ -135,7 +135,12 @@ const wiringLog = createLogger('node-wiring');
 /** Factory function or instance for creating raw storage */
 export type RawStorageProvider = IRawStorage | (() => IRawStorage);
 
-export type NodeOptions = {
+/**
+ * `ClusterPolicyOptions` is intersected in, not restated: `resolveClusterPolicy` consumes those
+ * fields structurally, so a second copy of the shape here would let a newly added knob compile and
+ * be silently ignored. See `cluster/cluster-policy.ts` for what each one resolves to.
+ */
+export type NodeOptions = ClusterPolicyOptions & {
 	/**
 	 * Network port. Only used by the default `listenAddrs` fallback.
 	 * For non-TCP transports (e.g. WebSockets), set `listenAddrs` explicitly.
@@ -174,29 +179,20 @@ export type NodeOptions = {
 	relayServerInit?: CircuitRelayServerInit;
 	/** Storage provider - either an IRawStorage instance or a factory function. Defaults to MemoryRawStorage if not provided. */
 	storage?: RawStorageProvider;
-	/**
-	 * Desired cluster size per key (default 10). Beyond sizing the cohort, this is the
-	 * node's declaration of how many peers *should* exist to corroborate a claim: the
-	 * read-repair corroboration floor is measured against it, so a genuine two-node
-	 * deployment must set `clusterSize: 2` for its members to be able to repair each
-	 * other (see `CoordinatorRepo.corroboratorCapacity`).
-	 */
-	clusterSize?: number;
-	clusterPolicy?: {
-		allowDownsize?: boolean;
-		sizeTolerance?: number; // acceptable relative difference (e.g. 0.5 = +/-50%)
-		superMajorityThreshold?: number; // fraction of peers needed for super-majority (default: DEFAULT_SUPER_MAJORITY_THRESHOLD = 0.75)
-		/**
-		 * Opt in to transacting below the safe cluster-size floor when FRET has no confident
-		 * network-size estimate — the membership-admission and coordinator small-cluster gates
-		 * both fail closed without it. Default false. Turn on only for single-node / local dev
-		 * meshes that knowingly run undersized.
-		 */
-		allowUnvalidatedSmallCluster?: boolean;
-	};
-
 	/** Override libp2p listen multiaddrs. */
 	listenAddrs?: string[];
+	/**
+	 * Multiaddrs to advertise INSTEAD OF the listen addrs. For a node behind a NAT / reverse proxy /
+	 * DNS front that binds one address but is reachable at another. When non-empty these REPLACE the
+	 * advertised set entirely — observed/relayed addresses and {@link NodeOptions.appendAnnounceAddrs}
+	 * are all dropped from it. An empty array means "unset" (libp2p's own semantics).
+	 */
+	announceAddrs?: string[];
+	/**
+	 * Multiaddrs to advertise IN ADDITION TO the listen addrs. Ignored while
+	 * {@link NodeOptions.announceAddrs} is non-empty.
+	 */
+	appendAnnounceAddrs?: string[];
 	/** Override libp2p transports. */
 	transports?: Libp2pTransports;
 
@@ -305,9 +301,10 @@ export type NodeOptions = {
 	 * dialing peer's `PeerId.toString()`. See {@link AuthorizeInboundStream} and
 	 * `docs/internals.md` § Inbound Stream Authorization.
 	 *
-	 * NOTE: this covers the four database protocols only. The reactivity, matchmaking,
+	 * NOTE: this covers the four database protocols only. The dispute, reactivity, matchmaking,
 	 * cohort-topic and libp2p built-in (identify/ping/…) protocols this node also registers are
-	 * NOT gated by it.
+	 * NOT gated by it. To refuse a peer at the connection level instead — every protocol at once,
+	 * including identify — use {@link NodeOptions.connectionGater}.
 	 */
 	authorizeInboundStream?: AuthorizeInboundStream;
 
@@ -464,8 +461,13 @@ export async function createLibp2pNodeBase(
 	const libp2pOptions: Libp2pInit = {
 		start: false,
 		privateKey: nodePrivateKey,
+		// NOTE: libp2p's `AddressManagerInit` also carries `noAnnounce` and `announceFilter`; neither is
+		// exposed on `NodeOptions`. Add them here the same way if a deployment ever needs to suppress a
+		// specific advertised address rather than replace the whole set.
 		addresses: {
-			listen: listenAddrs
+			listen: listenAddrs,
+			...(options.announceAddrs ? { announce: options.announceAddrs } : {}),
+			...(options.appendAnnounceAddrs ? { appendAnnounce: options.appendAnnounceAddrs } : {})
 		},
 		connectionManager: {
 			// `autoDial`, `minConnections`, and `dialQueue` were stale libp2p option keys silently
@@ -603,7 +605,9 @@ export async function createLibp2pNodeBase(
 				});
 				return serviceFactory({
 					registrar: components.registrar,
-					repo: storageRepo
+					repo: storageRepo,
+					// So this service's authorization denials reach the same error sink as the other three.
+					logger: components.logger
 				});
 			},
 
@@ -696,21 +700,11 @@ export async function createLibp2pNodeBase(
 	const partitionDetector = new PartitionDetector();
 	const fretSvc = (node as any).services?.fret as FretService | undefined;
 
-	const consensusConfig = {
-		superMajorityThreshold: options.clusterPolicy?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
-		simpleMajorityThreshold: 0.51,
-		minAbsoluteClusterSize: 2,
-		allowClusterDownsize: options.clusterPolicy?.allowDownsize ?? true,
-		clusterSizeTolerance: options.clusterPolicy?.sizeTolerance ?? 0.5,
-		// Fail closed by default (an undersized cluster with no confident network-size estimate is
-		// rejected); embedders running knowingly-small meshes opt in through clusterPolicy.
-		allowUnvalidatedSmallCluster: options.clusterPolicy?.allowUnvalidatedSmallCluster ?? false,
-		partitionDetectionWindow: 60000,
-		// Configured full cluster size — the member's own reference for "full size" in the membership
-		// admission gate (a below-full-size declared set under low FRET confidence is refused as a possible
-		// self-shrink). Matches the size threaded into the coordinator below.
-		clusterSize: options.clusterSize ?? 10
-	};
+	// Every cluster-policy default lives in `cluster/cluster-policy.ts` — including WHY the admission
+	// gate and the repair corroboration floor resolve the one operator field
+	// (`clusterPolicy.assumedClusterSize`) to different values when it is absent. Extracted so those
+	// defaults can be asserted without booting a node.
+	const consensusConfig = resolveClusterPolicy(options);
 
 	// Fetch a block archive from one cohort peer over the sync protocol, bounded by a
 	// per-peer timeout so an unreachable peer can't stall reconciliation. Mirrors the
@@ -741,12 +735,17 @@ export async function createLibp2pNodeBase(
 	// (cohort drift, or a refused `missing-base-revision` commit). See `reconcile-block.ts` for
 	// the corroboration rules — in particular why both quorums are capped by how many peers
 	// could answer at all, which is what lets a genuinely two-node cohort heal.
+	// NOTE: this and the CoordinatorRepo below must cap against the SAME
+	// repairCorroborationClusterSize, or the two restoration paths disagree about how much trust a
+	// lone peer gets. Safe today because both read the one `resolveClusterPolicy` result above; if
+	// either ever resolves its own value, add a fail-fast coupling check like
+	// `assertSuperMajorityCoupling` rather than relying on proximity.
 	const reconcileBlock: ReconcileBlockCallback = createReconcileBlock({
 		selfPeerId: node.peerId.toString(),
 		fetchArchive: fetchArchiveFromPeer,
 		saveReplicatedBlock: (blockId, block, source) => storageRepo.saveReplicatedBlock(blockId, block, source),
 		simpleMajorityThreshold: consensusConfig.simpleMajorityThreshold,
-		clusterSize: consensusConfig.clusterSize,
+		repairCorroborationClusterSize: consensusConfig.repairCorroborationClusterSize,
 		reputation
 	});
 
@@ -803,12 +802,20 @@ export async function createLibp2pNodeBase(
 		options.transactionStateStore
 	);
 
-	// Create callback for querying cluster peers for their latest block revision
+	// Create callback for querying cluster peers for their latest block revision. Three-way
+	// contract (see ClusterLatestCallback): an ActionRev is the peer's claim, a resolved
+	// `undefined` is the peer answering "I hold nothing", and a REJECTION is silence — the
+	// coordinator counts it as "did not answer" and refuses to report an authoritative absent
+	// over it. Transport errors must therefore propagate, not collapse into `undefined` (that
+	// collapse let a slow two-node cohort report a missing block as authoritatively absent —
+	// ticket cluster-read-consult-cannot-report-unreachable).
 	const clusterLatestCallback: ClusterLatestCallback = async (peerId, blockId, context?) => {
 		// Self-read short-circuit: dialling self via SyncClient is a round trip
 		// with no remote on the other end, and on nodes without listen addresses
 		// (solo WebSocket-only, bare-RN, etc.) the self-dial can hang the dial
-		// queue. Read directly from the local storage repo instead.
+		// queue. Read directly from the local storage repo instead. The catch stays:
+		// a local storage error is not a cohort peer being unreachable, and the
+		// coordinator ignores a self rejection anyway.
 		if (peerId.equals(node.peerId)) {
 			try {
 				const result = await storageRepo.get({ blockIds: [blockId], context });
@@ -818,21 +825,22 @@ export async function createLibp2pNodeBase(
 			}
 		}
 		const syncClient = new SyncClient(peerId, keyNetwork, protocolPrefix);
-		try {
-			const response = await syncClient.requestBlock({ blockId, rev: undefined });
-			if (response.success && response.archive) {
-				const revisions = Object.keys(response.archive.revisions).map(Number);
-				if (revisions.length > 0) {
-					const maxRev = Math.max(...revisions);
-					const revisionData = response.archive.revisions[maxRev];
-					if (revisionData?.action) {
-						return { actionId: revisionData.action.actionId, rev: maxRev };
-					}
+		// No try/catch: a dial or protocol failure rejects through to the coordinator, whose
+		// per-peer deadline also bounds a hung request — slowness needs no race here.
+		const response = await syncClient.requestBlock({ blockId, rev: undefined });
+		if (response.success && response.archive) {
+			const revisions = Object.keys(response.archive.revisions).map(Number);
+			if (revisions.length > 0) {
+				const maxRev = Math.max(...revisions);
+				const revisionData = response.archive.revisions[maxRev];
+				if (revisionData?.action) {
+					return { actionId: revisionData.action.actionId, rev: maxRev };
 				}
 			}
-		} catch {
-			// Peer may be unreachable - return undefined to skip this peer
 		}
+		// The peer DID answer, without data: `success:false` is the sync service's "Block not
+		// found in local storage", and an archive with no usable revisions holds nothing either
+		// way. Both are absent claims, not silence.
 		return undefined;
 	};
 

package/src/repo/cluster-coordinator.ts

@@ -1,7 +1,6 @@
 import { peerIdFromString } from "@libp2p/peer-id";
-import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig } from "@optimystic/db-core";
+import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster } from "@optimystic/db-core";
 import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, membershipDigest } from "@optimystic/db-core";
-import { ClusterClient } from "../cluster/client.js";
 import { Pending } from "@optimystic/db-core";
 import type { PeerId } from "@libp2p/interface";
 import { createLogger, verbose } from '../logger.js'
@@ -13,6 +12,26 @@ import type { ITransactionStateStore } from "../cluster/i-transaction-state-stor
 
 const log = createLogger('cluster')
 
+/**
+ * Consensus refused a transaction: enough members voted reject that super-majority became
+ * impossible. A typed error (rather than a bare `Error`) so the repo layer above can distinguish
+ * "the cluster voted this down" from transport/availability failures WITHOUT string-matching the
+ * rejection reasons — those are free-form text that is part of each member's signed vote payload
+ * (see cluster-repo's `computeSigningPayload`), so their wording must never become control flow.
+ * `CoordinatorRepo.pend` uses this to decide whether a rejection is a retryable stale-revision
+ * loss (confirmed against local storage) or a genuine validation fault.
+ */
+export class ValidatorRejectionError extends Error {
+	constructor(
+		message: string,
+		/** Per-peer reject reasons, verbatim from the vote signatures (free-form, wire-visible). */
+		readonly rejectReasons: Record<string, string>
+	) {
+		super(message);
+		this.name = 'ValidatorRejectionError';
+	}
+}
+
 /** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
 export type TimerCancel = () => void;
 
@@ -76,7 +95,8 @@ export class ClusterCoordinator {
 
 	constructor(
 		private readonly keyNetwork: IKeyNetwork,
-		private readonly createClusterClient: (peerId: PeerId) => ClusterClient,
+		/** Factory for a per-peer cluster RPC handle; only `update` is ever called, hence `ICluster`. */
+		private readonly createClusterClient: (peerId: PeerId) => ICluster,
 		private readonly cfg: ClusterConsensusConfig & { clusterSize: number },
 		private readonly localCluster?: {
 			update: (record: ClusterRecord) => Promise<ClusterRecord>;
@@ -322,9 +342,11 @@ export class ClusterCoordinator {
 		// If more than (peerCount - superMajority) nodes reject, we can never reach super-majority
 		const maxAllowedRejections = peerCount - superMajority;
 		if (rejectionCount > maxAllowedRejections) {
-			const rejectReasons = Object.entries(promises)
+			const rejectReasonsByPeer = Object.fromEntries(Object.entries(promises)
 				.filter(([_, sig]) => sig.type === 'reject')
-				.map(([peerId, sig]) => `${peerId}: ${sig.rejectReason ?? 'unknown'}`)
+				.map(([peerId, sig]) => [peerId, sig.rejectReason ?? 'unknown']));
+			const rejectReasons = Object.entries(rejectReasonsByPeer)
+				.map(([peerId, reason]) => `${peerId}: ${reason}`)
 				.join('; ');
 			log('cluster-tx:rejected-by-validators', {
 				messageHash: record.messageHash,
@@ -334,7 +356,9 @@ export class ClusterCoordinator {
 				reasons: rejectReasons
 			});
 			this.updateTransactionRecord(promised.record, 'rejected-by-validators');
-			throw new Error(`Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`);
+			throw new ValidatorRejectionError(
+				`Transaction rejected by validators (${rejectionCount}/${peerCount} rejected): ${rejectReasons}`,
+				rejectReasonsByPeer);
 		}
 
 		if (peerCount > 1 && approvalCount < superMajority) {