@optimystic/db-p2p 0.24.2 → 0.25.0

package/src/peer-address-book.ts

@@ -25,11 +25,12 @@ export const MAX_MERGED_ADDRS_PER_PEER = 8
  */
 export const MAX_LEARNED_PEERS_PER_RECORD = 64
 
-/** The narrow slice of libp2p this module needs — a peer id and (optionally) a peerStore writer. */
+/** The narrow slice of libp2p this module needs — a peer id and (optionally) a peerStore. */
 export interface PeerAddressBookHost {
 	peerId: PeerId
 	peerStore?: {
 		merge?: (id: PeerId, data: { multiaddrs: Multiaddr[] }) => Promise<unknown>
+		get?: (id: PeerId) => Promise<{ addresses?: Array<{ multiaddr: { toString(): string } }> }>
 	}
 }
 
@@ -91,7 +92,25 @@ export interface DirectionalConnection {
  * that merges it. So the cure has to be here, at the producer.
  *
  * An inbound-only peer loses nothing by this: its own advertised addresses reach us through
- * `identify`/`identifyPush` and are published from the peerStore instead.
+ * `identify`/`identifyPush` and are published from the peerStore instead — see
+ * {@link publishableAddrsForPeer}, which is where the two halves are joined.
+ *
+ * NOTE: accepted tradeoff — making an INBOUND connection's `remoteAddr` publishable when it is a
+ * circuit address was proposed (the "at least a relayed dialer has a real address" reading) and
+ * declined. Read from `@libp2p/circuit-relay-v2@4.1.3` as vendored under
+ * `packages/db-p2p/node_modules`: the destination side composes that address as
+ * `ourConnectionToTheRelay.remoteAddr` encapsulated with `/p2p-circuit/p2p/<dialer>`
+ * (`dist/src/transport/index.js:272`), so the relay it names is the one WE hold a reservation
+ * with, not one the dialer is reachable on. A relay's `handleConnect` requires a reservation for
+ * the DESTINATION only (`dist/src/server/index.js:219-222`, status `NO_RESERVATION`) — a dialer
+ * needs none — so a third party dialing that composed address reaches the dialer only if the
+ * dialer coincidentally also holds a reservation on our relay, which nothing establishes. When our
+ * own hop to the relay was itself inbound the prefix is an ephemeral source socket, making it
+ * undialable twice over. And in the one case where it would work — dialer and we share a relay —
+ * the dialer's genuine self-advertised circuit address has already reached us through `identify`,
+ * so publishing the composed form adds nothing and costs a slot against
+ * {@link MAX_MERGED_ADDRS_PER_PEER}. Revisit only on a MEASURED case where a peer's genuine
+ * circuit address reaches a third party by no other route.
  */
 export function publishableConnectionAddr(conn: DirectionalConnection, log: AddressLog): string | undefined {
 	if (conn.direction !== 'outbound') return undefined
@@ -100,6 +119,81 @@ export function publishableConnectionAddr(conn: DirectionalConnection, log: Addr
 	return isCarriableMultiaddrString(addr, log) ? addr : undefined
 }
 
+/**
+ * Join the two sources of a third-party-publishable address set: the publishable half of our live
+ * connections, then the peer's own advertised addresses. De-duplicated, **connection-first**.
+ *
+ * The ordering is not cosmetic. An address that reaches `connectionAddrs` is one we OUTBOUND-dialed,
+ * so libp2p has just succeeded with it; an advertised address is one we have never tried. So the
+ * proven one goes first, and the recipient — which caps what it merges at
+ * {@link MAX_MERGED_ADDRS_PER_PEER} — keeps the proven ones when it truncates.
+ *
+ * `connectionAddrs` are already validated by {@link publishableConnectionAddr}; the advertised half
+ * arrives from a peerStore or a record and is put through {@link validMultiaddrStrings} here, so the
+ * union is uniformly carriable regardless of which side an address came from.
+ *
+ * Split out from {@link publishableAddrsForPeer} for the one caller that has already read the
+ * peerStore for other reasons (`findCluster`'s membership-scoped path reads protocols and addresses
+ * in a single `store.get` per member) and must not pay a second read to reuse the rule.
+ */
+export function unionPublishableAddrs(connectionAddrs: string[], advertisedAddrs: string[], log: AddressLog): string[] {
+	return Array.from(new Set([...connectionAddrs, ...validMultiaddrStrings(advertisedAddrs, log)]))
+}
+
+/**
+ * Every address we may hand a **third** party for `peerId`.
+ *
+ * The single answer to that question: `findCluster` (via {@link unionPublishableAddrs}) and all
+ * three redirect-address resolvers — `RepoService.getPeerAddrs`, `ClusterService.getPeerAddrs`, and
+ * the `getConnectionAddrs` the node wires into the cluster service — go through this one rule.
+ * They used to answer it two different ways, and the connections-only half was the weaker one: a
+ * cohort member that only ever dialed US and is reachable only through a relay has its real circuit
+ * address in exactly one place — the peerStore, where `identify`/`identifyPush` put it — so a
+ * redirect built from connections alone described it as having no address at all.
+ *
+ * A peerStore read that fails or finds nothing yields the connection-derived half rather than
+ * throwing: a redirect carrying half the answer is strictly better than a redirect that errors.
+ */
+export async function publishableAddrsForPeer(
+	host: PeerAddressBookHost,
+	connections: DirectionalConnection[],
+	peerId: PeerId,
+	log: AddressLog
+): Promise<string[]> {
+	const connectionAddrs: string[] = []
+	for (const conn of connections) {
+		const addr = publishableConnectionAddr(conn, log)
+		if (addr !== undefined) connectionAddrs.push(addr)
+	}
+	return unionPublishableAddrs(connectionAddrs, await advertisedAddrsForPeer(host, peerId, log), log)
+}
+
+/**
+ * The addresses `peerId` has advertised to us, as libp2p's peerStore holds them.
+ *
+ * A miss is the common case, not an anomaly — libp2p's `peerStore.get` THROWS for a peer it has no
+ * record of, and a redirect routinely names cohort members we have never met — so this logs under
+ * the ordinary `peer-address-book:*` tag family rather than `WARN:`, which is reserved for input we
+ * were handed and rejected.
+ *
+ * NOTE: this adds one `peerStore.get` per redirect target where the redirect resolvers previously
+ * did none. A redirect names at most `clusterSize` peers — single digits — and only fires when this
+ * node is NOT responsible for the key, so the reads are bounded and off the hot path. Unmeasured;
+ * if redirect volume ever shows up in a profile, batch the reads per payload (they are already
+ * issued concurrently by `Promise.all` at both call sites) rather than dropping the peerStore arm.
+ */
+async function advertisedAddrsForPeer(host: PeerAddressBookHost, peerId: PeerId, log: AddressLog): Promise<string[]> {
+	const get = host.peerStore?.get
+	if (typeof get !== 'function') return []
+	try {
+		const peer = await get.call(host.peerStore, peerId)
+		return (peer?.addresses ?? []).map(a => a.multiaddr.toString())
+	} catch (err) {
+		log('peer-address-book:peerstore-miss peer=%s %o', peerId.toString().substring(0, 12), err)
+		return []
+	}
+}
+
 /**
  * How useful the addresses we hold for a peer are **to this node's own dialer**.
  *

package/src/pend-validation.ts

@@ -0,0 +1,89 @@
+import type { PendRequest, PendValidationResult, UnvalidatablePendPolicy } from "@optimystic/db-core";
+
+/**
+ * Stable, greppable prefix on the failure reason a validating receiver emits when it refuses a pend
+ * that carries no `validation` payload (nothing to re-execute — the single-collection
+ * `Collection.sync` shape) under the fail-closed `unvalidatablePendPolicy: 'reject'`.
+ */
+export const PEND_NOT_VALIDATABLE = 'pend-not-validatable';
+
+/**
+ * Stable, greppable prefix on the failure reason a validating receiver emits when the checker
+ * itself THREW (engine fault, missing table, parse error) rather than returning a verdict. Distinct
+ * from a content verdict on purpose: an operator reading a reject reason can tell "this transaction
+ * is wrong" from "this node could not tell".
+ */
+export const VALIDATOR_FAULT = 'validator-fault';
+
+/**
+ * One re-check attempt: hand the pend's `validation` pair to whatever checker this tier holds
+ * (a `ClusterMember`'s `ITransactionValidator`, a `StorageRepo`'s `validatePend` hook) and
+ * return its verdict. Throwing is allowed — {@link checkPendValidation} converts a throw into a
+ * {@link VALIDATOR_FAULT} rejection.
+ */
+export type PendChecker = (validation: NonNullable<PendRequest['validation']>) => Promise<PendValidationResult>;
+
+/** Structured trace of the decision this helper took, rendered by whichever tier called it. */
+export type PendValidationEvent =
+	| { kind: 'unvalidatable'; policy: UnvalidatablePendPolicy }
+	| { kind: 'validator-fault'; error: string };
+
+/**
+ * The fail-closed pend re-check, shared by BOTH tiers that hold a checker — `ClusterMember`
+ * (voting on a cluster record) and `StorageRepo` (applying a pend locally). One implementation
+ * rather than two mirrored copies, because the two tiers refusing with *different* prefixes or
+ * *different* policy semantics is a silent inconsistency nothing would catch: they are read by the
+ * same operator, and a member that votes approve while its own storage refuses at apply burns a
+ * consensus round for nothing.
+ *
+ * Two decisions live here:
+ *
+ * - **No `validation` pair** (the single-collection `Collection.sync` shape — bare transforms, so
+ *   there is nothing to re-execute): an explicit, LOGGED policy branch on both arms, never a silent
+ *   fall-through. `'accept'` admits it unchecked, preserving the historical behaviour; `'reject'`
+ *   refuses with {@link PEND_NOT_VALIDATABLE}. The presence test is on the whole pair, so a sender
+ *   cannot talk a receiver out of validating by omitting half of it.
+ * - **A checker that throws**: caught and turned into a {@link VALIDATOR_FAULT} rejection, never an
+ *   escaping error. At the cluster tier an escaping throw costs the member its vote entirely —
+ *   indistinguishable from an unreachable peer, and with no signed reason for the dispute path.
+ *
+ * NOTE: a TRANSIENT checker fault (database busy, momentary connection loss) therefore produces a
+ * terminal reject where a redelivery might have produced an approve. `CoordinatorRepo`'s two
+ * rejection classifiers confirm retryability against LOCAL storage state and never read this prose,
+ * so a validator-fault reject is returned as retryable only when local state independently shows a
+ * stale revision or a rival pending — otherwise it reaches the writer as a throw. If transient
+ * validator faults ever show up in practice, give the classifier an arm keyed on this prefix rather
+ * than reverting to a silent pass or a lost vote.
+ *
+ * @param request the pend under consideration
+ * @param check the tier's checker, or undefined when this node re-validates nothing (then every
+ *   pend passes and the policy is irrelevant — a storage-only node)
+ * @param policy what to do with a pend that carries no `validation` pair
+ * @param onEvent trace sink; called on every decision this helper takes, so the two tiers log the
+ *   same facts in their own formats
+ */
+export async function checkPendValidation(
+	request: PendRequest,
+	check: PendChecker | undefined,
+	policy: UnvalidatablePendPolicy,
+	onEvent: (event: PendValidationEvent) => void
+): Promise<PendValidationResult> {
+	if (!check) {
+		return { valid: true };
+	}
+	if (!request.validation) {
+		onEvent({ kind: 'unvalidatable', policy });
+		return policy === 'reject'
+			// Plain prose after the stable prefix: at the cluster tier this reason is fed to
+			// computeSigningPayload and carried as Signature.rejectReason, so it must stay a string.
+			? { valid: false, reason: `${PEND_NOT_VALIDATABLE}: pend carries no transaction to re-execute` }
+			: { valid: true };
+	}
+	try {
+		return await check(request.validation);
+	} catch (err) {
+		const error = (err as Error).message;
+		onEvent({ kind: 'validator-fault', error });
+		return { valid: false, reason: `${VALIDATOR_FAULT}: ${error}` };
+	}
+}

package/src/reactivity/notify-transport.ts

@@ -27,6 +27,7 @@ import { readFramed } from "p2p-fret";
 import { peerIdToBytes } from "../cohort-topic/peer-codec.js";
 import { sendOneWay, DEFAULT_STREAM_MAX_BYTES } from "../cohort-topic/stream-util.js";
 import { PROTOCOL_REACTIVITY_NOTIFY } from "./protocols.js";
+import { registerProtocolHandler } from "../network/register-protocol-handler.js";
 import { createLogger } from "../logger.js";
 
 const log = createLogger("reactivity-notify");
@@ -126,7 +127,7 @@ export function registerNotifyHandler(
 	transport: ReactivityNotifyTransport,
 	maxBytes = DEFAULT_STREAM_MAX_BYTES,
 ): void {
-	void node.handle(protocol, (stream: Stream, connection: Connection) => {
+	void registerProtocolHandler(node, protocol, (stream: Stream, connection: Connection) => {
 		void (async (): Promise<void> => {
 			try {
 				const frame = await readFramed(stream, maxBytes);

package/src/reactivity/push-state-gossip.ts

@@ -45,6 +45,7 @@ import type { FretCohortGossipTransport } from "../cohort-topic/cohort-gossip-tr
 import { DEFAULT_GOSSIP_INTERVAL_MS } from "../cohort-topic/cohort-gossip-driver.js";
 import { DEFAULT_STREAM_MAX_BYTES } from "../cohort-topic/stream-util.js";
 import { PROTOCOL_REACTIVITY_PUSH_STATE_GOSSIP } from "./protocols.js";
+import { registerProtocolHandler } from "../network/register-protocol-handler.js";
 import { createLogger } from "../logger.js";
 
 const log = createLogger("reactivity-push-state-gossip");
@@ -273,7 +274,7 @@ export function registerPushStateGossipHandler(
 	driver: Pick<ReactivityPushStateGossipDriver, "deliver">,
 	maxBytes = DEFAULT_STREAM_MAX_BYTES,
 ): void {
-	void node.handle(protocol, (stream: Stream, connection: Connection) => {
+	void registerProtocolHandler(node, protocol, (stream: Stream, connection: Connection) => {
 		void (async (): Promise<void> => {
 			try {
 				const frame = await readFramed(stream, maxBytes);

package/src/repo/cluster-coordinator.ts

@@ -1,5 +1,5 @@
 import { peerIdFromString } from "@libp2p/peer-id";
-import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster } from "@optimystic/db-core";
+import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig, ICluster, PendResult, CommitResult } from "@optimystic/db-core";
 import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, membershipDigest } from "@optimystic/db-core";
 import { Pending } from "@optimystic/db-core";
 import type { PeerId } from "@libp2p/interface";
@@ -37,10 +37,17 @@ export class ValidatorRejectionError extends Error {
  * vote (they hold a rival transaction that won the deterministic race on the same blocks) and
  * approvals fell short of super-majority. Distinct from {@link ValidatorRejectionError} — nobody
  * judged this write invalid; it lost an optimistic-concurrency race and a fresh retry can win.
- * `CoordinatorRepo.pend` converts this into a `StaleFailure` with `conflict: true` so the normal
- * retry machinery (`isConflictFailure`) absorbs it; it should escape as a thrown error only from
- * paths other than pend. The conflicting peers and the winning hashes ride as structured data
- * (from the signed `conflictWith` fields), never parsed out of prose.
+ * `CoordinatorRepo.pend` AND `CoordinatorRepo.commit` both convert this into a `StaleFailure` with
+ * `conflict: true` so the normal retry machinery (`isConflictFailure`) absorbs it; it should escape
+ * as a thrown error only from other paths. The commit conversion matters as much as the pend one:
+ * at the moment this is thrown zero members approved and the members hold the winner — nothing of
+ * the loser landed — yet a THROWN commit error is retried verbatim by db-core's `commitCollection`
+ * (it treats throws as transport faults), and that re-driven commit races into the window after
+ * members apply the winner and clear its reservation, where it can assemble a consensus no member
+ * will durably store. A returned conflict is instead surfaced immediately as a stale loss, and the
+ * writer re-reads and re-drives the whole pend+commit at a fresh revision. The conflicting peers
+ * and the winning hashes ride as structured data (from the signed `conflictWith` fields), never
+ * parsed out of prose.
  */
 export class ConflictRaceLostError extends Error {
 	constructor(
@@ -123,6 +130,10 @@ export class ClusterCoordinator {
 			update: (record: ClusterRecord) => Promise<ClusterRecord>;
 			peerId: PeerId;
 			wasTransactionExecuted?: (messageHash: string) => boolean;
+			/** Local storage's verdict for a pend applied during consensus; see ClusterMember.getExecutedPendResult. */
+			getExecutedPendResult?: (messageHash: string) => PendResult | undefined;
+			/** Local storage's verdict for a commit applied during consensus; see ClusterMember.getExecutedCommitResult. */
+			getExecutedCommitResult?: (messageHash: string) => CommitResult | undefined;
 		},
 		private readonly fretService?: FretService,
 		private readonly reputation?: IPeerReputation,
@@ -183,7 +194,7 @@ export class ClusterCoordinator {
 	 * reproduces the legacy v1 hash byte-for-byte.
 	 *
 	 * NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
-	 * which rides inside the pend operation as `pend.transaction.priority` (multi-collection) or
+	 * which rides inside the pend operation as `pend.validation.transaction.priority` (multi-collection) or
 	 * `pend.priority` (single-collection) — is automatically covered here and by the derived
 	 * promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
 	 * cannot strip or inflate it without invalidating the message hash the members verify. No separate
@@ -219,7 +230,6 @@ export class ClusterCoordinator {
 			membershipVersion: CURRENT_MEMBERSHIP_VERSION,
 			membershipDigest: membershipDigestValue,
 			message,
-			coordinatingBlockIds: message.coordinatingBlockIds,
 			promises: {},
 			commits: {},
 			suggestedClusterSize: peerCount || undefined,
@@ -249,7 +259,41 @@ export class ClusterCoordinator {
 	async executeClusterTransaction(blockId: BlockId, message: RepoMessage, _options?: MessageOptions): Promise<{
 		record: ClusterRecord;
 		localExecuted: boolean;
+		/**
+		 * Local storage's verdict for a pend operation this node's own cluster member applied during
+		 * consensus, when the member retained one. Meaningful only when `localExecuted` is true;
+		 * absent for non-pend messages, for a member that predates the retention, or after the
+		 * retention TTL. `CoordinatorRepo.pend` returns this instead of fabricating a success.
+		 */
+		localPendResult?: PendResult;
+		/**
+		 * Local storage's verdict for a commit operation this node's own cluster member applied
+		 * during consensus, when the member retained one. Same availability contract as
+		 * `localPendResult`. `CoordinatorRepo.commit` uses a retained refusal to detect a rival's
+		 * win swallowed by the member-side ahead-divergence tolerance, instead of fabricating a
+		 * success no member durably stored.
+		 */
+		localCommitResult?: CommitResult;
 	}> {
+		// The coordinating block id is derived HERE, from the key this method is already handed, rather
+		// than being set by each caller's message builder: a member's membership admission gate derives
+		// its own cohort view from this field, and a builder that forgets it silently downgrades the gate
+		// to its fallback floor on that path (which is how `commit` and `cancel` used to strand writes —
+		// admitted at pend, refused at commit). Doing it at the single choke point means a future message
+		// builder cannot reintroduce the gap.
+		//
+		// Two constraints this shape exists to satisfy:
+		//  - COPY, never mutate: `CoordinatorRepo.cancel` builds ONE message and hands the same object to
+		//    N concurrent calls, one per block. In-place mutation would leak one block's id into another
+		//    block's transaction.
+		//  - Preserve an already-present list: `pend` deliberately declares the whole consolidated batch,
+		//    not just its first block, so this must not overwrite it. Tested on `length`, not on the
+		//    field: an empty list carries no id for a member to derive from, so preserving one would be
+		//    the same silent downgrade to the fallback floor this choke point exists to prevent.
+		const coordinated: RepoMessage = message.coordinatingBlockIds?.length
+			? message
+			: { ...message, coordinatingBlockIds: [blockId] };
+
 		// Get the cluster peers for this block
 		const peers = await this.getClusterForBlock(blockId);
 
@@ -258,11 +302,14 @@ export class ClusterCoordinator {
 		// than one hash with a silent internal disagreement about who is responsible.
 		const membershipDigestValue = await membershipDigest(peers);
 
-		// Create a unique hash for this transaction (over message + membership digest)
-		const messageHash = await this.createMessageHash(message, membershipDigestValue);
+		// Create a unique hash for this transaction (over message + membership digest). Hashing the
+		// coordinating-block-bearing copy is what makes the field tamper-evident in transit — and it also
+		// makes a multi-block `cancel` produce a distinct hash per block, where before two blocks with
+		// identical cohorts collided on one `messageHash` in `this.transactions` / `wasTransactionExecuted`.
+		const messageHash = await this.createMessageHash(coordinated, membershipDigestValue);
 
 		// Create a cluster record for this transaction
-		const record = this.makeRecord(peers, messageHash, message, membershipDigestValue);
+		const record = this.makeRecord(peers, messageHash, coordinated, membershipDigestValue);
 		log('cluster-tx:start', {
 			messageHash,
 			blockId,
@@ -296,7 +343,14 @@ export class ClusterCoordinator {
 			const result = await pending.result();
 			// Check if the local cluster already executed the operations during consensus
 			const localExecuted = this.localCluster?.wasTransactionExecuted?.(messageHash) ?? false;
-			return { record: result, localExecuted };
+			const localPendResult = localExecuted ? this.localCluster?.getExecutedPendResult?.(messageHash) : undefined;
+			const localCommitResult = localExecuted ? this.localCluster?.getExecutedCommitResult?.(messageHash) : undefined;
+			return {
+				record: result,
+				localExecuted,
+				...(localPendResult === undefined ? {} : { localPendResult }),
+				...(localCommitResult === undefined ? {} : { localCommitResult })
+			};
 		} finally {
 			const stored = this.transactions.get(messageHash);
 			const retrySnapshot = stored?.retry ? {