@optimystic/db-p2p 0.13.5 → 0.14.0

package/src/cluster/commit-cert.ts

@@ -0,0 +1,139 @@
+import type { ActionId, ClusterRecord, CollectionChangeEvent, CommitCert } from "@optimystic/db-core";
+import { fromString as uint8ArrayFromString } from 'uint8arrays';
+import { createLogger } from "../logger.js";
+
+const log = createLogger('commit-cert');
+
+/**
+ * Build the {@link CommitCert} for a consensus-committed action from its {@link ClusterRecord}.
+ *
+ * The threshold signature is the cluster's `approve` **commit** signatures — the per-member Ed25519
+ * commit votes the cohort already produced over the commit hash — concatenated in ascending
+ * signer-id order, so the blob is reproducible and `signers[i]` aligns with chunk `i` (the
+ * collected-multisig convention the cohort-topic verifier uses). This is the **authoritative**
+ * commit cert: reactivity reuses these exact bytes as a notification's signature and never re-signs.
+ *
+ * `signedPayload` is the exact byte preimage every approving member signed — the cluster commit-vote
+ * payload `utf8(commitHash + ":approve")`, identical across signers. The caller supplies it (this stays
+ * pure and does **not** recompute the commit hash, since the canonical-JSON helper that derives it is
+ * private to `ClusterMember` and duplicating it here risks drift). Reactivity sets a notification's
+ * `digest` to base64url(signedPayload) so a subscriber's threshold-verify over `digest` reproduces the
+ * exact signed image. Pure.
+ */
+export function buildCommitCert(record: ClusterRecord, minSigs: number, signedPayload: Uint8Array): CommitCert {
+	const approvals = Object.entries(record.commits)
+		.filter(([, sig]) => sig.type === 'approve')
+		.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+	const signers = approvals.map(([peerId]) => peerId);
+	const thresholdSig = concatBytes(approvals.map(([, sig]) => uint8ArrayFromString(sig.signature, 'base64url')));
+	return { thresholdSig, signers, minSigs, signedPayload };
+}
+
+/**
+ * The deterministic change-event/notification `actionId` for a durable invalidation, derived from the
+ * `(invalidatedActionId, disputeId)` pair that uniquely identifies the reversal. An invalidation has no
+ * transaction `actionId` of its own (it reverses one), but reactivity keys both the captured commit cert
+ * and the emitted {@link CollectionChangeEvent} by an `actionId` so the bridge's cert extractor can
+ * resolve the reused cert. Both the cluster member's cert capture and the invalidation-apply sink that
+ * emits the change event derive this same id from the same pair, closing the loop. Distinct from the
+ * per-block deterministic revert `actionId` in `dispute/invalidation.ts` (which also carries blockId+rev);
+ * this is the single transaction-level id for the whole reversal.
+ */
+export function invalidationActionId(invalidatedActionId: ActionId, disputeId: string): ActionId {
+	return `inv:${invalidatedActionId}:${disputeId}`;
+}
+
+function concatBytes(parts: readonly Uint8Array[]): Uint8Array {
+	const total = parts.reduce((n, p) => n + p.length, 0);
+	const out = new Uint8Array(total);
+	let offset = 0;
+	for (const part of parts) {
+		out.set(part, offset);
+		offset += part.length;
+	}
+	return out;
+}
+
+/** Default retention for a captured commit cert — long enough for the post-commit change event to
+ * resolve it, short enough that the store stays bounded under sustained commit load. */
+export const DEFAULT_COMMIT_CERT_TTL_MS = 60_000;
+
+/** Default cap on retained certs — a backstop against unbounded growth if commits ever stop emitting. */
+export const DEFAULT_COMMIT_CERT_MAX_ENTRIES = 4096;
+
+/**
+ * Bounded, TTL'd cache of recently-committed {@link CommitCert}s keyed by `actionId`. The cluster
+ * member records a cert here just before consensus applies the commit to local storage; the
+ * change-notifier bridge's extractor reads it when the StorageRepo emits the matching
+ * {@link CollectionChangeEvent}. Both happen on the same node within one commit, so the window is
+ * tiny — but a TTL + cap keep the map from leaking if a commit ever lands without emitting a change
+ * event (e.g. an idempotent re-commit, or a path with no catch-all subscriber).
+ */
+export interface CommitCertStore {
+	/** Record `cert` for `actionId` (overwrites any prior entry). `now` is unix ms (injectable for tests). */
+	put(actionId: ActionId, cert: CommitCert, now?: number): void;
+	/** The cert for `actionId` if still retained, else `undefined`. */
+	get(actionId: ActionId, now?: number): CommitCert | undefined;
+}
+
+export interface CommitCertStoreOptions {
+	/** Retention per entry (ms). Default {@link DEFAULT_COMMIT_CERT_TTL_MS}. */
+	readonly ttlMs?: number;
+	/** Maximum retained entries (oldest evicted first). Default {@link DEFAULT_COMMIT_CERT_MAX_ENTRIES}. */
+	readonly maxEntries?: number;
+}
+
+/** Build an in-memory {@link CommitCertStore} (insertion-ordered Map → recency for eviction). */
+export function createCommitCertStore(options: CommitCertStoreOptions = {}): CommitCertStore {
+	const ttlMs = options.ttlMs ?? DEFAULT_COMMIT_CERT_TTL_MS;
+	const maxEntries = options.maxEntries ?? DEFAULT_COMMIT_CERT_MAX_ENTRIES;
+	const entries = new Map<ActionId, { cert: CommitCert; expiresAt: number }>();
+
+	const dropExpired = (now: number): void => {
+		for (const [actionId, entry] of entries) {
+			if (entry.expiresAt <= now) {
+				entries.delete(actionId);
+			}
+		}
+	};
+
+	return {
+		put(actionId, cert, now = Date.now()): void {
+			dropExpired(now);
+			// Re-insert so a refreshed key moves to the most-recent end (Map preserves insertion order).
+			entries.delete(actionId);
+			entries.set(actionId, { cert, expiresAt: now + ttlMs });
+			while (entries.size > maxEntries) {
+				const oldest = entries.keys().next().value;
+				if (oldest === undefined) break;
+				entries.delete(oldest);
+			}
+		},
+		get(actionId, now = Date.now()): CommitCert | undefined {
+			const entry = entries.get(actionId);
+			if (!entry) {
+				return undefined;
+			}
+			if (entry.expiresAt <= now) {
+				entries.delete(actionId);
+				return undefined;
+			}
+			return entry.cert;
+		},
+	};
+}
+
+/**
+ * The bridge's `extractCommitCert` over a {@link CommitCertStore}: resolve a change event's committed
+ * action to the cluster commit cert captured for it. Returns `undefined` (logged) when no cert is
+ * retained — the bridge then skips origination rather than fabricating an unsigned notification.
+ */
+export function makeClusterCommitCertExtractor(store: CommitCertStore): (event: CollectionChangeEvent) => CommitCert | undefined {
+	return (event) => {
+		const cert = store.get(event.actionId);
+		if (!cert) {
+			log('no commit cert retained for actionId=%s (collection=%s rev=%d); skipping origination', event.actionId, event.collectionId, event.rev);
+		}
+		return cert;
+	};
+}

package/src/cluster/service.ts

@@ -1,7 +1,10 @@
 import { pipe } from 'it-pipe';
 import { decode as lpDecode, encode as lpEncode } from 'it-length-prefixed';
-import type { Startable, Logger, Stream, Connection, StreamHandler } from '@libp2p/interface';
+import { peerIdFromString } from '@libp2p/peer-id';
+import type { Startable, Logger, Stream, Connection, StreamHandler, PeerId } from '@libp2p/interface';
 import type { ICluster, ClusterRecord } from '@optimystic/db-core';
+import { encodePeers, type RedirectPayload } from '../repo/redirect.js';
+import { toClusterErrorEnvelope } from './cluster-error.js';
 import type { Uint8ArrayList } from 'uint8arraylist';
 
 interface BaseComponents {
@@ -14,6 +17,17 @@ interface BaseComponents {
 
 export interface ClusterServiceComponents extends BaseComponents {
 	cluster: ICluster
+	/**
+	 * This node's own peer id, used to decide whether we are a member of a
+	 * cluster record's peer set. When absent the service cannot scope membership
+	 * and processes every update locally (no redirect).
+	 */
+	peerId?: PeerId
+	/**
+	 * Optional resolver for a peer's dialable multiaddrs, used as a fallback when
+	 * a redirect target has no multiaddrs embedded in `record.peers`.
+	 */
+	getConnectionAddrs?: (peerId: PeerId) => string[]
 }
 
 export interface ClusterServiceInit {
@@ -22,17 +36,13 @@ export interface ClusterServiceInit {
 	maxInboundStreams?: number,
 	maxOutboundStreams?: number,
 	logPrefix?: string,
-	kBucketSize?: number,
-	configuredClusterSize?: number,
-	allowClusterDownsize?: boolean,
-	clusterSizeTolerance?: number,
 	/**
 	 * Responsibility K - the replica set size for determining cluster membership.
-	 * This is distinct from kBucketSize (DHT routing) and configuredClusterSize (consensus quorum).
-	 * When set, this determines how many peers (by XOR distance) are considered
-	 * responsible for a key. If this node is not in the top responsibilityK peers,
-	 * it will redirect requests to closer peers.
-	 * Default: 1 (only the closest peer handles requests)
+	 * When the cluster record's peer set is smaller than this, the mesh is treated
+	 * as "small" and the update is processed locally regardless of membership. When
+	 * the peer set is at least this size and we are not a member, the update is
+	 * redirected to the responsible peers.
+	 * Default: 1 (only members process; any larger non-member set redirects)
 	 */
 	responsibilityK?: number,
 }
@@ -52,6 +62,8 @@ export class ClusterService implements Startable {
 	private readonly cluster: ICluster;
 	private readonly components: ClusterServiceComponents;
 	private running: boolean;
+	/** Responsibility K - small-mesh bypass threshold for redirect decisions */
+	private readonly responsibilityK: number;
 
 	constructor(components: ClusterServiceComponents, init: ClusterServiceInit = {}) {
 		this.components = components;
@@ -61,10 +73,74 @@ export class ClusterService implements Startable {
 		this.log = components.logger.forComponent(init.logPrefix ?? 'db-p2p:cluster');
 		this.cluster = components.cluster;
 		this.running = false;
+		this.responsibilityK = init.responsibilityK ?? 1;
 	}
 
 	readonly [Symbol.toStringTag] = '@libp2p/cluster';
 
+	private getSelfId(): PeerId | undefined {
+		if (this.components.peerId) return this.components.peerId;
+		return (this.components as any).libp2p?.peerId as PeerId | undefined;
+	}
+
+	private getPeerAddrs(id: string): string[] {
+		let pid: PeerId;
+		try {
+			pid = peerIdFromString(id);
+		} catch {
+			return [];
+		}
+		if (this.components.getConnectionAddrs) return this.components.getConnectionAddrs(pid);
+		const libp2p = (this.components as any).libp2p;
+		if (!libp2p?.getConnections) return [];
+		const conns: any[] = libp2p.getConnections(pid) ?? [];
+		const addrs: string[] = [];
+		for (const c of conns) {
+			const addr = c.remoteAddr?.toString?.();
+			if (addr) addrs.push(addr);
+		}
+		return addrs;
+	}
+
+	/**
+	 * Decide whether this node should redirect a cluster update instead of
+	 * participating in its consensus.
+	 *
+	 * Membership is scoped against `record.peers` — the authoritative set the
+	 * coordinator already computed and embedded (it only ever dials peers in this
+	 * set). Using it directly (rather than independently recomputing the cluster
+	 * from the key) is regression-proof against the "empty promises" symptom: a
+	 * peer the coordinator legitimately included is, by construction, present in
+	 * `record.peers` and is therefore never redirected.
+	 *
+	 * Returns a {@link RedirectPayload} when this node is not responsible, or null
+	 * when the update should be processed locally (we are a member, the mesh is too
+	 * small to scope, or we lack the identity/peer set to make a decision).
+	 */
+	checkRedirect(record: ClusterRecord): RedirectPayload | null {
+		const selfId = this.getSelfId();
+		if (!selfId) return null;					// no identity → can't scope, process locally
+
+		const peers = record.peers ?? {};
+		const peerIds = Object.keys(peers);
+		if (peerIds.length === 0) return null;		// nothing to scope against → process locally
+
+		const selfStr = selfId.toString();
+		const isMember = peerIds.includes(selfStr);
+		const smallMesh = peerIds.length < this.responsibilityK;
+
+		if (!smallMesh && !isMember) {
+			const others = peerIds.filter(id => id !== selfStr);
+			return encodePeers(others.map(id => {
+				const recAddrs = peers[id]?.multiaddrs ?? [];
+				const addrs = recAddrs.length > 0 ? recAddrs : this.getPeerAddrs(id);
+				return { id, addrs };
+			}));
+		}
+
+		return null;
+	}
+
 	async start(): Promise<void> {
 		if (this.running) {
 			return;
@@ -87,51 +163,47 @@ export class ClusterService implements Startable {
 		this.running = false;
 	}
 
+	/**
+	 * Run a single decoded protocol message. An application-level throw
+	 * (validation / signature / merge / consensus failure inside `cluster.update`)
+	 * propagates to the caller, which turns it into a structured error envelope;
+	 * a redirect or a successful {@link ClusterRecord} is returned as-is.
+	 */
+	private async processOperation(message: { operation: string; record: ClusterRecord }): Promise<unknown> {
+		if (message.operation === 'update') {
+			// Scope consensus to responsible peers: redirect when we are not a
+			// member of the record's authoritative peer set, otherwise process.
+			const redirect = this.checkRedirect(message.record);
+			return redirect ?? await this.cluster.update(message.record);
+		}
+		throw new Error(`Unknown operation: ${message.operation}`);
+	}
+
 	private handleIncomingStream(stream: Stream, connection: Connection): void {
 		const peerId = connection.remotePeer;
 
 		const processStream = async function* (this: ClusterService, source: AsyncIterable<Uint8ArrayList>) {
 			for await (const msg of source) {
-				// Decode the message
+				// Decode the framing. A malformed/undecodable message is a transport
+				// fault handled by the outer abort path, not an application error.
 				const decoded = new TextDecoder().decode(msg.subarray());
 				const message = JSON.parse(decoded) as { operation: string; record: ClusterRecord };
 
-				// Process the operation
-				let response: any;
-					if (message.operation === 'update') {
-          // TEMPORARY: Disable cluster membership check to fix empty promises issue
-          // The membership check was causing peers to return redirect responses
-          // instead of processing cluster updates, leading to empty promise arrays.
-          // TODO: Re-enable and fix cluster membership logic for proper DHT routing
-          response = await this.cluster.update(message.record)
-				} else {
-					throw new Error(`Unknown operation: ${message.operation}`);
+				// Application-level processing: surface any throw to the coordinator as
+				// a structured error envelope (closing the stream normally) instead of
+				// aborting, so the real cause — not an opaque StreamResetError — reaches
+				// the coordinator, which already enables debug logging. The abort path
+				// is reserved for genuinely unrecoverable framing/transport faults.
+				let response: unknown;
+				try {
+					response = await this.processOperation(message);
+				} catch (err) {
+					this.log.error('error processing cluster %s from %p - %e', message.operation, peerId, err);
+					response = toClusterErrorEnvelope(err);
 				}
 
 				// Encode and yield the response
-				if (message.operation === 'update') {
-					const rec = response as any;
-					this.log('cluster-service:pre-serialize', {
-						messageHash: rec?.messageHash,
-						responseType: typeof response,
-						hasPromises: 'promises' in (rec ?? {}),
-						hasCommits: 'commits' in (rec ?? {}),
-						promiseKeys: Object.keys(rec?.promises ?? {}),
-						commitKeys: Object.keys(rec?.commits ?? {}),
-						promiseValues: rec?.promises,
-						commitValues: rec?.commits
-					});
-				}
-				const serialized = JSON.stringify(response);
-				if (message.operation === 'update') {
-					const deserialized = JSON.parse(serialized);
-					this.log('cluster-service:post-serialize', {
-						messageHash: (deserialized as any)?.messageHash,
-						promiseKeys: Object.keys((deserialized as any)?.promises ?? {}),
-						commitKeys: Object.keys((deserialized as any)?.commits ?? {})
-					});
-				}
-				yield new TextEncoder().encode(serialized);
+				yield new TextEncoder().encode(JSON.stringify(response));
 			}
 		};