@optimystic/db-p2p 0.14.1 → 0.16.3

package/src/matchmaking/query-transport.ts

@@ -112,6 +112,8 @@ export interface MatchmakingQueryServeDeps {
 	readonly gate?: (from: PeerId, topicId: Uint8Array) => boolean;
 	/** Per-frame ceiling for encode/decode; default {@link DEFAULT_STREAM_MAX_BYTES}. */
 	readonly maxBytes?: number;
+	/** Wall clock (unix ms) stamped on the query-accounting bump; default {@link Date.now}. Injectable for tests. */
+	readonly clock?: () => number;
 }
 
 /**
@@ -151,6 +153,13 @@ export function createMatchmakingQueryHandler(
 				return undefined;
 			}
 
+			// Query accounting (matchmaking-query-accounting-seam): a served query bumps `queriesPerMin` for the
+			// topic on the served coord's TrafficCounters; surfaces in a later reply's snapshot (lags one round).
+			// Sits AFTER the gate + no-engine guards (a dropped / unserved query never inflates the barometer) and
+			// BEFORE the reply build (a query that resolved a serving engine is counted on serve, even if signing
+			// later fails transiently). A decode failure threw before `topicId` resolved, so it never counts.
+			engine.recordQuery(topicId, (deps.clock ?? Date.now)());
+
 			// Build the reply from a single synchronous read (records + traffic + epoch) so a concurrent gossip
 			// round cannot tear the snapshot between read and sign. `handleMatchmakingQuery` forwards each record's
 			// fields verbatim through the pure `evaluateQuery` (capability filter + limit truncation + entry

package/src/network/network-manager-service.ts

@@ -72,6 +72,7 @@ export class NetworkManagerService implements Startable {
 	initRebalanceMonitor(
 		partitionDetector: PartitionDetector,
 		fretAdapter: ArachnodeFretAdapter,
+		trackedBlocks: Set<string>,
 		config?: RebalanceMonitorConfig
 	): RebalanceMonitor {
 		const libp2p = this.getLibp2p()
@@ -79,8 +80,10 @@ export class NetworkManagerService implements Startable {
 		if (!libp2p || !fret) {
 			throw new Error('Cannot init RebalanceMonitor: libp2p or FRET not available')
 		}
+		// trackedBlocks is the shared owned-block set: the node passes the SAME Set instance into
+		// both monitor inits so spread and rebalance act on one source of truth and never drift.
 		this.rebalanceMonitor = new RebalanceMonitor(
-			{ libp2p, fret, partitionDetector, fretAdapter },
+			{ libp2p, fret, partitionDetector, fretAdapter, trackedBlocks },
 			config
 		)
 		return this.rebalanceMonitor
@@ -102,6 +105,7 @@ export class NetworkManagerService implements Startable {
 		peerNetwork: SpreadOnChurnDeps['peerNetwork'],
 		clusterSize: number,
 		protocolPrefix: string,
+		trackedBlocks: Set<string>,
 		config?: Partial<SpreadOnChurnConfig>
 	): SpreadOnChurnMonitor {
 		const libp2p = this.getLibp2p()
@@ -113,8 +117,10 @@ export class NetworkManagerService implements Startable {
 		// BlockTransferClient with it, and the receiving node registers the block-transfer
 		// handler under /optimystic/<networkName>. A missing/empty prefix dials the wrong
 		// protocol and every churn push fails to connect.
+		// trackedBlocks is the shared owned-block set: the node passes the SAME Set instance into
+		// both monitor inits so spread and rebalance act on one source of truth and never drift.
 		this.spreadOnChurnMonitor = new SpreadOnChurnMonitor(
-			{ libp2p, fret, partitionDetector, repo, peerNetwork, clusterSize, protocolPrefix },
+			{ libp2p, fret, partitionDetector, repo, peerNetwork, clusterSize, protocolPrefix, trackedBlocks },
 			config
 		)
 		return this.spreadOnChurnMonitor

package/src/owned-block-seed.ts

@@ -0,0 +1,40 @@
+import type { IRawStorage } from './storage/i-raw-storage.js';
+
+/**
+ * Seed the resilience monitors' shared owned-block set from blocks already
+ * durable in raw storage from a previous process run.
+ *
+ * The live owned-block feed (`storageRepo.onAnyCollectionChange`, wired in
+ * `createLibp2pNodeBase`) only fires on NEW commits and RECEIVED replicas, so a
+ * block that was already durable on disk from a previous run is not tracked
+ * after a restart until it happens to be committed or replicated again — a
+ * freshly restarted node would under-protect exactly the data it already holds.
+ * This scan closes that gap by enumerating the metadata store (one id per block
+ * with a committed revision or persisted replica — the same population the live
+ * feed tracks) and adding each id to `ownedBlocks`.
+ *
+ * Called AFTER the live feed is already subscribed, so a block committed/replicated
+ * mid-scan is independently caught by the feed; `Set.add` is idempotent, so the
+ * overlap is harmless. `isStopping()` is checked each iteration so a scan over a
+ * huge store aborts promptly when the node is stopping (the `for await` then calls
+ * the iterator's `return()` to release the backend cursor). A cooperative yield
+ * every `yieldEvery` ids keeps a tight add-loop from monopolizing an event-loop tick.
+ *
+ * Backends that omit `listBlockIds` (or an in-memory backend with nothing durable
+ * across a restart) yield no seed — the monitors still populate over time via the
+ * live feed.
+ */
+export async function seedOwnedBlocksFromStorage(
+	rawStorage: Pick<IRawStorage, 'listBlockIds'>,
+	ownedBlocks: Set<string>,
+	isStopping: () => boolean,
+	yieldEvery = 1000,
+): Promise<void> {
+	if (typeof rawStorage.listBlockIds !== 'function') return;
+	let n = 0;
+	for await (const blockId of rawStorage.listBlockIds()) {
+		if (isStopping()) break;
+		ownedBlocks.add(blockId);
+		if (++n % yieldEvery === 0) await new Promise((resolve) => setTimeout(resolve, 0));
+	}
+}

package/src/protocol-client.ts

@@ -4,6 +4,7 @@ import type { Stream as Libp2pStream } from '@libp2p/interface';
 import type { PeerId, IPeerNetwork } from '@optimystic/db-core';
 import { first } from './it-utility.js';
 import { createLogger } from './logger.js';
+import { MAX_BLOCK_MESSAGE_BYTES } from './protocol-limits.js';
 
 const log = createLogger('protocol-client');
 
@@ -50,7 +51,7 @@ export class ProtocolClient {
 	protected async processMessage<T>(
 		message: unknown,
 		protocol: string,
-		options?: { signal?: AbortSignal; correlationId?: string; dialTimeoutMs?: number; responseTimeoutMs?: number }
+		options?: { signal?: AbortSignal; correlationId?: string; dialTimeoutMs?: number; responseTimeoutMs?: number; maxDataLength?: number }
 	): Promise<T> {
 		const peer = this.peerId.toString();
 		const cid = options?.correlationId;
@@ -144,11 +145,17 @@ export class ProtocolClient {
 				stream.send(chunk);
 			}
 
-			// Read the response from the stream (which is now directly AsyncIterable)
+			// Read the response from the stream (which is now directly AsyncIterable).
+			// Cap the response frame size so a peer can't flood the client with an
+			// oversized reply; each caller passes the cap matching its response shape
+			// (control vs block), defaulting to the block cap so no existing caller
+			// regresses. An oversized frame rejects at the length-prefix (before
+			// allocation) and surfaces through the read's catch, tearing the stream down.
+			const maxDataLength = options?.maxDataLength ?? MAX_BLOCK_MESSAGE_BYTES;
 			let firstByte = true;
 			const source = pipe(
 				stream,
-				lpDecode,
+				(source) => lpDecode(source, { maxDataLength }),
 				async function* (source) {
 					for await (const data of source) {
 						if (firstByte) {

package/src/protocol-limits.ts

@@ -0,0 +1,33 @@
+/**
+ * Inbound-message size caps for the db-p2p consensus protocols.
+ *
+ * Every inbound stream handler frames its wire data with length-prefixed
+ * encoding (`it-length-prefixed`) and `JSON.parse`s each frame. Passing one of
+ * these constants as the decoder's `maxDataLength` makes an oversized frame
+ * reject at the length-prefix (before any allocation against the declared size)
+ * instead of buffering up to the library default (4 MiB) of attacker-controlled
+ * JSON per frame. Mirrors the existing precedent
+ * `DEFAULT_MAX_MESSAGE_BYTES = 1024 * 1024` in
+ * `packages/db-core/src/cohort-topic/wire/codec.ts`.
+ */
+
+/**
+ * Max size (bytes) of a single inbound framed message on a *control-plane*
+ * protocol — cluster records, dispute votes, sync requests. These carry a peer
+ * set + signatures + small metadata, never bulk block data.
+ */
+export const MAX_CONTROL_MESSAGE_BYTES = 1 * 1024 * 1024; // 1 MiB
+
+/**
+ * Max size (bytes) of a single inbound framed message on a *block-carrying*
+ * protocol — repo operations (pend/commit transforms + block bodies),
+ * block-transfer push payloads (base64-inflated block data, multiple per
+ * request), and the block-bearing responses those protocols return.
+ *
+ * NOTE: the repo enforces no hard per-block byte ceiling today, so this is a
+ * heuristic. If a legitimate block/transform can exceed this, the transfer will
+ * be rejected — raise this constant (or thread a config override) rather than
+ * removing the cap. If the repo later grows a real max-block constant, derive
+ * this cap from it.
+ */
+export const MAX_BLOCK_MESSAGE_BYTES = 8 * 1024 * 1024; // 8 MiB

package/src/repo/client.ts

@@ -5,6 +5,7 @@ import type {
 import type { RepoMessage } from "@optimystic/db-core";
 import { blockIdsForTransforms, blockIdToBytes } from "@optimystic/db-core";
 import { ProtocolClient } from "../protocol-client.js";
+import { MAX_BLOCK_MESSAGE_BYTES } from "../protocol-limits.js";
 import { peerIdFromString } from "@libp2p/peer-id";
 
 export class RepoClient extends ProtocolClient implements IRepo {
@@ -94,6 +95,8 @@ export class RepoClient extends ProtocolClient implements IRepo {
 				signal: combinedSignal,
 				correlationId,
 				dialTimeoutMs: options?.dialTimeoutMs,
+				// A get response carries block data → block cap (not control).
+				maxDataLength: MAX_BLOCK_MESSAGE_BYTES,
 			})
 		} finally {
 			clearTimeout(timer)

package/src/repo/cluster-coordinator.ts

@@ -1,7 +1,6 @@
 import { peerIdFromString } from "@libp2p/peer-id";
 import type { ClusterRecord, IKeyNetwork, RepoMessage, BlockId, ClusterPeers, MessageOptions, ClusterConsensusConfig } from "@optimystic/db-core";
-import { base58btc } from "multiformats/bases/base58";
-import { sha256 } from "multiformats/hashes/sha2";
+import { CURRENT_MEMBERSHIP_VERSION, computeClusterMessageHash, membershipDigest } from "@optimystic/db-core";
 import { ClusterClient } from "../cluster/client.js";
 import { Pending } from "@optimystic/db-core";
 import type { PeerId } from "@libp2p/interface";
@@ -14,6 +13,34 @@ import type { ITransactionStateStore } from "../cluster/i-transaction-state-stor
 
 const log = createLogger('cluster')
 
+/** Cancel handle for an injected timer; cancels a not-yet-fired timer (safe no-op after fire/cancel). */
+export type TimerCancel = () => void;
+
+/**
+ * Production timer binding: a one-shot `setTimeout` whose handle is **unref'd** so a pending
+ * commit-retry (or the deferred transaction cleanup) never keeps an otherwise-idle process alive.
+ * The returned handle clears the timeout (idempotent). Mirrors the reactivity rotation
+ * re-registration scheduler's `defaultSetTimer` (see reactivity/rotation-rereg-scheduler.ts).
+ */
+function defaultSetTimer(fn: () => void, delayMs: number): TimerCancel {
+	const handle = setTimeout(fn, delayMs);
+	// An idle retry/cleanup timer must not pin a process (mirror rotation re-registration + push-state gossip).
+	(handle as { unref?: () => void }).unref?.();
+	return (): void => clearTimeout(handle);
+}
+
+/**
+ * Optional injection seam for deterministic time. Production leaves both undefined and gets
+ * `Date.now` + an unref'd `setTimeout`; tests inject a fake clock + timer queue so scheduled
+ * commit-retries fire in virtual (not wall-clock) time.
+ */
+export interface ClusterCoordinatorClock {
+	/** Clock (Unix ms). Defaults to `Date.now`. */
+	now?: () => number;
+	/** Schedule a one-shot timer, returning a cancel handle. Defaults to an unref'd `setTimeout`. */
+	setTimer?: (fn: () => void, delayMs: number) => TimerCancel;
+}
+
 /**
  * Manages the state of cluster transactions for a specific block ID
  */
@@ -21,7 +48,7 @@ interface CommitRetryState {
 	pendingPeers: Set<string>;
 	attempt: number;
 	intervalMs: number;
-	timer?: NodeJS.Timeout;
+	cancel?: TimerCancel;
 }
 
 interface ClusterTransactionState {
@@ -42,6 +69,10 @@ export class ClusterCoordinator {
 	private readonly retryMaxIntervalMs: number;
 	private readonly retryMaxAttempts: number;
 	private readonly commitBroadcastImmediateRetries: number;
+	private readonly promiseImmediateRetries: number;
+	/** Injected clock/timer seam; production defaults to `Date.now` + unref'd `setTimeout`. */
+	private readonly now: () => number;
+	private readonly setTimer: (fn: () => void, delayMs: number) => TimerCancel;
 
 	constructor(
 		private readonly keyNetwork: IKeyNetwork,
@@ -54,31 +85,71 @@ export class ClusterCoordinator {
 		},
 		private readonly fretService?: FretService,
 		private readonly reputation?: IPeerReputation,
-		private readonly stateStore?: ITransactionStateStore
+		private readonly stateStore?: ITransactionStateStore,
+		clock?: ClusterCoordinatorClock
 	) {
 		this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
 		this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
 		this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
 		this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
 		this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
+		this.promiseImmediateRetries = cfg.promiseImmediateRetries ?? 1;
+		this.now = clock?.now ?? ((): number => Date.now());
+		this.setTimer = clock?.setTimer ?? defaultSetTimer;
 	}
 
 	/**
-	 * Creates a base 58 BTC string hash for a message to uniquely identify a transaction
+	 * Invoke one cluster member's `update`, retrying transient REMOTE failures up to
+	 * `immediateRetries` times before surfacing the error. The local cluster is invoked
+	 * exactly once — a local throw is a real fault (validation / merge / consensus), not a
+	 * transient transport blip. A remote call rides a libp2p stream that a circuit-relay
+	 * ("limited") connection can reset once a per-circuit cap or reservation lapses, which
+	 * surfaces as a StreamResetError; an immediate retry on the (usually still-warm)
+	 * connection recovers most of those without escalating the peer to a failure. Shared by
+	 * the promise-collection, commit-collection, and commit-broadcast phases so all three
+	 * react to a relayed reset the same way.
 	 */
-	/** Deterministic JSON: sorts object keys so hash is order-independent */
-	private static canonicalJson(value: unknown): string {
-		return JSON.stringify(value, (_, v) =>
-			v && typeof v === 'object' && !Array.isArray(v)
-				? Object.keys(v).sort().reduce((o: Record<string, unknown>, k) => { o[k] = v[k]; return o; }, {})
-				: v
-		);
+	private async updateMember(peerIdStr: string, record: ClusterRecord, immediateRetries: number, phase: string): Promise<ClusterRecord> {
+		const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
+		if (isLocal) {
+			return await this.localCluster!.update(record);
+		}
+		const maxAttempts = 1 + Math.max(0, immediateRetries);
+		let lastError: unknown;
+		for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+			try {
+				return await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
+			} catch (err) {
+				lastError = err;
+				if (attempt < maxAttempts) {
+					log('cluster-tx:member-update-retry', {
+						messageHash: record.messageHash,
+						peerId: peerIdStr,
+						phase,
+						attempt,
+						error: err instanceof Error ? err.message : String(err)
+					});
+				}
+			}
+		}
+		throw lastError;
 	}
 
-	private async createMessageHash(message: RepoMessage): Promise<string> {
-		const msgBytes = new TextEncoder().encode(ClusterCoordinator.canonicalJson(message));
-		const hashBytes = await sha256.digest(msgBytes);
-		return base58btc.encode(hashBytes.digest);
+	/**
+	 * Creates a base58btc string hash uniquely identifying a transaction. For a v2 record the caller
+	 * threads in the {@link membershipDigest} of the peer set so the responsible membership is bound into
+	 * the identity (two different peer sets ⇒ two different hashes). Omitting `membershipDigestValue`
+	 * reproduces the legacy v1 hash byte-for-byte.
+	 *
+	 * NOTE: the whole `message` is hashed (canonicalJson), so a transaction's advisory aged priority —
+	 * which rides inside the pend operation as `pend.transaction.priority` (multi-collection) or
+	 * `pend.priority` (single-collection) — is automatically covered here and by the derived
+	 * promise/commit hashes. That is what makes priority integrity-protected in transit: a relaying peer
+	 * cannot strip or inflate it without invalidating the message hash the members verify. No separate
+	 * priority-hashing step is needed.
+	 */
+	private async createMessageHash(message: RepoMessage, membershipDigestValue?: string): Promise<string> {
+		return computeClusterMessageHash(message, membershipDigestValue);
 	}
 
 	/**
@@ -97,11 +168,15 @@ export class ClusterCoordinator {
 		}
 	}
 
-	private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage): ClusterRecord {
+	private makeRecord(peers: ClusterPeers, messageHash: string, message: RepoMessage, membershipDigestValue: string): ClusterRecord {
 		const peerCount = Object.keys(peers ?? {}).length;
 		const record: ClusterRecord = {
 			messageHash,
 			peers,
+			// v2: bind the responsible membership into the signed identity. messageHash was computed over
+			// this same digest, so a different peer set would have produced a different messageHash.
+			membershipVersion: CURRENT_MEMBERSHIP_VERSION,
+			membershipDigest: membershipDigestValue,
 			message,
 			coordinatingBlockIds: message.coordinatingBlockIds,
 			promises: {},
@@ -137,11 +212,16 @@ export class ClusterCoordinator {
 		// Get the cluster peers for this block
 		const peers = await this.getClusterForBlock(blockId);
 
-		// Create a unique hash for this transaction
-		const messageHash = await this.createMessageHash(message);
+		// Bind the responsible membership into the transaction identity (v2): the digest is folded into
+		// the messageHash below, so two different peer sets produce two different messageHashes rather
+		// than one hash with a silent internal disagreement about who is responsible.
+		const membershipDigestValue = await membershipDigest(peers);
+
+		// Create a unique hash for this transaction (over message + membership digest)
+		const messageHash = await this.createMessageHash(message, membershipDigestValue);
 
 		// Create a cluster record for this transaction
-		const record = this.makeRecord(peers, messageHash, message);
+		const record = this.makeRecord(peers, messageHash, message, membershipDigestValue);
 		log('cluster-tx:start', {
 			messageHash,
 			blockId,
@@ -161,7 +241,7 @@ export class ClusterCoordinator {
 			messageHash,
 			record,
 			pending,
-			lastUpdate: Date.now()
+			lastUpdate: this.now()
 		};
 		this.transactions.set(messageHash, state);
 		this.persistCoordinatorState(messageHash, record, 'promising');
@@ -192,7 +272,7 @@ export class ClusterCoordinator {
 			// Let the retry completion or abort handle cleanup
 			if (!stored?.retry) {
 				// Wait a bit before cleanup to allow any in-flight responses to arrive
-				setTimeout(() => {
+				this.setTimer(() => {
 					this.transactions.delete(messageHash);
 					this.deleteCoordinatorState(messageHash);
 					log('cluster-tx:transaction-remove', {
@@ -290,6 +370,11 @@ export class ClusterCoordinator {
 				rejectionCount,
 				peerCount
 			});
+			// [dispute-subsystem-dormant] Evidence is computed and persisted but initiateDispute() is
+			// intentionally NOT called here. Dispute origination stays dormant pending arbitrator-set
+			// anchoring — without it a forged synthetic cohort passes resolution.
+			// Gate: tickets/backlog/hardening/invalidation-live-wiring-requires-arbitrator-set-anchoring
+			// Wiring plan: tickets/backlog/feat-dispute-subsystem-live-activation
 		}
 
 		this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
@@ -331,13 +416,18 @@ export class ClusterCoordinator {
 			}
 		}
 
-		// Fallback: accept small clusters in development/testing scenarios
-		// In production, FRET should provide validation
-		log('cluster-tx:small-cluster-accepted-without-validation', {
+		// Fallback: with no confident network-size estimate, fail CLOSED by default.
+		// An undersized cluster with no way to justify its size is unsafe (a lone/
+		// near-lone node could rubber-stamp its own writes), so reject unless the
+		// operator has explicitly opted in via allowUnvalidatedSmallCluster (e.g.
+		// single-node / local dev knowingly running below the floor).
+		const admit = this.cfg.allowUnvalidatedSmallCluster ?? false;
+		log('cluster-tx:small-cluster-no-confident-estimate', {
 			localSize,
-			reason: 'no-confident-network-size-estimate'
+			reason: 'no-confident-network-size-estimate',
+			admit
 		});
-		return true;
+		return admit;
 	}
 
 	/**
@@ -353,14 +443,15 @@ export class ClusterCoordinator {
 			}));
 			log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
 		}
-		// For each peer, create a client and request a promise
+		// For each peer, create a client and request a promise. A remote promise rides
+		// a libp2p stream that a relayed (limited) connection can reset transiently, so
+		// each remote request gets `promiseImmediateRetries` in-line re-attempts before
+		// it counts as a failure — without this a single relayed reset drops the peer and
+		// sinks super-majority (the commit broadcast already has the same guard).
 		const promiseRequests = peerIds.map(peerIdStr => {
 			const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
 			log('cluster-tx:promise-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
-			const promise = isLocal
-				? this.localCluster!.update(record)
-				: this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
-			return new Pending(promise);
+			return new Pending(this.updateMember(peerIdStr, record, this.promiseImmediateRetries, 'promise'));
 		});
 
 		// Wait for all promises to complete
@@ -455,6 +546,10 @@ export class ClusterCoordinator {
 		const commitPayload = {
 			...record
 		};
+		// No per-peer immediate retry here: a commit-collection failure is recovered
+		// downstream by broadcastMergedRecord's in-line retry and the scheduled
+		// commit-retry timer. (The promise phase has no such backstop, which is why
+		// collectPromises gets the immediate retry instead.)
 		const commitRequests = peerIds.map(peerIdStr => {
 			const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
 			log('cluster-tx:commit-request', { messageHash: record.messageHash, peerId: peerIdStr, isLocal });
@@ -560,46 +655,18 @@ export class ClusterCoordinator {
 	 * Local cluster is invoked exactly once — local failures are fatal, not transient.
 	 */
 	private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[] }> {
-		const maxAttempts = 1 + Math.max(0, this.commitBroadcastImmediateRetries);
 		const results = await Promise.all(peerIds.map(async peerIdStr => {
-			const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
-			if (isLocal) {
-				try {
-					await this.localCluster!.update(record);
-					return { peerId: peerIdStr, success: true as const };
-				} catch (err) {
-					log('cluster-tx:consensus-broadcast-error', {
-						messageHash: record.messageHash,
-						peerId: peerIdStr,
-						error: (err as Error).message
-					});
-					return { peerId: peerIdStr, success: false as const };
-				}
-			}
-			let lastError: unknown;
-			for (let attempt = 1; attempt <= maxAttempts; attempt++) {
-				try {
-					await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
-					return { peerId: peerIdStr, success: true as const };
-				} catch (err) {
-					lastError = err;
-					if (attempt < maxAttempts) {
-						log('cluster-tx:consensus-broadcast-retry', {
-							messageHash: record.messageHash,
-							peerId: peerIdStr,
-							attempt,
-							error: (err as Error).message
-						});
-					}
-				}
+			try {
+				await this.updateMember(peerIdStr, record, this.commitBroadcastImmediateRetries, 'commit-broadcast');
+				return { peerId: peerIdStr, success: true as const };
+			} catch (err) {
+				log('cluster-tx:consensus-broadcast-error', {
+					messageHash: record.messageHash,
+					peerId: peerIdStr,
+					error: err instanceof Error ? err.message : String(err)
+				});
+				return { peerId: peerIdStr, success: false as const };
 			}
-			log('cluster-tx:consensus-broadcast-error', {
-				messageHash: record.messageHash,
-				peerId: peerIdStr,
-				attempts: maxAttempts,
-				error: lastError instanceof Error ? lastError.message : String(lastError)
-			});
-			return { peerId: peerIdStr, success: false as const };
 		}));
 		const failures = results.filter(r => !r.success).map(r => r.peerId);
 		return { failures };
@@ -612,7 +679,7 @@ export class ClusterCoordinator {
 			return;
 		}
 		state.record = { ...record };
-		state.lastUpdate = Date.now();
+		state.lastUpdate = this.now();
 		log('cluster-tx:transaction-update', {
 			messageHash: record.messageHash,
 			stage,
@@ -638,17 +705,15 @@ export class ClusterCoordinator {
 		}
 		const pendingPeers = new Set(missingPeers);
 		const baseInterval = existing ? Math.min(existing.intervalMs * this.retryBackoffFactor, this.retryMaxIntervalMs) : this.retryInitialIntervalMs;
-		if (existing?.timer) {
-			clearTimeout(existing.timer);
-		}
-		const timer = setTimeout(() => {
+		existing?.cancel?.();
+		const cancel = this.setTimer(() => {
 			void this.retryCommits(messageHash);
 		}, baseInterval);
 		state.retry = {
 			pendingPeers,
 			attempt: nextAttempt,
 			intervalMs: baseInterval,
-			timer
+			cancel
 		};
 		this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
 			pendingPeers: Array.from(pendingPeers),
@@ -713,12 +778,10 @@ export class ClusterCoordinator {
 		if (!state?.retry) {
 			return;
 		}
-		if (state.retry.timer) {
-			clearTimeout(state.retry.timer);
-		}
+		state.retry.cancel?.();
 		state.retry = undefined;
 		// Clean up the transaction after retry is complete
-		setTimeout(() => {
+		this.setTimer(() => {
 			this.transactions.delete(messageHash);
 			this.deleteCoordinatorState(messageHash);
 			log('cluster-tx:transaction-remove', {
@@ -739,7 +802,7 @@ export class ClusterCoordinator {
 		this.stateStore.saveCoordinatorState(messageHash, {
 			messageHash,
 			record,
-			lastUpdate: Date.now(),
+			lastUpdate: this.now(),
 			phase,
 			retryState
 		}).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
@@ -762,7 +825,7 @@ export class ClusterCoordinator {
 		for (const state of states) {
 			const { messageHash } = state;
 			// Expired — clean up
-			if (state.record.message.expiration && state.record.message.expiration < Date.now()) {
+			if (state.record.message.expiration && state.record.message.expiration < this.now()) {
 				log('cluster-tx:recovery-expired', { messageHash });
 				await this.stateStore.deleteCoordinatorState(messageHash);
 				continue;