@optimystic/db-p2p 0.13.4 → 0.13.5

package/src/repo/cluster-coordinator.ts

@@ -37,10 +37,11 @@ interface ClusterTransactionState {
 /** Manages distributed transactions across clusters */
 export class ClusterCoordinator {
 	private transactions: Map<string, ClusterTransactionState> = new Map();
-	private readonly retryInitialIntervalMs = 2000;
-	private readonly retryBackoffFactor = 2;
-	private readonly retryMaxIntervalMs = 30000;
-	private readonly retryMaxAttempts = 5;
+	private readonly retryInitialIntervalMs: number;
+	private readonly retryBackoffFactor: number;
+	private readonly retryMaxIntervalMs: number;
+	private readonly retryMaxAttempts: number;
+	private readonly commitBroadcastImmediateRetries: number;
 
 	constructor(
 		private readonly keyNetwork: IKeyNetwork,
@@ -54,7 +55,13 @@ export class ClusterCoordinator {
 		private readonly fretService?: FretService,
 		private readonly reputation?: IPeerReputation,
 		private readonly stateStore?: ITransactionStateStore
-	) { }
+	) {
+		this.retryInitialIntervalMs = cfg.commitBroadcastRetryInitialMs ?? 250;
+		this.retryBackoffFactor = cfg.commitBroadcastRetryBackoffFactor ?? 2;
+		this.retryMaxIntervalMs = cfg.commitBroadcastRetryMaxIntervalMs ?? 8000;
+		this.retryMaxAttempts = cfg.commitBroadcastRetryMaxAttempts ?? 5;
+		this.commitBroadcastImmediateRetries = cfg.commitBroadcastImmediateRetries ?? 1;
+	}
 
 	/**
 	 * Creates a base 58 BTC string hash for a message to uniquely identify a transaction
@@ -528,27 +535,7 @@ export class ClusterCoordinator {
 			// so each peer can independently reach consensus and execute the operations.
 			// Without this, only the coordinator's local cluster executes — remote peers
 			// never see enough commits to reach consensus on their own.
-			const broadcastResults = await Promise.allSettled(
-				peerIds.map(peerIdStr => {
-					const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
-					return isLocal
-						? this.localCluster!.update(record)
-						: this.createClusterClient(peerIdFromString(peerIdStr)).update(record).catch(err => {
-							log('cluster-tx:consensus-broadcast-error', { messageHash: record.messageHash, peerId: peerIdStr, error: (err as Error).message });
-							return null;
-						});
-				})
-			);
-
-			// Check for broadcast failures (excluding local)
-			const broadcastFailures: string[] = [];
-			broadcastResults.forEach((result, idx) => {
-				const peerId = peerIds[idx]!;
-				const isLocal = this.localCluster && peerId === this.localCluster.peerId.toString();
-				if (!isLocal && (result.status === 'rejected' || result.value === null)) {
-					broadcastFailures.push(peerId);
-				}
-			});
+			const { failures: broadcastFailures } = await this.broadcastMergedRecord(record, peerIds);
 			if (broadcastFailures.length > 0) {
 				this.scheduleCommitRetry(record.messageHash, record, broadcastFailures);
 			} else {
@@ -565,6 +552,59 @@ export class ClusterCoordinator {
 		return record;
 	}
 
+	/**
+	 * Broadcast the merged commit record to every peer, with `commitBroadcastImmediateRetries`
+	 * in-line re-attempts per peer before giving up. The libp2p connection used during
+	 * the prior commit phase is typically still warm, so a single immediate retry recovers
+	 * most transient stream errors without falling back to the scheduled retry timer.
+	 * Local cluster is invoked exactly once — local failures are fatal, not transient.
+	 */
+	private async broadcastMergedRecord(record: ClusterRecord, peerIds: string[]): Promise<{ failures: string[] }> {
+		const maxAttempts = 1 + Math.max(0, this.commitBroadcastImmediateRetries);
+		const results = await Promise.all(peerIds.map(async peerIdStr => {
+			const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
+			if (isLocal) {
+				try {
+					await this.localCluster!.update(record);
+					return { peerId: peerIdStr, success: true as const };
+				} catch (err) {
+					log('cluster-tx:consensus-broadcast-error', {
+						messageHash: record.messageHash,
+						peerId: peerIdStr,
+						error: (err as Error).message
+					});
+					return { peerId: peerIdStr, success: false as const };
+				}
+			}
+			let lastError: unknown;
+			for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+				try {
+					await this.createClusterClient(peerIdFromString(peerIdStr)).update(record);
+					return { peerId: peerIdStr, success: true as const };
+				} catch (err) {
+					lastError = err;
+					if (attempt < maxAttempts) {
+						log('cluster-tx:consensus-broadcast-retry', {
+							messageHash: record.messageHash,
+							peerId: peerIdStr,
+							attempt,
+							error: (err as Error).message
+						});
+					}
+				}
+			}
+			log('cluster-tx:consensus-broadcast-error', {
+				messageHash: record.messageHash,
+				peerId: peerIdStr,
+				attempts: maxAttempts,
+				error: lastError instanceof Error ? lastError.message : String(lastError)
+			});
+			return { peerId: peerIdStr, success: false as const };
+		}));
+		const failures = results.filter(r => !r.success).map(r => r.peerId);
+		return { failures };
+	}
+
 	private updateTransactionRecord(record: ClusterRecord, stage: string): void {
 		const state = this.transactions.get(record.messageHash);
 		if (!state) {

package/src/repo/coordinator-repo.ts

@@ -65,6 +65,14 @@ export class CoordinatorRepo implements IRepo {
 	private readonly localPeerId?: PeerId;
 	private readonly responsibilityCache = new LruMap<string, { inCluster: boolean, expires: number }>(1000);
 	private static readonly RESPONSIBILITY_TTL_MS = 60_000;
+	private readonly lastSeenCommitMs = new LruMap<string, number>(1000);
+	private readonly readRepairMode: 'off' | 'lazy' | 'paranoid';
+	private readonly readRepairWindowMs: number;
+	private readonly readRepairSampleRate: number;
+	/** Test seam: overridable clock for window-based read-repair gating. */
+	now: () => number = () => Date.now();
+	/** Test seam: overridable RNG (0..1) for sample-rate gating. */
+	rand: () => number = () => Math.random();
 
 	constructor(
 		readonly keyNetwork: IKeyNetwork,
@@ -86,8 +94,19 @@ export class CoordinatorRepo implements IRepo {
 			minAbsoluteClusterSize: cfg?.minAbsoluteClusterSize ?? 3,
 			allowClusterDownsize: cfg?.allowClusterDownsize ?? true,
 			clusterSizeTolerance: cfg?.clusterSizeTolerance ?? 0.5,
-			partitionDetectionWindow: cfg?.partitionDetectionWindow ?? 60000
+			partitionDetectionWindow: cfg?.partitionDetectionWindow ?? 60000,
+			commitBroadcastRetryInitialMs: cfg?.commitBroadcastRetryInitialMs ?? 250,
+			commitBroadcastRetryBackoffFactor: cfg?.commitBroadcastRetryBackoffFactor ?? 2,
+			commitBroadcastRetryMaxIntervalMs: cfg?.commitBroadcastRetryMaxIntervalMs ?? 8000,
+			commitBroadcastRetryMaxAttempts: cfg?.commitBroadcastRetryMaxAttempts ?? 5,
+			commitBroadcastImmediateRetries: cfg?.commitBroadcastImmediateRetries ?? 1,
+			readRepairMode: cfg?.readRepairMode ?? 'lazy',
+			readRepairWindowMs: cfg?.readRepairWindowMs ?? 10000,
+			readRepairSampleRate: cfg?.readRepairSampleRate ?? 0
 		};
+		this.readRepairMode = policy.readRepairMode!;
+		this.readRepairWindowMs = policy.readRepairWindowMs!;
+		this.readRepairSampleRate = policy.readRepairSampleRate!;
 		const localClusterRef = localCluster && localPeerId ? {
 			update: localCluster.update.bind(localCluster),
 			peerId: localPeerId,
@@ -159,24 +178,44 @@ export class CoordinatorRepo implements IRepo {
 		// First try local storage
 		const localResult = await this.storageRepo.get(blockGets, options);
 
-		// Check for blocks that weren't found locally - try to fetch from cluster peers
-		// Skip cluster fetch if this is already a sync request (to prevent recursive queries)
+		// Decide per-block whether to consult cluster peers. Two triggers:
+		//   (a) Missing — block isn't present locally at all (legacy behavior).
+		//   (b) Stale-by-policy — block is present but read-repair policy says verify.
+		// Skip cluster fetch if this is already a sync request (to prevent recursive queries).
 		const skipClusterFetch = (options as any)?.skipClusterFetch;
 		if (this.clusterLatestCallback && !skipClusterFetch) {
 			for (const blockId of blockGets.blockIds) {
 				const localEntry = localResult[blockId];
-				// If block not found locally (no state), try cluster peers
-				if (!localEntry?.state?.latest) {
-					try {
-						await this.fetchBlockFromCluster(blockId, blockGets.context);
-						// Re-fetch after sync
-						const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
-						if (refreshed[blockId]) {
-							localResult[blockId] = refreshed[blockId];
+				const localRev = localEntry?.state?.latest?.rev;
+				const isMissing = !localEntry?.state?.latest;
+				const isStale = !isMissing && this.shouldReadRepair(blockId);
+				if (!isMissing && !isStale) continue;
+
+				if (isStale) {
+					log('cluster-tx:read-repair-triggered', {
+						blockId,
+						mode: this.readRepairMode,
+						ageMs: this.ageMs(blockId),
+						localRev
+					});
+				}
+
+				try {
+					await this.fetchBlockFromCluster(blockId, blockGets.context);
+					const refreshed = await this.storageRepo.get({ blockIds: [blockId], context: blockGets.context }, options);
+					const newRev = refreshed[blockId]?.state?.latest?.rev;
+					if (refreshed[blockId]) {
+						localResult[blockId] = refreshed[blockId];
+					}
+					if (isStale) {
+						if (typeof newRev === 'number' && typeof localRev === 'number' && newRev > localRev) {
+							log('cluster-tx:read-repair-applied', { blockId, oldRev: localRev, newRev });
+						} else {
+							log('cluster-tx:read-repair-noop', { blockId });
 						}
-					} catch (err) {
-						log('cluster-fetch:error', { blockId, error: (err as Error).message });
 					}
+				} catch (err) {
+					log('cluster-fetch:error', { blockId, error: (err as Error).message });
 				}
 			}
 		}
@@ -184,6 +223,44 @@ export class CoordinatorRepo implements IRepo {
 		return localResult;
 	}
 
+	/** Decide whether the read-repair policy wants us to consult the cluster for a present-but-possibly-stale block. */
+	private shouldReadRepair(blockId: BlockId): boolean {
+		switch (this.readRepairMode) {
+			case 'off': return false;
+			case 'paranoid': return true;
+			case 'lazy': {
+				const lastSeen = this.lastSeenCommitMs.get(blockId);
+				if (lastSeen == null) return true;
+				if (this.now() - lastSeen > this.readRepairWindowMs) return true;
+				if (this.readRepairSampleRate > 0 && this.rand() < this.readRepairSampleRate) return true;
+				return false;
+			}
+		}
+	}
+
+	/** Milliseconds since we last marked this block fresh, or undefined if never. */
+	private ageMs(blockId: BlockId): number | undefined {
+		const lastSeen = this.lastSeenCommitMs.get(blockId);
+		return lastSeen == null ? undefined : this.now() - lastSeen;
+	}
+
+	/** Mark blocks as freshly observed from cluster authority (post-commit or post-fetch). */
+	private markBlocksSeen(blockIds: BlockId[]): void {
+		const now = this.now();
+		for (const id of blockIds) {
+			this.lastSeenCommitMs.set(id, now);
+		}
+	}
+
+	/**
+	 * Test seam: directly set the last-seen timestamp for a block. Used by read-repair
+	 * specs to simulate "the local commit happened at time T" without needing to drive
+	 * a full pend/commit cycle through the cluster coordinator.
+	 */
+	setLastSeenForTest(blockId: BlockId, ts: number): void {
+		this.lastSeenCommitMs.set(blockId, ts);
+	}
+
 	private async fetchBlockFromCluster(blockId: BlockId, context?: ActionContext): Promise<void> {
 		if (!this.clusterLatestCallback) return;
 
@@ -210,6 +287,7 @@ export class CoordinatorRepo implements IRepo {
 			// Found on cluster - trigger restoration to sync the block
 			await this.storageRepo.get({ blockIds: [blockId], context: { committed: [clusterLatest], rev: clusterLatest.rev } });
 			log('cluster-fetch:synced', { blockId, rev: clusterLatest.rev });
+			this.markBlocksSeen([blockId]);
 		}
 	}
 
@@ -327,7 +405,9 @@ export class CoordinatorRepo implements IRepo {
 
 		const peerCount = await this.coordinator.getClusterSize(blockIds[0]!);
 		if (peerCount <= 1) {
-			return await this.storageRepo.commit(request, options);
+			const result = await this.storageRepo.commit(request, options);
+			if (result.success) this.markBlocksSeen(blockIds);
+			return result;
 		}
 
 		const message: RepoMessage = {
@@ -338,6 +418,7 @@ export class CoordinatorRepo implements IRepo {
 		try {
 			const { record, localExecuted } = await this.coordinator.executeClusterTransaction(blockIds[0]!, message, options);
 			if (localExecuted) {
+				this.markBlocksSeen(blockIds);
 				return { success: true };
 			}
 			// Local cluster didn't execute during consensus. Attempt a local commit,
@@ -346,13 +427,16 @@ export class CoordinatorRepo implements IRepo {
 			// after missing the pend phase (unreachable during pend, fresh join, etc.).
 			// The cluster's majority is authoritative; this peer will catch up via sync.
 			try {
-				return await this.storageRepo.commit(request, options);
+				const result = await this.storageRepo.commit(request, options);
+				if (result.success) this.markBlocksSeen(blockIds);
+				return result;
 			} catch (err) {
 				if (clusterReachedCommitConsensus(record)) {
 					log('coordinator-repo:commit-local-failed-cluster-succeeded', {
 						actionId: request.actionId,
 						error: (err as Error).message
 					});
+					this.markBlocksSeen(blockIds);
 					return { success: true };
 				}
 				throw err;