@optimystic/db-p2p 0.9.3 → 0.11.0

package/src/libp2p-node-base.ts

@@ -8,6 +8,7 @@ import { bootstrap } from '@libp2p/bootstrap';
 import { circuitRelayServer } from '@libp2p/circuit-relay-v2';
 import { peerIdFromString } from '@libp2p/peer-id';
 import { generateKeyPair } from '@libp2p/crypto/keys';
+import type { PrivateKey } from '@libp2p/interface';
 import { clusterService } from './cluster/service.js';
 import { repoService } from './repo/service.js';
 import { StorageRepo } from './storage/storage-repo.js';
@@ -19,6 +20,7 @@ import { coordinatorRepo } from './repo/coordinator-repo.js';
 import { Libp2pKeyPeerNetwork, type NetworkMode, type NetworkStatePersistence } from './libp2p-key-network.js';
 import { ClusterClient } from './cluster/client.js';
 import type { IRepo, ICluster, ITransactionValidator } from '@optimystic/db-core';
+import type { ITransactionStateStore } from './cluster/i-transaction-state-store.js';
 import { networkManagerService } from './network/network-manager-service.js';
 import { fretService, Libp2pFretService } from 'p2p-fret';
 import { syncService } from './sync/service.js';
@@ -93,6 +95,19 @@ export type NodeOptions = {
 
 	/** Dispute protocol configuration */
 	dispute?: Partial<DisputeConfig>;
+
+	/** Optional persistent store for 2PC transaction state (enables crash recovery) */
+	transactionStateStore?: ITransactionStateStore;
+
+	/**
+	 * Optional Ed25519 private key for this node. When provided, the libp2p
+	 * node uses this identity instead of generating a fresh keypair. Use this
+	 * to persist peer identity across process restarts.
+	 *
+	 * Accepts a libp2p `PrivateKey` (as returned by `generateKeyPair('Ed25519')`
+	 * or `privateKeyFromProtobuf(...)` from `@libp2p/crypto/keys`).
+	 */
+	privateKey?: PrivateKey;
 };
 
 function resolveStorage(provider: RawStorageProvider | undefined): IRawStorage {
@@ -152,8 +167,7 @@ export async function createLibp2pNodeBase(
 		}
 	};
 
-	// Generate or derive the private key for this node
-	const nodePrivateKey = await generateKeyPair('Ed25519');
+	const nodePrivateKey = options.privateKey ?? await generateKeyPair('Ed25519');
 
 	const listenAddrs = options.listenAddrs ?? defaults.listenAddrs;
 	const transports = options.transports ?? defaults.transports;
@@ -300,7 +314,8 @@ export async function createLibp2pNodeBase(
 		fretService: fretSvc,
 		validator: options.validator,
 		reputation,
-		consensusConfig
+		consensusConfig,
+		stateStore: options.transactionStateStore
 	});
 
 	const coordinatorRepoFactory = coordinatorRepo(
@@ -311,7 +326,8 @@ export async function createLibp2pNodeBase(
 			...consensusConfig
 		},
 		fretSvc,
-		reputation
+		reputation,
+		options.transactionStateStore
 	);
 
 	// Create callback for querying cluster peers for their latest block revision
@@ -342,6 +358,12 @@ export async function createLibp2pNodeBase(
 		clusterLatestCallback
 	});
 
+	// Recover persisted transaction state before accepting new requests
+	if (options.transactionStateStore) {
+		await (clusterImpl as import('./cluster/cluster-repo.js').ClusterMember).recoverTransactions();
+		await (coordinatedRepo as import('./repo/coordinator-repo.js').CoordinatorRepo).recoverTransactions();
+	}
+
 	// Initialize Arachnode ring membership and restoration
 	const enableArachnode = options.arachnode?.enableRingZulu ?? true;
 	if (enableArachnode) {
@@ -439,6 +461,15 @@ export async function createLibp2pNodeBase(
 		});
 	}
 
+	// Cleanup cluster member intervals on node stop
+	{
+		const previousStop = node.stop.bind(node);
+		node.stop = async () => {
+			(clusterImpl as import('./cluster/cluster-repo.js').ClusterMember).dispose();
+			await previousStop();
+		};
+	}
+
 	// Expose coordinated repo and storage for external use
 	(node as any).coordinatedRepo = coordinatedRepo;
 	(node as any).storageRepo = storageRepo;

package/src/network/network-manager-service.ts

@@ -6,6 +6,7 @@ import { toString as u8ToString } from 'uint8arrays/to-string'
 import type { IPeerReputation } from '../reputation/types.js'
 import { PenaltyReason } from '../reputation/types.js'
 import { RebalanceMonitor, type RebalanceMonitorConfig } from '../cluster/rebalance-monitor.js'
+import { SpreadOnChurnMonitor, type SpreadOnChurnConfig, type SpreadOnChurnDeps } from '../cluster/spread-on-churn.js'
 import type { PartitionDetector } from '../cluster/partition-detector.js'
 import type { ArachnodeFretAdapter } from '../storage/arachnode-fret-adapter.js'
 
@@ -41,6 +42,7 @@ export class NetworkManagerService implements Startable {
 	private reputation?: IPeerReputation
 	private libp2pRef: Libp2p | undefined
 	private rebalanceMonitor?: RebalanceMonitor
+	private spreadOnChurnMonitor?: SpreadOnChurnMonitor
 
 	constructor(private readonly components: Components, init: NetworkManagerServiceInit = {}) {
 		this.log = components.logger.forComponent('db-p2p:network-manager')
@@ -88,6 +90,33 @@ export class NetworkManagerService implements Startable {
 		return this.rebalanceMonitor
 	}
 
+	/**
+	 * Initialize the spread-on-churn monitor. Call after libp2p, FRET are available.
+	 * Caller provides repo and peerNetwork (not held by NetworkManagerService directly).
+	 */
+	initSpreadOnChurnMonitor(
+		partitionDetector: PartitionDetector,
+		repo: SpreadOnChurnDeps['repo'],
+		peerNetwork: SpreadOnChurnDeps['peerNetwork'],
+		clusterSize: number,
+		config?: Partial<SpreadOnChurnConfig>
+	): SpreadOnChurnMonitor {
+		const libp2p = this.getLibp2p()
+		const fret = this.getFret()
+		if (!libp2p || !fret) {
+			throw new Error('Cannot init SpreadOnChurnMonitor: libp2p or FRET not available')
+		}
+		this.spreadOnChurnMonitor = new SpreadOnChurnMonitor(
+			{ libp2p, fret, partitionDetector, repo, peerNetwork, clusterSize },
+			config
+		)
+		return this.spreadOnChurnMonitor
+	}
+
+	getSpreadOnChurnMonitor(): SpreadOnChurnMonitor | undefined {
+		return this.spreadOnChurnMonitor
+	}
+
 	private getLibp2p(): Libp2p | undefined {
 		return this.libp2pRef ?? this.components.libp2p;
 	}
@@ -113,6 +142,9 @@ export class NetworkManagerService implements Startable {
 		if (this.rebalanceMonitor) {
 			await this.rebalanceMonitor.stop()
 		}
+		if (this.spreadOnChurnMonitor) {
+			await this.spreadOnChurnMonitor.stop()
+		}
 		this.running = false
 	}
 

package/src/repo/cluster-coordinator.ts

@@ -10,6 +10,7 @@ import type { ClusterLogPeerOutcome } from './types.js'
 import type { FretService } from "p2p-fret";
 import type { IPeerReputation } from "../reputation/types.js";
 import { PenaltyReason } from "../reputation/types.js";
+import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
 
 const log = createLogger('cluster')
 
@@ -35,7 +36,6 @@ interface ClusterTransactionState {
 
 /** Manages distributed transactions across clusters */
 export class ClusterCoordinator {
-	// TODO: move this into a state management interface so that transaction state can be persisted
 	private transactions: Map<string, ClusterTransactionState> = new Map();
 	private readonly retryInitialIntervalMs = 2000;
 	private readonly retryBackoffFactor = 2;
@@ -52,7 +52,8 @@ export class ClusterCoordinator {
 			wasTransactionExecuted?: (messageHash: string) => boolean;
 		},
 		private readonly fretService?: FretService,
-		private readonly reputation?: IPeerReputation
+		private readonly reputation?: IPeerReputation,
+		private readonly stateStore?: ITransactionStateStore
 	) { }
 
 	/**
@@ -156,6 +157,7 @@ export class ClusterCoordinator {
 			lastUpdate: Date.now()
 		};
 		this.transactions.set(messageHash, state);
+		this.persistCoordinatorState(messageHash, record, 'promising');
 		log('cluster-tx:transaction-store', {
 			messageHash,
 			transactionKeys: Array.from(this.transactions.keys())
@@ -185,6 +187,7 @@ export class ClusterCoordinator {
 				// Wait a bit before cleanup to allow any in-flight responses to arrive
 				setTimeout(() => {
 					this.transactions.delete(messageHash);
+					this.deleteCoordinatorState(messageHash);
 					log('cluster-tx:transaction-remove', {
 						messageHash,
 						remaining: Array.from(this.transactions.keys())
@@ -282,6 +285,7 @@ export class ClusterCoordinator {
 			});
 		}
 
+		this.persistCoordinatorState(promised.record.messageHash, promised.record, 'committing');
 		return await this.commitTransaction(promised.record);
 	}
 
@@ -606,6 +610,11 @@ export class ClusterCoordinator {
 			intervalMs: baseInterval,
 			timer
 		};
+		this.persistCoordinatorState(messageHash, state.record, 'broadcasting', {
+			pendingPeers: Array.from(pendingPeers),
+			attempt: nextAttempt,
+			intervalMs: baseInterval
+		});
 		log('cluster-tx:retry-scheduled', { messageHash, attempt: nextAttempt, missingPeers, delayMs: baseInterval });
 	}
 
@@ -671,10 +680,71 @@ export class ClusterCoordinator {
 		// Clean up the transaction after retry is complete
 		setTimeout(() => {
 			this.transactions.delete(messageHash);
+			this.deleteCoordinatorState(messageHash);
 			log('cluster-tx:transaction-remove', {
 				messageHash,
 				remaining: Array.from(this.transactions.keys())
 			});
 		}, 100);
 	}
+
+	/** Fire-and-forget persist — errors are logged, never thrown. */
+	private persistCoordinatorState(
+		messageHash: string,
+		record: ClusterRecord,
+		phase: 'promising' | 'committing' | 'broadcasting',
+		retryState?: { pendingPeers: string[]; attempt: number; intervalMs: number }
+	): void {
+		if (!this.stateStore) return;
+		this.stateStore.saveCoordinatorState(messageHash, {
+			messageHash,
+			record,
+			lastUpdate: Date.now(),
+			phase,
+			retryState
+		}).catch(err => log('cluster-tx:persist-error', { messageHash, error: (err as Error).message }));
+	}
+
+	/** Fire-and-forget delete — errors are logged, never thrown. */
+	private deleteCoordinatorState(messageHash: string): void {
+		if (!this.stateStore) return;
+		this.stateStore.deleteCoordinatorState(messageHash)
+			.catch(err => log('cluster-tx:persist-delete-error', { messageHash, error: (err as Error).message }));
+	}
+
+	/**
+	 * Recover coordinator transactions from persistent store after a restart.
+	 * Called during node startup, before accepting new requests.
+	 */
+	async recoverTransactions(): Promise<void> {
+		if (!this.stateStore) return;
+		const states = await this.stateStore.getAllCoordinatorStates();
+		for (const state of states) {
+			const { messageHash } = state;
+			// Expired — clean up
+			if (state.record.message.expiration && state.record.message.expiration < Date.now()) {
+				log('cluster-tx:recovery-expired', { messageHash });
+				await this.stateStore.deleteCoordinatorState(messageHash);
+				continue;
+			}
+			// Broadcasting phase with retry state — resume retries
+			if (state.phase === 'broadcasting' && state.retryState) {
+				log('cluster-tx:recovery-resume-broadcast', { messageHash, attempt: state.retryState.attempt });
+				const pending = new Pending(Promise.resolve(state.record));
+				const txState: ClusterTransactionState = {
+					messageHash,
+					record: state.record,
+					pending,
+					lastUpdate: state.lastUpdate
+				};
+				this.transactions.set(messageHash, txState);
+				// Schedule retry from where we left off
+				this.scheduleCommitRetry(messageHash, state.record, state.retryState.pendingPeers);
+				continue;
+			}
+			// Promising or committing — cannot resume (caller context is gone)
+			log('cluster-tx:recovery-stale', { messageHash, phase: state.phase });
+			await this.stateStore.deleteCoordinatorState(messageHash);
+		}
+	}
 }

package/src/repo/coordinator-repo.ts

@@ -7,6 +7,7 @@ import { peerIdFromString } from "@libp2p/peer-id";
 import type { FretService } from "p2p-fret";
 import { createLogger } from '../logger.js';
 import type { IPeerReputation } from "../reputation/types.js";
+import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
 
 const log = createLogger('coordinator-repo');
 
@@ -40,7 +41,8 @@ export function coordinatorRepo(
 	createClusterClient: (peerId: PeerId) => ClusterClient,
 	cfg?: Partial<ClusterConsensusConfig> & { clusterSize?: number },
 	fretService?: FretService,
-	reputation?: IPeerReputation
+	reputation?: IPeerReputation,
+	stateStore?: ITransactionStateStore
 ): (components: CoordinatorRepoComponents) => CoordinatorRepo {
 	return (components: CoordinatorRepoComponents) => new CoordinatorRepo(
 		keyNetwork,
@@ -51,7 +53,8 @@ export function coordinatorRepo(
 		components.localPeerId,
 		fretService,
 		components.clusterLatestCallback,
-		reputation
+		reputation,
+		stateStore
 	);
 }
 
@@ -72,7 +75,8 @@ export class CoordinatorRepo implements IRepo {
 		localPeerId?: PeerId,
 		fretService?: FretService,
 		private readonly clusterLatestCallback?: ClusterLatestCallback,
-		reputation?: IPeerReputation
+		reputation?: IPeerReputation,
+		stateStore?: ITransactionStateStore
 	) {
 		this.localPeerId = localPeerId;
 		const policy: ClusterConsensusConfig & { clusterSize: number } = {
@@ -89,7 +93,12 @@ export class CoordinatorRepo implements IRepo {
 			peerId: localPeerId,
 			wasTransactionExecuted: localCluster.wasTransactionExecuted?.bind(localCluster)
 		} : undefined;
-		this.coordinator = new ClusterCoordinator(keyNetwork, createClusterClient, policy, localClusterRef, fretService, reputation);
+		this.coordinator = new ClusterCoordinator(keyNetwork, createClusterClient, policy, localClusterRef, fretService, reputation, stateStore);
+	}
+
+	/** Recover coordinator transactions from persistent store after a restart. */
+	async recoverTransactions(): Promise<void> {
+		await this.coordinator.recoverTransactions();
 	}
 
 	/**

package/src/storage/i-kv-store.ts

@@ -0,0 +1,8 @@
+/** Portable async key-value store. Platform packages provide implementations. */
+export interface IKVStore {
+	get(key: string): Promise<string | undefined>;
+	set(key: string, value: string): Promise<void>;
+	delete(key: string): Promise<void>;
+	/** Return all keys matching the given prefix */
+	list(prefix: string): Promise<string[]>;
+}

package/src/storage/memory-kv-store.ts

@@ -0,0 +1,28 @@
+import type { IKVStore } from "./i-kv-store.js";
+
+/** In-memory IKVStore backed by a Map. Used for testing. */
+export class MemoryKVStore implements IKVStore {
+	private readonly store = new Map<string, string>();
+
+	async get(key: string): Promise<string | undefined> {
+		return this.store.get(key);
+	}
+
+	async set(key: string, value: string): Promise<void> {
+		this.store.set(key, value);
+	}
+
+	async delete(key: string): Promise<void> {
+		this.store.delete(key);
+	}
+
+	async list(prefix: string): Promise<string[]> {
+		const result: string[] = [];
+		for (const key of this.store.keys()) {
+			if (key.startsWith(prefix)) {
+				result.push(key);
+			}
+		}
+		return result;
+	}
+}