@optimystic/db-p2p 0.2.3 → 0.3.0

package/src/network/network-manager-service.ts

@@ -3,6 +3,11 @@ import { peerIdFromString } from '@libp2p/peer-id'
 import type { FretService } from 'p2p-fret'
 import { hashKey } from 'p2p-fret'
 import { toString as u8ToString } from 'uint8arrays/to-string'
+import type { IPeerReputation } from '../reputation/types.js'
+import { PenaltyReason } from '../reputation/types.js'
+import { RebalanceMonitor, type RebalanceMonitorConfig } from '../cluster/rebalance-monitor.js'
+import type { PartitionDetector } from '../cluster/partition-detector.js'
+import type { ArachnodeFretAdapter } from '../storage/arachnode-fret-adapter.js'
 
 export type NetworkManagerServiceInit = {
 	clusterSize?: number
@@ -33,9 +38,9 @@ export class NetworkManagerService implements Startable {
 	private readonly coordinatorCache = new Map<string, { id: PeerId, expires: number }>()
 	private readonly clusterCache = new Map<string, { ids: PeerId[], expires: number }>()
 	private lastEstimate: { estimate: number, samples: number, updated: number } | null = null
-	// lightweight blacklist (local reputation)
-	private readonly blacklist = new Map<string, { score: number, expires: number }>()
+	private reputation?: IPeerReputation
 	private libp2pRef: Libp2p | undefined
+	private rebalanceMonitor?: RebalanceMonitor
 
 	constructor(private readonly components: Components, init: NetworkManagerServiceInit = {}) {
 		this.log = components.logger.forComponent('db-p2p:network-manager')
@@ -55,6 +60,34 @@ export class NetworkManagerService implements Startable {
 		this.libp2pRef = libp2p;
 	}
 
+	setReputation(reputation: IPeerReputation): void {
+		this.reputation = reputation;
+	}
+
+	/**
+	 * Initialize the rebalance monitor. Call after libp2p, FRET, and adapter are available.
+	 */
+	initRebalanceMonitor(
+		partitionDetector: PartitionDetector,
+		fretAdapter: ArachnodeFretAdapter,
+		config?: RebalanceMonitorConfig
+	): RebalanceMonitor {
+		const libp2p = this.getLibp2p()
+		const fret = this.getFret()
+		if (!libp2p || !fret) {
+			throw new Error('Cannot init RebalanceMonitor: libp2p or FRET not available')
+		}
+		this.rebalanceMonitor = new RebalanceMonitor(
+			{ libp2p, fret, partitionDetector, fretAdapter },
+			config
+		)
+		return this.rebalanceMonitor
+	}
+
+	getRebalanceMonitor(): RebalanceMonitor | undefined {
+		return this.rebalanceMonitor
+	}
+
 	private getLibp2p(): Libp2p | undefined {
 		return this.libp2pRef ?? this.components.libp2p;
 	}
@@ -77,6 +110,9 @@ export class NetworkManagerService implements Startable {
 	}
 
 	async stop(): Promise<void> {
+		if (this.rebalanceMonitor) {
+			await this.rebalanceMonitor.stop()
+		}
 		this.running = false
 	}
 
@@ -162,23 +198,14 @@ export class NetworkManagerService implements Startable {
 	}
 
 	/**
-	 * Record a misbehaving peer. Higher score means worse reputation.
-	 * Entries expire to allow eventual forgiveness.
+	 * Record a misbehaving peer. Delegates to IPeerReputation if available.
 	 */
-	reportBadPeer(peerId: PeerId, penalty: number = 1, ttlMs: number = 10 * 60_000): void {
-		const id = peerId.toString()
-		const prev = this.blacklist.get(id)
-		const score = (prev?.score ?? 0) + Math.max(1, penalty)
-		this.blacklist.set(id, { score, expires: Date.now() + ttlMs })
+	reportBadPeer(peerId: PeerId, reason: PenaltyReason = PenaltyReason.ConnectionFailure): void {
+		this.reputation?.reportPeer(peerId.toString(), reason)
 	}
 
 	private isBlacklisted(peerId: PeerId): boolean {
-		const id = peerId.toString()
-		const rec = this.blacklist.get(id)
-		if (!rec) return false
-		if (rec.expires <= Date.now()) { this.blacklist.delete(id); return false }
-		// simple threshold; can be tuned or exposed later
-		return rec.score >= 3
+		return this.reputation?.isBanned(peerId.toString()) ?? false
 	}
 
 	recordCoordinator(key: Uint8Array, peerId: PeerId): void {
@@ -298,7 +325,11 @@ export class NetworkManagerService implements Startable {
 		if (!libp2p) {
 			throw new Error('Libp2p not initialized');
 		}
-		const candidate = cluster.find(p => !this.isBlacklisted(p)) ?? libp2p.peerId;
+		// Prefer non-banned, non-deprioritized peers; fall back to deprioritized before self
+		const candidate = cluster
+			.filter(p => !this.isBlacklisted(p))
+			.sort((a, b) => (this.reputation?.getScore(a.toString()) ?? 0) - (this.reputation?.getScore(b.toString()) ?? 0))
+			[0] ?? libp2p.peerId;
 		this.recordCoordinator(key, candidate);
 		return candidate;
 	}

package/src/protocol-client.ts

@@ -3,6 +3,9 @@ import { encode as lpEncode, decode as lpDecode } from 'it-length-prefixed';
 import type { Stream as Libp2pStream } from '@libp2p/interface';
 import type { PeerId, IPeerNetwork } from '@optimystic/db-core';
 import { first } from './it-utility.js';
+import { createLogger } from './logger.js';
+
+const log = createLogger('protocol-client');
 
 /** Base class for clients that communicate via a libp2p protocol */
 export class ProtocolClient {
@@ -14,13 +17,25 @@ export class ProtocolClient {
 	protected async processMessage<T>(
 		message: unknown,
 		protocol: string,
-		options?: { signal?: AbortSignal }
+		options?: { signal?: AbortSignal; correlationId?: string }
 	): Promise<T> {
-		const stream = await this.peerNetwork.connect(
-			this.peerId,
-			protocol,
-			{ signal: options?.signal }
-		) as unknown as Libp2pStream;
+		const peer = this.peerId.toString();
+		const cid = options?.correlationId;
+		log('dial peer=%s protocol=%s%s', peer, protocol, cid ? ` cid=${cid}` : '');
+		const t0 = Date.now();
+
+		let stream: Libp2pStream;
+		try {
+			stream = await this.peerNetwork.connect(
+				this.peerId,
+				protocol,
+				{ signal: options?.signal }
+			) as unknown as Libp2pStream;
+		} catch (err) {
+			log('dial:fail peer=%s protocol=%s ms=%d%s', peer, protocol, Date.now() - t0, cid ? ` cid=${cid}` : '');
+			throw err;
+		}
+		log('dial:ok peer=%s ms=%d%s', peer, Date.now() - t0, cid ? ` cid=${cid}` : '');
 
 		try {
 			// Send the request using length-prefixed encoding
@@ -33,11 +48,16 @@ export class ProtocolClient {
 			}
 
 			// Read the response from the stream (which is now directly AsyncIterable)
+			let firstByte = true;
 			const source = pipe(
 				stream,
 				lpDecode,
 				async function* (source) {
 					for await (const data of source) {
+						if (firstByte) {
+							log('first-byte peer=%s ms=%d%s', peer, Date.now() - t0, cid ? ` cid=${cid}` : '');
+							firstByte = false;
+						}
 						const decoded = new TextDecoder().decode(data.subarray());
 						const parsed = JSON.parse(decoded);
 						yield parsed;
@@ -45,7 +65,9 @@ export class ProtocolClient {
 				}
 			) as AsyncIterable<T>;
 
-			return await first(() => source, () => { throw new Error('No response received') });
+			const result = await first(() => source, () => { throw new Error('No response received') });
+			log('response peer=%s protocol=%s ms=%d%s', peer, protocol, Date.now() - t0, cid ? ` cid=${cid}` : '');
+			return result;
 		} finally {
 			await stream.close();
 		}

package/src/repo/client.ts

@@ -44,6 +44,15 @@ export class RepoClient extends ProtocolClient implements IRepo {
 		);
 	}
 
+	private extractCorrelationId(operations: RepoMessage['operations']): string | undefined {
+		const op = operations[0];
+		if (!op) return undefined;
+		if ('pend' in op) return op.pend.actionId;
+		if ('commit' in op) return op.commit.actionId;
+		if ('cancel' in op) return op.cancel.actionRef.actionId;
+		return undefined;
+	}
+
 	private async processRepoMessage<T>(
 		operations: RepoMessage['operations'],
 		options: MessageOptions,
@@ -53,6 +62,7 @@ export class RepoClient extends ProtocolClient implements IRepo {
 			operations,
 			expiration: options.expiration,
 		};
+		const correlationId = this.extractCorrelationId(operations);
 		const deadline = options.expiration ?? (Date.now() + 30_000)
 		const msLeft = Math.max(1, deadline - Date.now())
 		const withTimeout = async <U>(fn: () => Promise<U>): Promise<U> => {
@@ -63,7 +73,7 @@ export class RepoClient extends ProtocolClient implements IRepo {
 		}
 		let response: any
 		const preferred = (this.protocolPrefix ?? '/db-p2p') + '/repo/1.0.0'
-		response = await withTimeout(() => super.processMessage<any>(message, preferred, { signal: options?.signal }))
+		response = await withTimeout(() => super.processMessage<any>(message, preferred, { signal: options?.signal, correlationId }))
 
 		if (response?.redirect?.peers?.length) {
 			if (hop >= 2) {
@@ -75,11 +85,11 @@ export class RepoClient extends ProtocolClient implements IRepo {
 			if (next.id === currentIdStr) {
 				throw new Error('Redirect loop detected in RepoClient (same peer)')
 			}
-		// cache hint
-		this.recordCoordinatorForOpsIfSupported(operations, nextId)
-		// single-hop retry against target peer using repo protocol
-		const nextClient = RepoClient.create(nextId, this.peerNetwork, this.protocolPrefix)
-		return await nextClient.processRepoMessage<T>(operations, options, hop + 1)
+			// cache hint
+			this.recordCoordinatorForOpsIfSupported(operations, nextId)
+			// single-hop retry against target peer using repo protocol
+			const nextClient = RepoClient.create(nextId, this.peerNetwork, this.protocolPrefix)
+			return await nextClient.processRepoMessage<T>(operations, options, hop + 1)
 		}
 		return response as T;
 	}
@@ -97,6 +107,10 @@ export class RepoClient extends ProtocolClient implements IRepo {
 		if ('commit' in op) {
 			return new TextEncoder().encode(op.commit.tailId);
 		}
+		if ('cancel' in op) {
+			const id = op.cancel.actionRef.blockIds[0];
+			return id ? new TextEncoder().encode(id) : undefined;
+		}
 		return undefined;
 	}
 

package/src/repo/cluster-coordinator.ts

@@ -5,9 +5,11 @@ import { sha256 } from "multiformats/hashes/sha2";
 import { ClusterClient } from "../cluster/client.js";
 import { Pending } from "@optimystic/db-core";
 import type { PeerId } from "@libp2p/interface";
-import { createLogger } from '../logger.js'
+import { createLogger, verbose } from '../logger.js'
 import type { ClusterLogPeerOutcome } from './types.js'
 import type { FretService } from "p2p-fret";
+import type { IPeerReputation } from "../reputation/types.js";
+import { PenaltyReason } from "../reputation/types.js";
 
 const log = createLogger('cluster')
 
@@ -49,7 +51,8 @@ export class ClusterCoordinator {
 			peerId: PeerId;
 			wasTransactionExecuted?: (messageHash: string) => boolean;
 		},
-		private readonly fretService?: FretService
+		private readonly fretService?: FretService,
+		private readonly reputation?: IPeerReputation
 	) { }
 
 	/**
@@ -248,6 +251,28 @@ export class ClusterCoordinator {
 			throw new Error(`Failed to get super-majority: ${approvalCount}/${peerCount} approvals (needed ${superMajority}, ${rejectionCount} rejections)`);
 		}
 
+		// Mark as disputed when minority rejections exist but super-majority approves
+		if (rejectionCount > 0 && approvalCount >= superMajority) {
+			const rejectingPeers: string[] = [];
+			const rejectReasons: { [peerId: string]: string } = {};
+			for (const [peerId, sig] of Object.entries(promises)) {
+				if (sig.type === 'reject') {
+					rejectingPeers.push(peerId);
+					rejectReasons[peerId] = sig.rejectReason ?? 'unknown';
+				}
+			}
+			promised.record.disputed = true;
+			promised.record.disputeEvidence = { rejectingPeers, rejectReasons };
+			log('cluster-tx:disputed', {
+				messageHash: record.messageHash,
+				rejectingPeers,
+				rejectReasons,
+				approvalCount,
+				rejectionCount,
+				peerCount
+			});
+		}
+
 		return await this.commitTransaction(promised.record);
 	}
 
@@ -301,6 +326,13 @@ export class ClusterCoordinator {
 	private async collectPromises(peers: ClusterPeers, record: ClusterRecord): Promise<{ record: ClusterRecord }> {
 		const peerIds = Object.keys(peers);
 		const summary: ClusterLogPeerOutcome[] = [];
+		if (verbose) {
+			const peerDetail = peerIds.map(id => ({
+				id: id.substring(0, 12),
+				addrs: peers[id]?.multiaddrs?.length ?? 0
+			}));
+			log('cluster-tx:promise-peers', { messageHash: record.messageHash, peers: peerDetail });
+		}
 		// For each peer, create a client and request a promise
 		const promiseRequests = peerIds.map(peerIdStr => {
 			const isLocal = this.localCluster && peerIdStr === this.localCluster.peerId.toString();
@@ -327,6 +359,7 @@ export class ClusterCoordinator {
 			const peerIdStr = peerIds[idx]!;
 			log('cluster-tx:promise-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
 			summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
+			this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `promise:${record.messageHash}`);
 			return null;
 		})));
 		const successes = summary.filter(entry => entry.success).map(entry => entry.peerId);
@@ -390,6 +423,13 @@ export class ClusterCoordinator {
 		// For each peer, create a client and send the commit
 		const peerIds = Object.keys(record.peers);
 		const summary: ClusterLogPeerOutcome[] = [];
+		if (verbose) {
+			const peerDetail = peerIds.map(id => ({
+				id: id.substring(0, 12),
+				addrs: record.peers[id]?.multiaddrs?.length ?? 0
+			}));
+			log('cluster-tx:commit-peers', { messageHash: record.messageHash, peers: peerDetail });
+		}
 		// Send the record with promises to all peers
 		// Each peer will add its own commit signature
 		const commitPayload = {
@@ -414,6 +454,7 @@ export class ClusterCoordinator {
 			const peerIdStr = peerIds[idx]!;
 			log('cluster-tx:commit-response', { messageHash: record.messageHash, peerId: peerIdStr, success: false, error: err });
 			summary.push({ peerId: peerIdStr, success: false, error: err instanceof Error ? err.message : String(err) });
+			this.reputation?.reportPeer(peerIdStr, PenaltyReason.ConsensusTimeout, `commit:${record.messageHash}`);
 			return null;
 		})));
 		const commitSuccesses = summary.filter(entry => entry.success).map(entry => entry.peerId);

package/src/repo/coordinator-repo.ts

@@ -1,10 +1,12 @@
 import type { PendRequest, ActionBlocks, IRepo, MessageOptions, CommitResult, GetBlockResults, PendResult, BlockGets, CommitRequest, RepoMessage, IKeyNetwork, ICluster, ClusterConsensusConfig, BlockId, ActionRev } from "@optimystic/db-core";
+import { LruMap } from "@optimystic/db-core";
 import { ClusterCoordinator } from "./cluster-coordinator.js";
 import type { ClusterClient } from "../cluster/client.js";
 import type { PeerId } from "@libp2p/interface";
 import { peerIdFromString } from "@libp2p/peer-id";
 import type { FretService } from "p2p-fret";
 import { createLogger } from '../logger.js';
+import type { IPeerReputation } from "../reputation/types.js";
 
 const log = createLogger('coordinator-repo');
 
@@ -37,7 +39,8 @@ export function coordinatorRepo(
 	keyNetwork: IKeyNetwork,
 	createClusterClient: (peerId: PeerId) => ClusterClient,
 	cfg?: Partial<ClusterConsensusConfig> & { clusterSize?: number },
-	fretService?: FretService
+	fretService?: FretService,
+	reputation?: IPeerReputation
 ): (components: CoordinatorRepoComponents) => CoordinatorRepo {
 	return (components: CoordinatorRepoComponents) => new CoordinatorRepo(
 		keyNetwork,
@@ -47,7 +50,8 @@ export function coordinatorRepo(
 		components.localCluster,
 		components.localPeerId,
 		fretService,
-		components.clusterLatestCallback
+		components.clusterLatestCallback,
+		reputation
 	);
 }
 
@@ -55,6 +59,9 @@ export function coordinatorRepo(
 export class CoordinatorRepo implements IRepo {
 	private coordinator: ClusterCoordinator;
 	private readonly DEFAULT_TIMEOUT = 30000; // 30 seconds default timeout
+	private readonly localPeerId?: PeerId;
+	private readonly responsibilityCache = new LruMap<string, { inCluster: boolean, expires: number }>(1000);
+	private static readonly RESPONSIBILITY_TTL_MS = 60_000;
 
 	constructor(
 		readonly keyNetwork: IKeyNetwork,
@@ -64,8 +71,10 @@ export class CoordinatorRepo implements IRepo {
 		localCluster?: LocalClusterWithExecutionTracking,
 		localPeerId?: PeerId,
 		fretService?: FretService,
-		private readonly clusterLatestCallback?: ClusterLatestCallback
+		private readonly clusterLatestCallback?: ClusterLatestCallback,
+		reputation?: IPeerReputation
 	) {
+		this.localPeerId = localPeerId;
 		const policy: ClusterConsensusConfig & { clusterSize: number } = {
 			clusterSize: cfg?.clusterSize ?? 10,
 			superMajorityThreshold: cfg?.superMajorityThreshold ?? 0.75,
@@ -80,10 +89,64 @@ export class CoordinatorRepo implements IRepo {
 			peerId: localPeerId,
 			wasTransactionExecuted: localCluster.wasTransactionExecuted?.bind(localCluster)
 		} : undefined;
-		this.coordinator = new ClusterCoordinator(keyNetwork, createClusterClient, policy, localClusterRef, fretService);
+		this.coordinator = new ClusterCoordinator(keyNetwork, createClusterClient, policy, localClusterRef, fretService, reputation);
+	}
+
+	/**
+	 * Check if this node is in the cluster for a given block.
+	 * Uses findCluster membership — in the real network layer, self is always
+	 * included in the cohort when this node is responsible. This serves as a
+	 * defense-in-depth guard for requests that arrive at the wrong node.
+	 * Returns true if localPeerId is not set (backward compat for single-node/test setups).
+	 */
+	private async isResponsibleForBlock(blockId: BlockId): Promise<boolean> {
+		if (!this.localPeerId) return true;
+
+		const cached = this.responsibilityCache.get(blockId);
+		if (cached && cached.expires > Date.now()) {
+			return cached.inCluster;
+		}
+
+		const blockIdBytes = new TextEncoder().encode(blockId);
+		let inCluster: boolean;
+		try {
+			const peers = await this.keyNetwork.findCluster(blockIdBytes);
+			inCluster = this.localPeerId.toString() in peers;
+		} catch (err) {
+			log('proximity:check-error', { blockId, error: (err as Error).message });
+			// On failure, assume responsible to avoid false rejections
+			return true;
+		}
+
+		this.responsibilityCache.set(blockId, { inCluster, expires: Date.now() + CoordinatorRepo.RESPONSIBILITY_TTL_MS });
+		log('proximity:checked', { blockId, inCluster });
+		return inCluster;
+	}
+
+	/**
+	 * Verify this node is responsible for all given block IDs. Throws if not.
+	 */
+	private async verifyResponsibility(blockIds: BlockId[]): Promise<void> {
+		const notResponsible: BlockId[] = [];
+		for (const blockId of blockIds) {
+			if (!await this.isResponsibleForBlock(blockId)) {
+				notResponsible.push(blockId);
+			}
+		}
+		if (notResponsible.length > 0) {
+			log('proximity:rejected', { blockIds: notResponsible });
+			throw new Error(`Not responsible for block(s): ${notResponsible.join(', ')}`);
+		}
 	}
 
 	async get(blockGets: BlockGets, options?: MessageOptions): Promise<GetBlockResults> {
+		// Soft proximity check — warn but still serve reads for graceful degradation
+		for (const blockId of blockGets.blockIds) {
+			if (!await this.isResponsibleForBlock(blockId)) {
+				log('proximity:get-warning', { blockId, msg: 'serving read for non-responsible block' });
+			}
+		}
+
 		// First try local storage
 		const localResult = await this.storageRepo.get(blockGets, options);
 
@@ -166,6 +229,7 @@ export class CoordinatorRepo implements IRepo {
 
 	async pend(request: PendRequest, options?: MessageOptions): Promise<PendResult> {
 		const allBlockIds = Object.keys(request.transforms);
+		await this.verifyResponsibility(allBlockIds);
 		const coordinatingBlockIds = (options as any)?.coordinatingBlockIds ?? allBlockIds;
 
 		const peerCount = await this.coordinator.getClusterSize(coordinatingBlockIds[0]!);
@@ -209,10 +273,8 @@ export class CoordinatorRepo implements IRepo {
 	}
 
 	async cancel(actionRef: ActionBlocks, options?: MessageOptions): Promise<void> {
-		// TODO: Verify that we are a proximate node for all block IDs in the request
-
-		// Extract all block IDs affected by this cancel operation
 		const blockIds = actionRef.blockIds;
+		await this.verifyResponsibility(blockIds);
 
 		// Create a message for this cancel operation with timeout
 		const message: RepoMessage = {
@@ -242,6 +304,7 @@ export class CoordinatorRepo implements IRepo {
 
 	async commit(request: CommitRequest, options?: MessageOptions): Promise<CommitResult> {
 		const blockIds = request.blockIds;
+		await this.verifyResponsibility(blockIds);
 
 		const peerCount = await this.coordinator.getClusterSize(blockIds[0]!);
 		if (peerCount <= 1) {

package/src/repo/redirect.ts

@@ -13,5 +13,3 @@ export function encodePeers(peers: Array<{ id: string, addrs: string[] }>): Redi
     }
   }
 }
-
-