@optimystic/db-p2p 0.2.3 → 0.3.0

package/src/cluster/cluster-repo.ts

@@ -1,16 +1,19 @@
-import type { IRepo, ClusterRecord, Signature, RepoMessage, ITransactionValidator } from "@optimystic/db-core";
+import type { IRepo, ClusterRecord, Signature, RepoMessage, ITransactionValidator, ClusterConsensusConfig } from "@optimystic/db-core";
 import type { ICluster } from "@optimystic/db-core";
 import type { IPeerNetwork } from "@optimystic/db-core";
 import { blockIdsForTransforms } from "@optimystic/db-core";
 import { ClusterClient } from "./client.js";
-import type { PeerId } from "@libp2p/interface";
+import type { PeerId, PrivateKey } from "@libp2p/interface";
 import { peerIdFromString } from "@libp2p/peer-id";
+import { publicKeyFromRaw } from "@libp2p/crypto/keys";
 import { sha256 } from "multiformats/hashes/sha2";
 import { base58btc } from "multiformats/bases/base58";
-import { toString as uint8ArrayToString } from 'uint8arrays/to-string';
+import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } from 'uint8arrays';
 import { createLogger } from '../logger.js'
 import type { PartitionDetector } from "./partition-detector.js";
 import type { FretService } from "p2p-fret";
+import type { IPeerReputation } from "../reputation/types.js";
+import { PenaltyReason } from "../reputation/types.js";
 
 const log = createLogger('cluster-member')
 
@@ -35,10 +38,13 @@ interface ClusterMemberComponents {
 	storageRepo: IRepo;
 	peerNetwork: IPeerNetwork;
 	peerId: PeerId;
+	privateKey: PrivateKey;
 	protocolPrefix?: string;
 	partitionDetector?: PartitionDetector;
 	fretService?: FretService;
 	validator?: ITransactionValidator;
+	reputation?: IPeerReputation;
+	consensusConfig?: ClusterConsensusConfig;
 }
 
 export function clusterMember(components: ClusterMemberComponents): ClusterMember {
@@ -46,10 +52,13 @@ export function clusterMember(components: ClusterMemberComponents): ClusterMembe
 		components.storageRepo,
 		components.peerNetwork,
 		components.peerId,
+		components.privateKey,
 		components.protocolPrefix,
 		components.partitionDetector,
 		components.fretService,
-		components.validator
+		components.validator,
+		components.reputation,
+		components.consensusConfig
 	);
 }
 
@@ -69,20 +78,29 @@ export class ClusterMember implements ICluster {
 	private cleanupQueue: string[] = [];
 	// Serialize concurrent updates for the same transaction
 	private pendingUpdates: Map<string, Promise<ClusterRecord>> = new Map();
+	// Temporarily set during validateSignatures so verifySignature can access the record
+	private currentValidationRecord?: ClusterRecord;
+
+	/** Effective super-majority threshold. Defaults to 1.0 (unanimity) for backward compatibility. */
+	private readonly superMajorityThreshold: number;
 
 	constructor(
 		private readonly storageRepo: IRepo,
 		private readonly peerNetwork: IPeerNetwork,
 		private readonly peerId: PeerId,
+		private readonly privateKey: PrivateKey,
 		private readonly protocolPrefix?: string,
 		private readonly partitionDetector?: PartitionDetector,
 		private readonly fretService?: FretService,
-		private readonly validator?: ITransactionValidator
+		private readonly validator?: ITransactionValidator,
+		private readonly reputation?: IPeerReputation,
+		consensusConfig?: ClusterConsensusConfig
 	) {
-		// Periodically clean up expired transactions
-		setInterval(() => this.queueExpiredTransactions(), 60000);
+		this.superMajorityThreshold = consensusConfig?.superMajorityThreshold ?? 1.0;
+		// Periodically clean up expired transactions (.unref() so tests/short-lived processes can exit)
+		setInterval(() => this.queueExpiredTransactions(), 60000).unref();
 		// Process cleanup queue
-		setInterval(() => this.processCleanupQueue(), 1000);
+		setInterval(() => this.processCleanupQueue(), 1000).unref();
 	}
 
 	/**
@@ -118,7 +136,7 @@ export class ClusterMember implements ICluster {
 			// Remove from pending updates after a short delay to allow concurrent calls to see it
 			setTimeout(() => {
 				this.pendingUpdates.delete(record.messageHash);
-			}, 100);
+			}, 100).unref();
 		}
 	}
 
@@ -344,20 +362,27 @@ export class ClusterMember implements ICluster {
 	}
 
 	private async validateSignatures(record: ClusterRecord): Promise<void> {
-		// Validate promise signatures
-		const promiseHash = await this.computePromiseHash(record);
-		for (const [peerId, signature] of Object.entries(record.promises)) {
-			if (!await this.verifySignature(peerId, promiseHash, signature)) {
-				throw new Error(`Invalid promise signature from ${peerId}`);
+		this.currentValidationRecord = record;
+		try {
+			// Validate promise signatures
+			const promiseHash = await this.computePromiseHash(record);
+			for (const [peerId, signature] of Object.entries(record.promises)) {
+				if (!await this.verifySignature(peerId, promiseHash, signature)) {
+					this.reputation?.reportPeer(peerId, PenaltyReason.InvalidSignature, `promise:${record.messageHash}`);
+					throw new Error(`Invalid promise signature from ${peerId}`);
+				}
 			}
-		}
 
-		// Validate commit signatures
-		const commitHash = await this.computeCommitHash(record);
-		for (const [peerId, signature] of Object.entries(record.commits)) {
-			if (!await this.verifySignature(peerId, commitHash, signature)) {
-				throw new Error(`Invalid commit signature from ${peerId}`);
+			// Validate commit signatures
+			const commitHash = await this.computeCommitHash(record);
+			for (const [peerId, signature] of Object.entries(record.commits)) {
+				if (!await this.verifySignature(peerId, commitHash, signature)) {
+					this.reputation?.reportPeer(peerId, PenaltyReason.InvalidSignature, `commit:${record.messageHash}`);
+					throw new Error(`Invalid commit signature from ${peerId}`);
+				}
 			}
+		} finally {
+			this.currentValidationRecord = undefined;
 		}
 	}
 
@@ -373,21 +398,40 @@ export class ClusterMember implements ICluster {
 		return uint8ArrayToString(hashBytes.digest, 'base64url');
 	}
 
+	private computeSigningPayload(hash: string, type: string, rejectReason?: string): Uint8Array {
+		const payload = hash + ':' + type + (rejectReason ? ':' + rejectReason : '');
+		return new TextEncoder().encode(payload);
+	}
+
+	private async signVote(hash: string, type: 'approve' | 'reject', rejectReason?: string): Promise<string> {
+		const payload = this.computeSigningPayload(hash, type, rejectReason);
+		const sigBytes = await this.privateKey.sign(payload);
+		return uint8ArrayToString(sigBytes, 'base64url');
+	}
+
 	private async verifySignature(peerId: string, hash: string, signature: Signature): Promise<boolean> {
-		// TODO: Implement actual signature verification
-		return true;
+		const peerInfo = this.currentValidationRecord?.peers[peerId];
+		if (!peerInfo?.publicKey?.length) {
+			throw new Error(`No public key for peer ${peerId}`);
+		}
+		const pubKey = publicKeyFromRaw(peerInfo.publicKey);
+		const payload = this.computeSigningPayload(hash, signature.type, signature.rejectReason);
+		const sigBytes = uint8ArrayFromString(signature.signature, 'base64url');
+		return pubKey.verify(payload, sigBytes);
 	}
 
 	private async getTransactionPhase(record: ClusterRecord): Promise<TransactionPhase> {
 		const peerCount = Object.keys(record.peers).length;
 		const promiseCount = Object.keys(record.promises).length;
-		const commitCount = Object.keys(record.commits).length;
 		const ourId = this.peerId.toString();
 
-		// Check for rejections
+		const superMajority = Math.ceil(peerCount * this.superMajorityThreshold);
+		const maxAllowedRejections = peerCount - superMajority;
+
+		// Check for rejections — rejected if too many rejections to ever reach super-majority
 		const rejectedPromises = Object.values(record.promises).filter(s => s.type === 'reject');
 		const rejectedCommits = Object.values(record.commits).filter(s => s.type === 'reject');
-		if (rejectedPromises.length > 0 || this.hasMajority(rejectedCommits.length, peerCount)) {
+		if (rejectedPromises.length > maxAllowedRejections || this.hasMajority(rejectedCommits.length, peerCount)) {
 			return TransactionPhase.Rejected;
 		}
 
@@ -396,14 +440,15 @@ export class ClusterMember implements ICluster {
 			return TransactionPhase.OurPromiseNeeded;
 		}
 
-		// Check if still collecting promises
-		if (promiseCount < peerCount) {
-			return TransactionPhase.Promising;
+		// Check if we have enough approved promises to proceed to commit
+		const approvedPromises = Object.values(record.promises).filter(s => s.type === 'approve');
+		if (approvedPromises.length >= superMajority && !record.commits[ourId]) {
+			return TransactionPhase.OurCommitNeeded;
 		}
 
-		// Check if we need to commit
-		if (promiseCount === peerCount && !record.commits[ourId]) {
-			return TransactionPhase.OurCommitNeeded;
+		// Check if still collecting promises
+		if (promiseCount < peerCount && approvedPromises.length < superMajority) {
+			return TransactionPhase.Promising;
 		}
 
 		// Check for consensus
@@ -423,9 +468,14 @@ export class ClusterMember implements ICluster {
 		// Validate pend operations if we have a validator
 		const validationResult = await this.validatePendOperations(record);
 
+		const promiseHash = await this.computePromiseHash(record);
+		const type = validationResult.valid ? 'approve' as const : 'reject' as const;
+		const rejectReason = validationResult.valid ? undefined : validationResult.reason;
+		const sig = await this.signVote(promiseHash, type, rejectReason);
+
 		const signature: Signature = validationResult.valid
-			? { type: 'approve', signature: 'approved' }
-			: { type: 'reject', signature: 'rejected', rejectReason: validationResult.reason };
+			? { type: 'approve', signature: sig }
+			: { type: 'reject', signature: sig, rejectReason };
 
 		if (!validationResult.valid) {
 			log('cluster-member:validation-rejected', {
@@ -491,9 +541,11 @@ export class ClusterMember implements ICluster {
 		if (this.hasLocalCommit(record)) {
 			return record;
 		}
+		const commitHash = await this.computeCommitHash(record);
+		const sig = await this.signVote(commitHash, 'approve');
 		const signature: Signature = {
 			type: 'approve',
-			signature: 'committed' // TODO: Actually sign the commit hash
+			signature: sig
 		};
 
 		return {
@@ -584,11 +636,11 @@ export class ClusterMember implements ICluster {
 			promiseTimeout: setTimeout(
 				() => this.handleExpiration(record.messageHash),
 				record.message.expiration - Date.now()
-			),
+			).unref(),
 			resolutionTimeout: setTimeout(
 				() => this.resolveWithPeers(record.messageHash),
 				record.message.expiration + 5000 - Date.now()
-			)
+			).unref()
 		};
 	}
 
@@ -749,10 +801,13 @@ export class ClusterMember implements ICluster {
 		if (!state) return;
 
 		if (!state.record.promises[this.peerId.toString()]) {
+			const rejectReason = 'Transaction expired';
+			const promiseHash = await this.computePromiseHash(state.record);
+			const sig = await this.signVote(promiseHash, 'reject', rejectReason);
 			const signature: Signature = {
 				type: 'reject',
-				signature: 'rejected',
-				rejectReason: 'Transaction expired'
+				signature: sig,
+				rejectReason
 			};
 
 			const updatedRecord = {

package/src/cluster/rebalance-monitor.ts

@@ -0,0 +1,225 @@
+import type { Startable, Libp2p, PeerId } from '@libp2p/interface'
+import { hashKey } from 'p2p-fret'
+import type { FretService } from 'p2p-fret'
+import type { PartitionDetector } from './partition-detector.js'
+import type { ArachnodeFretAdapter, ArachnodeInfo } from '../storage/arachnode-fret-adapter.js'
+import { createLogger } from '../logger.js'
+
+const log = createLogger('rebalance-monitor')
+
+export interface RebalanceEvent {
+	/** Block IDs this node has gained responsibility for */
+	gained: string[]
+	/** Block IDs this node has lost responsibility for */
+	lost: string[]
+	/** Peers that are now closer for the lost blocks: blockId → peerId[] */
+	newOwners: Map<string, string[]>
+	/** Timestamp of the topology change that triggered this */
+	triggeredAt: number
+}
+
+export interface RebalanceMonitorConfig {
+	/** Debounce window for topology changes (ms). Default: 5000 */
+	debounceMs?: number
+	/** Maximum frequency of full rebalance scans (ms). Default: 60000 */
+	minRebalanceIntervalMs?: number
+	/** Whether to suppress rebalancing during detected partitions. Default: true */
+	suppressDuringPartition?: boolean
+}
+
+export interface RebalanceMonitorDeps {
+	libp2p: Libp2p
+	fret: FretService
+	partitionDetector: PartitionDetector
+	fretAdapter: ArachnodeFretAdapter
+}
+
+type RebalanceHandler = (event: RebalanceEvent) => void
+
+export class RebalanceMonitor implements Startable {
+	private running = false
+	private readonly trackedBlocks = new Set<string>()
+	private readonly responsibilitySnapshot = new Map<string, boolean>()
+	private readonly handlers: RebalanceHandler[] = []
+	private debounceTimer: ReturnType<typeof setTimeout> | null = null
+	private lastRebalanceAt = 0
+	private pendingTopologyChange = false
+	private topologyChangeTimestamp = 0
+
+	private readonly debounceMs: number
+	private readonly minRebalanceIntervalMs: number
+	private readonly suppressDuringPartition: boolean
+
+	private readonly onConnectionOpen: () => void
+	private readonly onConnectionClose: () => void
+
+	constructor(
+		private readonly deps: RebalanceMonitorDeps,
+		config: RebalanceMonitorConfig = {}
+	) {
+		this.debounceMs = config.debounceMs ?? 5000
+		this.minRebalanceIntervalMs = config.minRebalanceIntervalMs ?? 60000
+		this.suppressDuringPartition = config.suppressDuringPartition ?? true
+
+		this.onConnectionOpen = () => this.handleTopologyChange()
+		this.onConnectionClose = () => this.handleTopologyChange()
+	}
+
+	async start(): Promise<void> {
+		if (this.running) return
+		this.running = true
+
+		this.deps.libp2p.addEventListener('connection:open', this.onConnectionOpen)
+		this.deps.libp2p.addEventListener('connection:close', this.onConnectionClose)
+
+		log('started, tracking %d blocks', this.trackedBlocks.size)
+	}
+
+	async stop(): Promise<void> {
+		if (!this.running) return
+		this.running = false
+
+		this.deps.libp2p.removeEventListener('connection:open', this.onConnectionOpen)
+		this.deps.libp2p.removeEventListener('connection:close', this.onConnectionClose)
+
+		if (this.debounceTimer) {
+			clearTimeout(this.debounceTimer)
+			this.debounceTimer = null
+		}
+
+		this.pendingTopologyChange = false
+		log('stopped')
+	}
+
+	onRebalance(handler: RebalanceHandler): void {
+		this.handlers.push(handler)
+	}
+
+	trackBlock(blockId: string): void {
+		this.trackedBlocks.add(blockId)
+	}
+
+	untrackBlock(blockId: string): void {
+		this.trackedBlocks.delete(blockId)
+		this.responsibilitySnapshot.delete(blockId)
+	}
+
+	getTrackedBlockCount(): number {
+		return this.trackedBlocks.size
+	}
+
+	async checkNow(): Promise<RebalanceEvent | null> {
+		return this.performRebalanceCheck(Date.now())
+	}
+
+	private handleTopologyChange(): void {
+		if (!this.running) return
+
+		if (!this.pendingTopologyChange) {
+			this.topologyChangeTimestamp = Date.now()
+		}
+		this.pendingTopologyChange = true
+
+		if (this.debounceTimer) {
+			clearTimeout(this.debounceTimer)
+		}
+
+		this.debounceTimer = setTimeout(() => {
+			this.debounceTimer = null
+			this.pendingTopologyChange = false
+			this.maybeRebalance()
+		}, this.debounceMs)
+	}
+
+	private async maybeRebalance(): Promise<void> {
+		if (!this.running) return
+
+		const now = Date.now()
+		const elapsed = now - this.lastRebalanceAt
+		if (elapsed < this.minRebalanceIntervalMs) {
+			log('throttled, %dms since last rebalance', elapsed)
+			return
+		}
+
+		const event = await this.performRebalanceCheck(this.topologyChangeTimestamp || now)
+		if (event) {
+			this.emitEvent(event)
+		}
+	}
+
+	private async performRebalanceCheck(triggeredAt: number): Promise<RebalanceEvent | null> {
+		if (this.suppressDuringPartition && this.deps.partitionDetector.detectPartition()) {
+			log('partition detected, suppressing rebalance')
+			return null
+		}
+
+		if (this.trackedBlocks.size === 0) {
+			this.lastRebalanceAt = Date.now()
+			return null
+		}
+
+		const selfId = this.deps.libp2p.peerId.toString()
+		const gained: string[] = []
+		const lost: string[] = []
+		const newOwners = new Map<string, string[]>()
+
+		for (const blockId of this.trackedBlocks) {
+			const key = new TextEncoder().encode(blockId)
+			const coord = await hashKey(key)
+
+			// Get the current cohort — assembleCohort returns peer IDs sorted by distance
+			const cohort = this.deps.fret.assembleCohort(coord, this.getCohortSize())
+			const isResponsible = cohort.includes(selfId)
+			const wasResponsible = this.responsibilitySnapshot.get(blockId) ?? false
+
+			if (isResponsible && !wasResponsible) {
+				gained.push(blockId)
+			} else if (!isResponsible && wasResponsible) {
+				lost.push(blockId)
+				// The cohort members are the new owners
+				newOwners.set(blockId, cohort.filter(id => id !== selfId))
+			}
+
+			this.responsibilitySnapshot.set(blockId, isResponsible)
+		}
+
+		this.lastRebalanceAt = Date.now()
+
+		if (gained.length === 0 && lost.length === 0) {
+			return null
+		}
+
+		log('rebalance check: gained=%d lost=%d', gained.length, lost.length)
+
+		return { gained, lost, newOwners, triggeredAt }
+	}
+
+	private getCohortSize(): number {
+		const diag: any = (this.deps.fret as any).getDiagnostics?.()
+		const estimate = diag?.estimate ?? diag?.n
+		if (typeof estimate === 'number' && Number.isFinite(estimate) && estimate > 0) {
+			return Math.max(1, Math.min(3, Math.ceil(Math.sqrt(estimate))))
+		}
+		return 3
+	}
+
+	private emitEvent(event: RebalanceEvent): void {
+		for (const handler of this.handlers) {
+			try {
+				handler(event)
+			} catch (err) {
+				log('handler error: %O', err)
+			}
+		}
+	}
+
+	/**
+	 * Update ArachnodeInfo status through the fret adapter.
+	 */
+	setStatus(status: ArachnodeInfo['status']): void {
+		const current = this.deps.fretAdapter.getMyArachnodeInfo()
+		if (current) {
+			this.deps.fretAdapter.setArachnodeInfo({ ...current, status })
+		}
+	}
+}

package/src/dispute/arbitrator-selection.ts

@@ -0,0 +1,28 @@
+import type { PeerId } from '@libp2p/interface';
+import { sortPeersByDistance, type KnownPeer } from '../routing/responsibility.js';
+
+/**
+ * Select arbitrators for a dispute using XOR-distance from the block ID.
+ * Selects the next K peers beyond the original cluster (positions K+1 through 2K).
+ * This ensures independence (arbitrators are not in the original cluster)
+ * and determinism (all parties agree on who arbitrates).
+ */
+export function selectArbitrators(
+	allPeers: KnownPeer[],
+	blockIdBytes: Uint8Array,
+	excludePeerIds: Set<string>,
+	count: number,
+): PeerId[] {
+	// Sort all peers by XOR distance to the block ID
+	const sorted = sortPeersByDistance(allPeers, blockIdBytes);
+
+	// Skip peers in the original cluster (and self), select the next K
+	const arbitrators: PeerId[] = [];
+	for (const peer of sorted) {
+		if (arbitrators.length >= count) break;
+		if (excludePeerIds.has(peer.id.toString())) continue;
+		arbitrators.push(peer.id);
+	}
+
+	return arbitrators;
+}

package/src/dispute/client.ts

@@ -0,0 +1,41 @@
+import type { PeerId, IPeerNetwork } from '@optimystic/db-core';
+import { ProtocolClient } from '../protocol-client.js';
+import type { DisputeChallenge, DisputeResolution, ArbitrationVote, DisputeMessage } from './types.js';
+
+/**
+ * Client for the dispute protocol. Sends challenges to arbitrators
+ * and broadcasts resolutions.
+ */
+export class DisputeClient extends ProtocolClient {
+	private readonly protocol: string;
+
+	constructor(peerId: PeerId, peerNetwork: IPeerNetwork, protocolPrefix?: string) {
+		super(peerId, peerNetwork);
+		this.protocol = (protocolPrefix ?? '/db-p2p') + '/dispute/1.0.0';
+	}
+
+	static create(peerId: PeerId, peerNetwork: IPeerNetwork, protocolPrefix?: string): DisputeClient {
+		return new DisputeClient(peerId, peerNetwork, protocolPrefix);
+	}
+
+	/** Send a challenge to an arbitrator and get their vote */
+	async sendChallenge(challenge: DisputeChallenge, timeoutMs?: number): Promise<ArbitrationVote> {
+		const message: DisputeMessage = { type: 'challenge', challenge };
+		const signal = timeoutMs ? AbortSignal.timeout(timeoutMs) : undefined;
+		const response = await this.processMessage<{ type: 'vote'; vote: ArbitrationVote }>(
+			message,
+			this.protocol,
+			{ signal }
+		);
+		return response.vote;
+	}
+
+	/** Send a resolution to a peer (broadcast) */
+	async sendResolution(resolution: DisputeResolution): Promise<void> {
+		const message: DisputeMessage = { type: 'resolution', resolution };
+		await this.processMessage<{ type: 'ack' }>(
+			message,
+			this.protocol,
+		);
+	}
+}