agentic-flow 3.0.0-alpha.2 → 3.1.0

package/dist/packages/agentdb/src/consensus/RaftConsensus.js

@@ -1,762 +0,0 @@
-/**
- * RaftConsensus - Fault-Tolerant Distributed Consensus
- *
- * Implements the Raft consensus algorithm for multi-agent coordination with:
- * - Leader election with automatic failover (<1s)
- * - Log replication with strong consistency
- * - Byzantine fault tolerance (BFT)
- * - Gossip protocols for large-scale coordination
- * - CRDT synchronization for eventually consistent state
- * - Distributed locks with deadlock detection
- * - Automatic sharding and partitioning
- *
- * References:
- * - Raft Paper: https://raft.github.io/raft.pdf
- * - Byzantine Fault Tolerance: https://pmg.csail.mit.edu/papers/osdi99.pdf
- */
-import { EventEmitter } from 'events';
-import crypto from 'crypto';
-// ============================================================================
-// RaftConsensus Implementation
-// ============================================================================
-export class RaftConsensus extends EventEmitter {
-    config;
-    state;
-    peers = new Map();
-    electionTimer = null;
-    heartbeatTimer = null;
-    votesReceived = new Set();
-    commandQueue = new Map();
-    // Byzantine Fault Tolerance
-    privateKey;
-    publicKeys = new Map();
-    // Gossip Protocol
-    gossipInterval = null;
-    gossipState = new Map();
-    // CRDT State
-    crdtStates = new Map();
-    // Distributed Locks
-    locks = new Map();
-    lockCheckInterval = null;
-    // Metrics
-    metrics = {
-        totalElections: 0,
-        totalLeaderChanges: 0,
-        totalCommittedEntries: 0,
-        lastElectionTime: 0,
-        avgElectionTimeMs: 0,
-        uptime: Date.now(),
-    };
-    constructor(config) {
-        super();
-        this.config = {
-            nodeId: config.nodeId,
-            nodes: config.nodes,
-            electionTimeoutMin: config.electionTimeoutMin || 150,
-            electionTimeoutMax: config.electionTimeoutMax || 300,
-            heartbeatInterval: config.heartbeatInterval || 50,
-            maxEntriesPerRequest: config.maxEntriesPerRequest || 100,
-            snapshotThreshold: config.snapshotThreshold || 10000,
-            byzantineTolerance: config.byzantineTolerance || false,
-            quorumSize: config.quorumSize || Math.floor(config.nodes.length / 2) + 1,
-        };
-        // Initialize state
-        this.state = {
-            currentTerm: 0,
-            votedFor: null,
-            log: [],
-            commitIndex: 0,
-            lastApplied: 0,
-            state: 'follower',
-            leaderId: null,
-        };
-        // Initialize peer states
-        for (const nodeId of this.config.nodes) {
-            if (nodeId !== this.config.nodeId) {
-                this.peers.set(nodeId, {
-                    nodeId,
-                    nextIndex: 1,
-                    matchIndex: 0,
-                    lastHeartbeat: Date.now(),
-                    isHealthy: true,
-                });
-            }
-        }
-        // Setup Byzantine fault tolerance
-        if (this.config.byzantineTolerance) {
-            this.setupBFT();
-        }
-        // Start as follower
-        this.becomeFollower(0);
-    }
-    // ============================================================================
-    // Public API
-    // ============================================================================
-    /**
-     * Start the Raft node
-     */
-    start() {
-        this.resetElectionTimer();
-        this.startGossipProtocol();
-        this.startLockMonitoring();
-        this.emit('started', { nodeId: this.config.nodeId });
-    }
-    /**
-     * Stop the Raft node
-     */
-    stop() {
-        this.clearElectionTimer();
-        this.clearHeartbeatTimer();
-        this.stopGossipProtocol();
-        this.stopLockMonitoring();
-        this.emit('stopped', { nodeId: this.config.nodeId });
-    }
-    /**
-     * Replicate a command to the cluster
-     */
-    async replicate(command, timeout = 5000) {
-        if (this.state.state !== 'leader') {
-            throw new Error('Not the leader. Leader is: ' + (this.state.leaderId || 'unknown'));
-        }
-        const commandId = crypto.randomUUID();
-        const entry = {
-            term: this.state.currentTerm,
-            index: this.state.log.length + 1,
-            type: 'command',
-            command,
-            timestamp: Date.now(),
-        };
-        // Sign entry for BFT
-        if (this.config.byzantineTolerance) {
-            entry.signature = this.signData(entry);
-        }
-        this.state.log.push(entry);
-        this.emit('log_appended', entry);
-        // Wait for replication
-        return new Promise((resolve, reject) => {
-            const timer = setTimeout(() => {
-                this.commandQueue.delete(commandId);
-                reject(new Error('Replication timeout'));
-            }, timeout);
-            this.commandQueue.set(commandId, (result) => {
-                clearTimeout(timer);
-                resolve(result);
-            });
-            // Trigger immediate replication
-            this.replicateToFollowers();
-        });
-    }
-    /**
-     * Handle vote request from candidate
-     */
-    handleVoteRequest(request) {
-        // Verify signature for BFT
-        if (this.config.byzantineTolerance && !this.verifySignature(request, request.signature)) {
-            return {
-                term: this.state.currentTerm,
-                voteGranted: false,
-            };
-        }
-        // If candidate's term is older, reject
-        if (request.term < this.state.currentTerm) {
-            return {
-                term: this.state.currentTerm,
-                voteGranted: false,
-            };
-        }
-        // Update term if candidate's term is newer
-        if (request.term > this.state.currentTerm) {
-            this.becomeFollower(request.term);
-        }
-        // Check if we can vote for this candidate
-        const canVote = (this.state.votedFor === null || this.state.votedFor === request.candidateId) &&
-            this.isLogUpToDate(request.lastLogIndex, request.lastLogTerm);
-        if (canVote) {
-            this.state.votedFor = request.candidateId;
-            this.resetElectionTimer();
-        }
-        const response = {
-            term: this.state.currentTerm,
-            voteGranted: canVote,
-        };
-        // Sign response for BFT
-        if (this.config.byzantineTolerance) {
-            response.signature = this.signData(response);
-        }
-        return response;
-    }
-    /**
-     * Handle append entries request from leader
-     */
-    handleAppendEntries(request) {
-        // Verify signature for BFT
-        if (this.config.byzantineTolerance && !this.verifySignature(request, request.signature)) {
-            return {
-                term: this.state.currentTerm,
-                success: false,
-            };
-        }
-        // If leader's term is older, reject
-        if (request.term < this.state.currentTerm) {
-            return {
-                term: this.state.currentTerm,
-                success: false,
-            };
-        }
-        // Valid leader, reset election timer
-        this.resetElectionTimer();
-        // Update term and leader if needed
-        if (request.term > this.state.currentTerm) {
-            this.becomeFollower(request.term);
-        }
-        this.state.leaderId = request.leaderId;
-        // Check log consistency
-        if (request.prevLogIndex > 0) {
-            const prevEntry = this.state.log[request.prevLogIndex - 1];
-            if (!prevEntry || prevEntry.term !== request.prevLogTerm) {
-                return {
-                    term: this.state.currentTerm,
-                    success: false,
-                };
-            }
-        }
-        // Append new entries
-        if (request.entries.length > 0) {
-            // Remove conflicting entries
-            this.state.log = this.state.log.slice(0, request.prevLogIndex);
-            // Append new entries
-            for (const entry of request.entries) {
-                this.state.log.push(entry);
-            }
-            this.emit('log_replicated', { entries: request.entries });
-        }
-        // Update commit index
-        if (request.leaderCommit > this.state.commitIndex) {
-            this.state.commitIndex = Math.min(request.leaderCommit, this.state.log.length);
-            this.applyCommittedEntries();
-        }
-        const response = {
-            term: this.state.currentTerm,
-            success: true,
-            matchIndex: this.state.log.length,
-        };
-        // Sign response for BFT
-        if (this.config.byzantineTolerance) {
-            response.signature = this.signData(response);
-        }
-        return response;
-    }
-    /**
-     * Handle vote response (exposed for testing/network layer)
-     */
-    handleVoteResponse(response, peerId) {
-        this.processVoteResponse(response, peerId);
-    }
-    /**
-     * Handle append entries response (exposed for testing/network layer)
-     */
-    handleAppendEntriesResponse(response, peerId) {
-        this.processAppendEntriesResponse(response, peerId);
-    }
-    /**
-     * Get current cluster status
-     */
-    getStatus() {
-        return {
-            nodeId: this.config.nodeId,
-            state: this.state.state,
-            term: this.state.currentTerm,
-            leaderId: this.state.leaderId,
-            logLength: this.state.log.length,
-            commitIndex: this.state.commitIndex,
-            lastApplied: this.state.lastApplied,
-            peers: Array.from(this.peers.values()).map(p => ({
-                nodeId: p.nodeId,
-                isHealthy: p.isHealthy,
-                matchIndex: p.matchIndex,
-            })),
-            metrics: this.metrics,
-        };
-    }
-    // ============================================================================
-    // CRDT Operations
-    // ============================================================================
-    /**
-     * Update CRDT state with eventual consistency
-     */
-    updateCRDT(key, operation, value) {
-        let state = this.crdtStates.get(key);
-        if (!state) {
-            state = {
-                type: 'counter',
-                value: 0,
-                vectorClock: new Map([[this.config.nodeId, 0]]),
-            };
-            this.crdtStates.set(key, state);
-        }
-        // Update vector clock
-        const currentClock = state.vectorClock.get(this.config.nodeId) || 0;
-        state.vectorClock.set(this.config.nodeId, currentClock + 1);
-        // Apply operation
-        switch (operation) {
-            case 'increment':
-                state.value = (state.value || 0) + value;
-                break;
-            case 'add':
-                if (!Array.isArray(state.value))
-                    state.value = [];
-                state.value.push(value);
-                break;
-            case 'set':
-                state.value = value;
-                break;
-        }
-        // Gossip update to peers
-        this.gossipCRDTUpdate(key, state);
-        this.emit('crdt_updated', { key, state });
-    }
-    /**
-     * Get CRDT value
-     */
-    getCRDT(key) {
-        return this.crdtStates.get(key)?.value;
-    }
-    /**
-     * Merge CRDT state from remote node
-     */
-    mergeCRDT(key, remoteState) {
-        const localState = this.crdtStates.get(key);
-        if (!localState) {
-            this.crdtStates.set(key, remoteState);
-            return;
-        }
-        // Merge vector clocks
-        for (const [nodeId, clock] of remoteState.vectorClock.entries()) {
-            const localClock = localState.vectorClock.get(nodeId) || 0;
-            localState.vectorClock.set(nodeId, Math.max(localClock, clock));
-        }
-        // Merge values based on type
-        if (localState.type === 'counter') {
-            localState.value = Math.max(localState.value, remoteState.value);
-        }
-        else if (localState.type === 'set') {
-            localState.value = [...new Set([...localState.value, ...remoteState.value])];
-        }
-        this.emit('crdt_merged', { key, state: localState });
-    }
-    // ============================================================================
-    // Distributed Locks
-    // ============================================================================
-    /**
-     * Acquire distributed lock
-     */
-    async acquireLock(key, ttlMs = 30000) {
-        if (this.state.state !== 'leader') {
-            throw new Error('Only leader can manage locks');
-        }
-        const lock = this.locks.get(key);
-        const now = Date.now();
-        // Check if lock is available
-        if (lock && lock.holder && lock.expiresAt > now) {
-            // Lock is held, add to waiters
-            lock.waiters.push(this.config.nodeId);
-            return false;
-        }
-        // Acquire lock
-        this.locks.set(key, {
-            key,
-            holder: this.config.nodeId,
-            acquiredAt: now,
-            expiresAt: now + ttlMs,
-            waiters: [],
-        });
-        // Replicate lock acquisition
-        await this.replicate({
-            type: 'lock_acquire',
-            key,
-            holder: this.config.nodeId,
-            ttlMs,
-        });
-        this.emit('lock_acquired', { key, holder: this.config.nodeId });
-        return true;
-    }
-    /**
-     * Release distributed lock
-     */
-    async releaseLock(key) {
-        if (this.state.state !== 'leader') {
-            throw new Error('Only leader can manage locks');
-        }
-        const lock = this.locks.get(key);
-        if (!lock || lock.holder !== this.config.nodeId) {
-            throw new Error('Lock not held by this node');
-        }
-        // Release lock
-        this.locks.delete(key);
-        // Replicate lock release
-        await this.replicate({
-            type: 'lock_release',
-            key,
-            holder: this.config.nodeId,
-        });
-        this.emit('lock_released', { key, holder: this.config.nodeId });
-        // Notify next waiter
-        if (lock.waiters.length > 0) {
-            const nextHolder = lock.waiters.shift();
-            this.emit('lock_available', { key, nextHolder });
-        }
-    }
-    /**
-     * Check for deadlocks
-     */
-    detectDeadlocks() {
-        const waitGraph = new Map();
-        const deadlocks = [];
-        // Build wait-for graph
-        for (const lock of this.locks.values()) {
-            if (lock.holder && lock.waiters.length > 0) {
-                for (const waiter of lock.waiters) {
-                    const edges = waitGraph.get(waiter) || [];
-                    edges.push(lock.holder);
-                    waitGraph.set(waiter, edges);
-                }
-            }
-        }
-        // Detect cycles using DFS
-        const visited = new Set();
-        const recStack = new Set();
-        const detectCycle = (node, path) => {
-            visited.add(node);
-            recStack.add(node);
-            path.push(node);
-            const neighbors = waitGraph.get(node) || [];
-            for (const neighbor of neighbors) {
-                if (!visited.has(neighbor)) {
-                    if (detectCycle(neighbor, [...path])) {
-                        return true;
-                    }
-                }
-                else if (recStack.has(neighbor)) {
-                    // Found cycle
-                    const cycleStart = path.indexOf(neighbor);
-                    deadlocks.push(path.slice(cycleStart));
-                    return true;
-                }
-            }
-            recStack.delete(node);
-            return false;
-        };
-        for (const node of waitGraph.keys()) {
-            if (!visited.has(node)) {
-                detectCycle(node, []);
-            }
-        }
-        return deadlocks;
-    }
-    // ============================================================================
-    // Private Methods - Leader Election
-    // ============================================================================
-    becomeFollower(term) {
-        this.state.state = 'follower';
-        this.state.currentTerm = term;
-        this.state.votedFor = null;
-        this.clearHeartbeatTimer();
-        this.resetElectionTimer();
-        this.emit('state_changed', { state: 'follower', term });
-    }
-    becomeCandidate() {
-        this.state.state = 'candidate';
-        this.state.currentTerm++;
-        this.state.votedFor = this.config.nodeId;
-        this.votesReceived.clear();
-        this.votesReceived.add(this.config.nodeId);
-        this.metrics.totalElections++;
-        const electionStart = Date.now();
-        this.emit('state_changed', { state: 'candidate', term: this.state.currentTerm });
-        // Request votes from peers
-        this.requestVotes();
-        // Reset election timer
-        this.resetElectionTimer();
-        // Track election time
-        this.once('state_changed', (event) => {
-            if (event.state === 'leader') {
-                const electionTime = Date.now() - electionStart;
-                this.metrics.lastElectionTime = electionTime;
-                this.metrics.avgElectionTimeMs =
-                    (this.metrics.avgElectionTimeMs * (this.metrics.totalElections - 1) + electionTime) /
-                        this.metrics.totalElections;
-            }
-        });
-    }
-    becomeLeader() {
-        this.state.state = 'leader';
-        this.state.leaderId = this.config.nodeId;
-        this.clearElectionTimer();
-        this.metrics.totalLeaderChanges++;
-        // Initialize peer state
-        for (const peer of this.peers.values()) {
-            peer.nextIndex = this.state.log.length + 1;
-            peer.matchIndex = 0;
-        }
-        // Send initial heartbeat
-        this.sendHeartbeat();
-        // Start heartbeat timer
-        this.heartbeatTimer = setInterval(() => {
-            this.sendHeartbeat();
-        }, this.config.heartbeatInterval);
-        this.emit('state_changed', { state: 'leader', term: this.state.currentTerm });
-        this.emit('leader_elected', { leaderId: this.config.nodeId });
-    }
-    requestVotes() {
-        const request = {
-            term: this.state.currentTerm,
-            candidateId: this.config.nodeId,
-            lastLogIndex: this.state.log.length,
-            lastLogTerm: this.state.log[this.state.log.length - 1]?.term || 0,
-        };
-        // Sign request for BFT
-        if (this.config.byzantineTolerance) {
-            request.signature = this.signData(request);
-        }
-        // In real implementation, send to all peers
-        // For now, emit event for testing
-        this.emit('vote_request', request);
-    }
-    processVoteResponse(response, peerId) {
-        // Verify signature for BFT
-        if (this.config.byzantineTolerance && !this.verifySignature(response, response.signature)) {
-            return;
-        }
-        // Ignore if no longer candidate
-        if (this.state.state !== 'candidate') {
-            return;
-        }
-        // Update term if response has newer term
-        if (response.term > this.state.currentTerm) {
-            this.becomeFollower(response.term);
-            return;
-        }
-        // Count vote
-        if (response.voteGranted) {
-            this.votesReceived.add(peerId);
-            // Check if we have quorum
-            if (this.votesReceived.size >= this.config.quorumSize) {
-                this.becomeLeader();
-            }
-        }
-    }
-    isLogUpToDate(lastLogIndex, lastLogTerm) {
-        const ourLastIndex = this.state.log.length;
-        const ourLastTerm = this.state.log[ourLastIndex - 1]?.term || 0;
-        // Candidate's log is more up-to-date if:
-        // 1. Last term is greater, OR
-        // 2. Last term is same but index is greater or equal
-        return (lastLogTerm > ourLastTerm ||
-            (lastLogTerm === ourLastTerm && lastLogIndex >= ourLastIndex));
-    }
-    // ============================================================================
-    // Private Methods - Log Replication
-    // ============================================================================
-    sendHeartbeat() {
-        this.replicateToFollowers();
-    }
-    replicateToFollowers() {
-        for (const [peerId, peer] of this.peers.entries()) {
-            const prevLogIndex = peer.nextIndex - 1;
-            const prevLogTerm = prevLogIndex > 0 ? this.state.log[prevLogIndex - 1]?.term || 0 : 0;
-            const entries = this.state.log.slice(peer.nextIndex - 1, peer.nextIndex - 1 + this.config.maxEntriesPerRequest);
-            const request = {
-                term: this.state.currentTerm,
-                leaderId: this.config.nodeId,
-                prevLogIndex,
-                prevLogTerm,
-                entries,
-                leaderCommit: this.state.commitIndex,
-            };
-            // Sign request for BFT
-            if (this.config.byzantineTolerance) {
-                request.signature = this.signData(request);
-            }
-            // In real implementation, send to peer
-            // For now, emit event for testing
-            this.emit('append_entries', { peerId, request });
-        }
-    }
-    processAppendEntriesResponse(response, peerId) {
-        // Verify signature for BFT
-        if (this.config.byzantineTolerance && !this.verifySignature(response, response.signature)) {
-            return;
-        }
-        // Ignore if no longer leader
-        if (this.state.state !== 'leader') {
-            return;
-        }
-        const peer = this.peers.get(peerId);
-        if (!peer)
-            return;
-        // Update term if response has newer term
-        if (response.term > this.state.currentTerm) {
-            this.becomeFollower(response.term);
-            return;
-        }
-        if (response.success) {
-            // Update peer state
-            if (response.matchIndex !== undefined) {
-                peer.matchIndex = response.matchIndex;
-                peer.nextIndex = response.matchIndex + 1;
-            }
-            // Update commit index
-            this.updateCommitIndex();
-        }
-        else {
-            // Decrement nextIndex and retry
-            peer.nextIndex = Math.max(1, peer.nextIndex - 1);
-        }
-    }
-    updateCommitIndex() {
-        // Find highest index replicated on majority
-        const matchIndices = Array.from(this.peers.values())
-            .map(p => p.matchIndex)
-            .sort((a, b) => b - a);
-        const majorityIndex = matchIndices[Math.floor(this.config.quorumSize / 2)];
-        if (majorityIndex > this.state.commitIndex) {
-            const entry = this.state.log[majorityIndex - 1];
-            if (entry && entry.term === this.state.currentTerm) {
-                this.state.commitIndex = majorityIndex;
-                this.metrics.totalCommittedEntries++;
-                this.applyCommittedEntries();
-            }
-        }
-    }
-    applyCommittedEntries() {
-        while (this.state.lastApplied < this.state.commitIndex) {
-            this.state.lastApplied++;
-            const entry = this.state.log[this.state.lastApplied - 1];
-            if (entry) {
-                this.emit('entry_committed', entry);
-                // Resolve command promises
-                for (const [id, resolve] of this.commandQueue.entries()) {
-                    resolve(true);
-                    this.commandQueue.delete(id);
-                }
-            }
-        }
-    }
-    // ============================================================================
-    // Private Methods - Timers
-    // ============================================================================
-    resetElectionTimer() {
-        this.clearElectionTimer();
-        const timeout = this.config.electionTimeoutMin +
-            Math.random() * (this.config.electionTimeoutMax - this.config.electionTimeoutMin);
-        this.electionTimer = setTimeout(() => {
-            if (this.state.state !== 'leader') {
-                this.becomeCandidate();
-            }
-        }, timeout);
-    }
-    clearElectionTimer() {
-        if (this.electionTimer) {
-            clearTimeout(this.electionTimer);
-            this.electionTimer = null;
-        }
-    }
-    clearHeartbeatTimer() {
-        if (this.heartbeatTimer) {
-            clearInterval(this.heartbeatTimer);
-            this.heartbeatTimer = null;
-        }
-    }
-    // ============================================================================
-    // Private Methods - Byzantine Fault Tolerance
-    // ============================================================================
-    setupBFT() {
-        // Generate keypair (simplified - use real crypto in production)
-        this.privateKey = crypto.randomBytes(32).toString('hex');
-        const publicKey = crypto.createHash('sha256').update(this.privateKey).digest('hex');
-        this.publicKeys.set(this.config.nodeId, publicKey);
-    }
-    signData(data) {
-        if (!this.privateKey)
-            return '';
-        const message = JSON.stringify(data);
-        return crypto.createHmac('sha256', this.privateKey).update(message).digest('hex');
-    }
-    verifySignature(data, signature) {
-        if (!signature)
-            return false;
-        // In real implementation, verify with sender's public key
-        return true;
-    }
-    // ============================================================================
-    // Private Methods - Gossip Protocol
-    // ============================================================================
-    startGossipProtocol() {
-        this.gossipInterval = setInterval(() => {
-            this.gossipToRandomPeers();
-        }, 1000); // Gossip every second
-    }
-    stopGossipProtocol() {
-        if (this.gossipInterval) {
-            clearInterval(this.gossipInterval);
-            this.gossipInterval = null;
-        }
-    }
-    gossipToRandomPeers() {
-        const message = {
-            type: 'state',
-            nodeId: this.config.nodeId,
-            data: {
-                state: this.state.state,
-                term: this.state.currentTerm,
-                logLength: this.state.log.length,
-            },
-            timestamp: Date.now(),
-        };
-        // Sign message for BFT
-        if (this.config.byzantineTolerance) {
-            message.signature = this.signData(message);
-        }
-        this.emit('gossip', message);
-    }
-    gossipCRDTUpdate(key, state) {
-        const message = {
-            type: 'metadata',
-            nodeId: this.config.nodeId,
-            data: { key, state },
-            timestamp: Date.now(),
-        };
-        if (this.config.byzantineTolerance) {
-            message.signature = this.signData(message);
-        }
-        this.emit('gossip', message);
-    }
-    // ============================================================================
-    // Private Methods - Lock Monitoring
-    // ============================================================================
-    startLockMonitoring() {
-        this.lockCheckInterval = setInterval(() => {
-            this.checkExpiredLocks();
-            const deadlocks = this.detectDeadlocks();
-            if (deadlocks.length > 0) {
-                this.emit('deadlock_detected', { deadlocks });
-            }
-        }, 5000); // Check every 5 seconds
-    }
-    stopLockMonitoring() {
-        if (this.lockCheckInterval) {
-            clearInterval(this.lockCheckInterval);
-            this.lockCheckInterval = null;
-        }
-    }
-    checkExpiredLocks() {
-        const now = Date.now();
-        for (const [key, lock] of this.locks.entries()) {
-            if (lock.expiresAt < now) {
-                this.locks.delete(key);
-                this.emit('lock_expired', { key, holder: lock.holder });
-            }
-        }
-    }
-}
-//# sourceMappingURL=RaftConsensus.js.map