@soulcraft/brainy 3.0.0 → 3.1.0

package/dist/distributed/shardManager.js

@@ -0,0 +1,357 @@
+/**
+ * Shard Manager for Horizontal Scaling
+ * Implements consistent hashing for data distribution across shards
+ */
+import { createHash } from 'crypto';
+import { EventEmitter } from 'events';
+/**
+ * Consistent Hash Ring for shard distribution
+ */
+class ConsistentHashRing {
+    constructor(virtualNodes = 150) {
+        this.ring = new Map();
+        this.sortedKeys = [];
+        this.virtualNodes = virtualNodes;
+    }
+    /**
+     * Add a node to the hash ring
+     */
+    addNode(nodeId) {
+        for (let i = 0; i < this.virtualNodes; i++) {
+            const virtualNodeId = `${nodeId}:${i}`;
+            const hash = this.hash(virtualNodeId);
+            this.ring.set(hash, nodeId);
+        }
+        this.updateSortedKeys();
+    }
+    /**
+     * Remove a node from the hash ring
+     */
+    removeNode(nodeId) {
+        const keysToRemove = [];
+        for (const [hash, node] of this.ring) {
+            if (node === nodeId) {
+                keysToRemove.push(hash);
+            }
+        }
+        for (const key of keysToRemove) {
+            this.ring.delete(key);
+        }
+        this.updateSortedKeys();
+    }
+    /**
+     * Get the node responsible for a given key
+     */
+    getNode(key) {
+        if (this.ring.size === 0)
+            return null;
+        const hash = this.hash(key);
+        // Find the first node with hash >= key hash
+        for (const nodeHash of this.sortedKeys) {
+            if (nodeHash >= hash) {
+                return this.ring.get(nodeHash) || null;
+            }
+        }
+        // Wrap around to the first node
+        return this.ring.get(this.sortedKeys[0]) || null;
+    }
+    /**
+     * Get N nodes for replication
+     */
+    getNodes(key, count) {
+        if (this.ring.size === 0)
+            return [];
+        const nodes = new Set();
+        const hash = this.hash(key);
+        // Start from the primary node position
+        let startIdx = 0;
+        for (let i = 0; i < this.sortedKeys.length; i++) {
+            if (this.sortedKeys[i] >= hash) {
+                startIdx = i;
+                break;
+            }
+        }
+        // Collect unique nodes
+        let idx = startIdx;
+        while (nodes.size < count && nodes.size < this.getUniqueNodeCount()) {
+            const nodeHash = this.sortedKeys[idx % this.sortedKeys.length];
+            const node = this.ring.get(nodeHash);
+            if (node) {
+                nodes.add(node);
+            }
+            idx++;
+        }
+        return Array.from(nodes);
+    }
+    /**
+     * Get unique node count
+     */
+    getUniqueNodeCount() {
+        return new Set(this.ring.values()).size;
+    }
+    /**
+     * Update sorted keys for efficient lookup
+     */
+    updateSortedKeys() {
+        this.sortedKeys = Array.from(this.ring.keys()).sort((a, b) => a - b);
+    }
+    /**
+     * Hash function for consistent hashing
+     */
+    hash(key) {
+        const hash = createHash('md5').update(key).digest();
+        return hash.readUInt32BE(0);
+    }
+    /**
+     * Get all nodes in the ring
+     */
+    getAllNodes() {
+        return Array.from(new Set(this.ring.values()));
+    }
+}
+/**
+ * Shard Manager for distributing data across multiple nodes
+ */
+export class ShardManager extends EventEmitter {
+    constructor(config = {}) {
+        super();
+        this.shards = new Map();
+        this.nodeToShards = new Map();
+        this.shardCount = config.shardCount || 64;
+        this.replicationFactor = config.replicationFactor || 3;
+        this.autoRebalance = config.autoRebalance ?? true;
+        this.hashRing = new ConsistentHashRing(config.virtualNodes || 150);
+        // Initialize shards
+        this.initializeShards();
+    }
+    /**
+     * Initialize shard configuration
+     */
+    initializeShards() {
+        for (let i = 0; i < this.shardCount; i++) {
+            const shardId = `shard-${i.toString().padStart(3, '0')}`;
+            this.shards.set(shardId, {
+                id: shardId,
+                nodeId: '',
+                virtualNodes: [],
+                itemCount: 0,
+                sizeBytes: 0,
+                status: 'offline'
+            });
+        }
+    }
+    /**
+     * Add a node to the cluster
+     */
+    addNode(nodeId) {
+        this.hashRing.addNode(nodeId);
+        this.nodeToShards.set(nodeId, new Set());
+        // Assign shards to the new node
+        this.rebalanceShards();
+        this.emit('nodeAdded', { nodeId });
+    }
+    /**
+     * Remove a node from the cluster
+     */
+    removeNode(nodeId) {
+        const affectedShards = this.nodeToShards.get(nodeId) || new Set();
+        this.hashRing.removeNode(nodeId);
+        this.nodeToShards.delete(nodeId);
+        // Reassign affected shards
+        for (const shardId of affectedShards) {
+            const shard = this.shards.get(shardId);
+            if (shard) {
+                shard.status = 'rebalancing';
+            }
+        }
+        this.rebalanceShards();
+        this.emit('nodeRemoved', { nodeId, affectedShards: Array.from(affectedShards) });
+    }
+    /**
+     * Get shard assignment for a key
+     */
+    getShardForKey(key) {
+        const shardId = this.getShardId(key);
+        const nodes = this.hashRing.getNodes(shardId, this.replicationFactor);
+        if (nodes.length === 0)
+            return null;
+        return {
+            shardId,
+            nodeId: nodes[0],
+            replicas: nodes.slice(1)
+        };
+    }
+    /**
+     * Get nodes responsible for a shard
+     */
+    getNodesForShard(shardId) {
+        const shard = this.shards.get(shardId);
+        if (!shard)
+            return [];
+        // Return primary node and replicas
+        const nodes = this.hashRing.getNodes(shardId, this.replicationFactor);
+        return nodes;
+    }
+    /**
+     * Get total number of shards
+     */
+    getTotalShards() {
+        return this.shardCount;
+    }
+    /**
+     * Update shard assignment to a new node
+     */
+    updateShardAssignment(shardId, newNodeId) {
+        const shard = this.shards.get(shardId);
+        if (!shard) {
+            throw new Error(`Shard ${shardId} not found`);
+        }
+        // Remove from old node
+        if (shard.nodeId) {
+            const oldNodeShards = this.nodeToShards.get(shard.nodeId);
+            if (oldNodeShards) {
+                oldNodeShards.delete(shardId);
+            }
+        }
+        // Add to new node
+        shard.nodeId = newNodeId;
+        const newNodeShards = this.nodeToShards.get(newNodeId);
+        if (newNodeShards) {
+            newNodeShards.add(shardId);
+        }
+        else {
+            this.nodeToShards.set(newNodeId, new Set([shardId]));
+        }
+        this.emit('shardReassigned', { shardId, newNodeId });
+    }
+    /**
+     * Get shard ID for a key
+     */
+    getShardId(key) {
+        const hash = createHash('md5').update(key).digest();
+        const shardIndex = hash.readUInt16BE(0) % this.shardCount;
+        return `shard-${shardIndex.toString().padStart(3, '0')}`;
+    }
+    /**
+     * Rebalance shards across nodes
+     */
+    rebalanceShards() {
+        if (!this.autoRebalance)
+            return;
+        const nodes = this.hashRing.getAllNodes();
+        if (nodes.length === 0)
+            return;
+        // Clear current assignments
+        for (const nodeSet of this.nodeToShards.values()) {
+            nodeSet.clear();
+        }
+        // Reassign each shard
+        for (const [shardId, shard] of this.shards) {
+            const assignedNodes = this.hashRing.getNodes(shardId, 1);
+            if (assignedNodes.length > 0) {
+                shard.nodeId = assignedNodes[0];
+                shard.status = 'active';
+                const nodeShards = this.nodeToShards.get(assignedNodes[0]);
+                if (nodeShards) {
+                    nodeShards.add(shardId);
+                }
+            }
+            else {
+                shard.status = 'offline';
+            }
+        }
+        this.emit('rebalanced', { nodes, shardCount: this.shardCount });
+    }
+    /**
+     * Get shard assignment for all shards
+     */
+    getShardAssignments() {
+        const assignments = [];
+        for (const [shardId, shard] of this.shards) {
+            if (shard.nodeId) {
+                assignments.push({
+                    shardId,
+                    nodeId: shard.nodeId,
+                    replicas: this.hashRing.getNodes(shardId, this.replicationFactor).slice(1)
+                });
+            }
+        }
+        return assignments;
+    }
+    /**
+     * Get shard statistics
+     */
+    getShardStats() {
+        let activeShards = 0;
+        let rebalancingShards = 0;
+        let offlineShards = 0;
+        let totalItems = 0;
+        for (const shard of this.shards.values()) {
+            switch (shard.status) {
+                case 'active':
+                    activeShards++;
+                    break;
+                case 'rebalancing':
+                    rebalancingShards++;
+                    break;
+                case 'offline':
+                    offlineShards++;
+                    break;
+            }
+            totalItems += shard.itemCount;
+        }
+        return {
+            totalShards: this.shardCount,
+            activeShards,
+            rebalancingShards,
+            offlineShards,
+            averageItemsPerShard: this.shardCount > 0 ? Math.floor(totalItems / this.shardCount) : 0
+        };
+    }
+    /**
+     * Update shard metrics
+     */
+    updateShardMetrics(shardId, itemCount, sizeBytes) {
+        const shard = this.shards.get(shardId);
+        if (shard) {
+            shard.itemCount = itemCount;
+            shard.sizeBytes = sizeBytes;
+        }
+    }
+    /**
+     * Get replication nodes for a shard
+     */
+    getReplicationNodes(shardId) {
+        return this.hashRing.getNodes(shardId, this.replicationFactor);
+    }
+    /**
+     * Check if rebalancing is needed
+     */
+    needsRebalancing() {
+        const stats = this.getShardStats();
+        return stats.offlineShards > 0 || stats.rebalancingShards > 0;
+    }
+    /**
+     * Get cluster health
+     */
+    getHealth() {
+        const nodes = this.hashRing.getAllNodes();
+        const stats = this.getShardStats();
+        return {
+            healthy: stats.activeShards >= this.shardCount * 0.9, // 90% shards active
+            nodes: nodes.length,
+            shards: {
+                total: this.shardCount,
+                active: stats.activeShards,
+                inactive: stats.offlineShards + stats.rebalancingShards
+            }
+        };
+    }
+}
+/**
+ * Create a shard manager instance
+ */
+export function createShardManager(config) {
+    return new ShardManager(config);
+}
+//# sourceMappingURL=shardManager.js.map

package/dist/distributed/shardMigration.d.ts

@@ -0,0 +1,110 @@
+/**
+ * Shard Migration System for Brainy 3.0
+ *
+ * Handles zero-downtime migration of data between nodes
+ * Uses streaming for efficient transfer of large datasets
+ */
+import { EventEmitter } from 'events';
+import type { StorageAdapter } from '../coreTypes.js';
+import type { ShardManager } from './shardManager.js';
+import type { HTTPTransport } from './httpTransport.js';
+import type { DistributedCoordinator } from './coordinator.js';
+export interface MigrationTask {
+    id: string;
+    shardId: string;
+    sourceNode: string;
+    targetNode: string;
+    status: 'pending' | 'transferring' | 'validating' | 'switching' | 'completed' | 'failed';
+    progress: number;
+    itemsTransferred: number;
+    totalItems: number;
+    startTime: number;
+    endTime?: number;
+    error?: string;
+}
+export interface MigrationOptions {
+    batchSize?: number;
+    validateData?: boolean;
+    maxRetries?: number;
+    timeout?: number;
+}
+export declare class ShardMigrationManager extends EventEmitter {
+    private storage;
+    private shardManager;
+    private transport;
+    private coordinator;
+    private nodeId;
+    private activeMigrations;
+    private migrationQueue;
+    private maxConcurrentMigrations;
+    constructor(nodeId: string, storage: StorageAdapter, shardManager: ShardManager, transport: HTTPTransport, coordinator: DistributedCoordinator);
+    /**
+     * Initiate migration of a shard to a new node
+     */
+    migrateShard(shardId: string, targetNode: string, options?: MigrationOptions): Promise<MigrationTask>;
+    /**
+     * Process migration queue
+     */
+    private processMigrationQueue;
+    /**
+     * Execute a single migration task
+     */
+    private executeMigration;
+    /**
+     * Transfer data from source to target node
+     */
+    private transferData;
+    /**
+     * Get items from a shard
+     */
+    private getShardItems;
+    /**
+     * Get count of items in a shard
+     */
+    private getShardItemCount;
+    /**
+     * Validate transferred data
+     */
+    private validateData;
+    /**
+     * Switch shard ownership atomically
+     */
+    private switchOwnership;
+    /**
+     * Wait for consensus on migration
+     */
+    private waitForConsensus;
+    /**
+     * Cleanup local shard data after migration
+     */
+    private cleanupShardData;
+    /**
+     * Handle incoming migration batch (when we're the target)
+     */
+    receiveMigrationBatch(data: {
+        migrationId: string;
+        shardId: string;
+        items: any[];
+        offset: number;
+        total: number;
+    }): Promise<void>;
+    /**
+     * Validate received migration data
+     */
+    validateMigration(data: {
+        migrationId: string;
+        shardId: string;
+        expectedCount: number;
+    }): Promise<{
+        valid: boolean;
+        error?: string;
+    }>;
+    /**
+     * Get status of all active migrations
+     */
+    getActiveMigrations(): MigrationTask[];
+    /**
+     * Cancel a migration
+     */
+    cancelMigration(migrationId: string): Promise<void>;
+}