@soulcraft/brainy 2.15.0 → 3.0.1

package/dist/brainyData.js

@@ -12,12 +12,11 @@ import { enforceNodeVersion } from './utils/nodeVersionCheck.js';
 import { createNamespacedMetadata, updateNamespacedMetadata, markDeleted, markRestored, isDeleted, getUserMetadata } from './utils/metadataNamespace.js';
 import { PeriodicCleanup } from './utils/periodicCleanup.js';
 import { NounType, VerbType } from './types/graphTypes.js';
-import { validateNounType } from './utils/typeValidation.js';
-import { createServerSearchAugmentations } from './augmentations/serverSearchAugmentations.js';
-import { augmentationPipeline } from './augmentationPipeline.js';
+import { validateNounType, validateSearchQuery, validateSearchOptions, validateDataInput } from './utils/typeValidation.js';
+import { BrainyError } from './errors/brainyError.js';
 import { prodLog } from './utils/logger.js';
 import { prepareJsonForVectorization, extractFieldFromJson } from './utils/jsonProcessing.js';
-import { DistributedConfigManager, HashPartitioner, OperationalModeFactory, DomainDetector } from './distributed/index.js';
+import { DistributedConfigManager, HashPartitioner, OperationalModeFactory, DomainDetector, DistributedCoordinator, ShardManager, CacheSync, ReadWriteSeparation } from './distributed/index.js';
 import { CacheAutoConfigurator } from './utils/cacheAutoConfig.js';
 import { RequestDeduplicator } from './utils/requestDeduplicator.js';
 import { AugmentationRegistry } from './augmentations/brainyAugmentation.js';
@@ -30,9 +29,7 @@ import { createDefaultAugmentations } from './augmentations/defaultAugmentations
 // import { RealtimeStreamingAugmentation } from './augmentations/realtimeStreamingAugmentation.js'
 import { IntelligentVerbScoringAugmentation } from './augmentations/intelligentVerbScoringAugmentation.js';
 import { ImprovedNeuralAPI } from './neural/improvedNeuralAPI.js';
-import { TripleIntelligenceEngine } from './triple/TripleIntelligence.js';
 export class BrainyData {
-    // REMOVED: HealthMonitor is now handled by MonitoringAugmentation
     // Statistics collector
     // REMOVED: StatisticsCollector is now handled by MetricsAugmentation
     // Clean augmentation accessors for internal use
@@ -54,6 +51,11 @@ export class BrainyData {
     get monitoring() {
         return this.augmentations.get('monitoring');
     }
+    // ================================================================
+    // SECTION 1: CONFIGURATION & PROPERTIES
+    // ================================================================
+    // Configuration getters, dimensions, and system properties
+    // These provide access to core system configuration and status
     /**
      * Get the vector dimensions
      */
@@ -80,6 +82,11 @@ export class BrainyData {
     get initialized() {
         return this.isInitialized;
     }
+    // ================================================================
+    // SECTION 2: CONSTRUCTOR & INITIALIZATION
+    // ================================================================
+    // Constructor, initialization logic, and system setup
+    // Handles zero-config setup and system bootstrapping
     /**
      * Create a new vector database
      * @param config - Zero-config string ('production', 'development', 'minimal'),
@@ -127,6 +134,17 @@ export class BrainyData {
         this.partitioner = null;
         this.operationalMode = null;
         this.domainDetector = null;
+        // REMOVED: HealthMonitor is now handled by MonitoringAugmentation
+        // New distributed components for 3.0
+        this.networkTransport = null;
+        this.coordinator = null;
+        this.shardManager = null;
+        this.cacheSync = null;
+        this.readWriteSeparation = null;
+        this.httpTransport = null; // HTTPTransport from dynamic import
+        this.storageDiscovery = null; // StorageDiscovery from dynamic import
+        this.queryPlanner = null; // DistributedQueryPlanner from dynamic import
+        this.shardMigrationManager = null; // ShardMigrationManager from dynamic import
         // Enforce Node.js version requirement for ONNX stability
         if (typeof process !== 'undefined' && process.version && !process.env.BRAINY_SKIP_VERSION_CHECK) {
             enforceNodeVersion();
@@ -405,7 +423,7 @@ export class BrainyData {
             await this.storage.init();
         }
         else {
-            throw new Error('Failed to resolve storage');
+            throw BrainyError.storage('Failed to resolve storage');
         }
     }
     /**
@@ -439,7 +457,7 @@ export class BrainyData {
      */
     async initializePeriodicCleanup() {
         if (!this.storage) {
-            throw new Error('Cannot initialize periodic cleanup: storage not available');
+            throw BrainyError.storage('Cannot initialize periodic cleanup: storage not available');
         }
         // Skip cleanup if in read-only or frozen mode
         if (this.readOnly || this.frozen) {
@@ -860,16 +878,12 @@ export class BrainyData {
      */
     getCurrentAugmentation() {
         try {
-            // Get all registered augmentations
-            const augmentationTypes = augmentationPipeline.getAvailableAugmentationTypes();
-            // Check each type of augmentation
-            for (const type of augmentationTypes) {
-                const augmentations = augmentationPipeline.getAugmentationsByType(type);
-                // Find the first augmentation (all registered augmentations are considered enabled)
-                for (const augmentation of augmentations) {
-                    if (augmentation) {
-                        return augmentation.name;
-                    }
+            // Get all registered augmentations from the new API
+            const allAugmentations = this.augmentations.getAll();
+            // Find the first enabled augmentation
+            for (const augmentation of allAugmentations) {
+                if (augmentation && augmentation.config?.enabled !== false) {
+                    return augmentation.name;
                 }
             }
             return 'default';
@@ -1028,16 +1042,6 @@ export class BrainyData {
                     offset += limit;
                 }
             }
-            // Connect to remote server if configured with autoConnect
-            if (this.remoteServerConfig && this.remoteServerConfig.autoConnect) {
-                try {
-                    await this.connectToRemoteServer(this.remoteServerConfig.url, this.remoteServerConfig.protocols);
-                }
-                catch (remoteError) {
-                    console.warn('Failed to auto-connect to remote server:', remoteError);
-                    // Continue initialization even if remote connection fails
-                }
-            }
             // Initialize statistics collector with existing data
             try {
                 const existingStats = await this.storage.getStatistics();
@@ -1146,6 +1150,8 @@ export class BrainyData {
         // Create operational mode based on role
         const role = this.configManager.getRole();
         this.operationalMode = OperationalModeFactory.createMode(role);
+        // Initialize NEW distributed components for 3.0
+        await this.initializeDistributedComponents(sharedConfig);
         // Validate that role matches the configured mode
         // Don't override explicitly set readOnly/writeOnly
         if (role === 'reader' && !this.readOnly) {
@@ -1206,75 +1212,212 @@ export class BrainyData {
         }
     }
     /**
-     * Get distributed health status
-     * @returns Health status if distributed mode is enabled
+     * Initialize NEW distributed components for Brainy 3.0
+     * This enables true multi-node operation with consensus and sharding
+     */
+    async initializeDistributedComponents(sharedConfig) {
+        // Extract distributed configuration with type safety
+        const distConfig = typeof this.distributedConfig === 'object' && this.distributedConfig !== null
+            ? this.distributedConfig
+            : {};
+        // Generate or use node ID
+        const nodeId = distConfig.instanceId || sharedConfig.nodeId ||
+            `node-${Math.random().toString(36).substring(2, 11)}`;
+        // Use zero-config discovery if storage is available
+        if (this.storage && distConfig.useStorageDiscovery !== false) {
+            // Use storage-based discovery for true zero-config
+            const { StorageDiscovery } = await import('./distributed/storageDiscovery.js');
+            this.storageDiscovery = new StorageDiscovery(this.storage, nodeId);
+            // Start discovery and join cluster
+            const port = distConfig.port || 0; // 0 = auto-detect
+            const cluster = await this.storageDiscovery.start(port);
+            // Use discovered nodes as seeds
+            distConfig.peers = cluster.nodes.map((n) => `${n.address}:${n.port}`);
+        }
+        // Initialize HTTP transport (replaces NetworkTransport)
+        const { HTTPTransport } = await import('./distributed/httpTransport.js');
+        this.httpTransport = new HTTPTransport(nodeId);
+        // Start HTTP transport
+        const actualPort = await this.httpTransport.start();
+        console.log(`[${nodeId}] HTTP transport listening on port ${actualPort}`);
+        // Initialize coordinator for consensus
+        this.coordinator = new DistributedCoordinator({
+            nodeId,
+            heartbeatInterval: distConfig.heartbeatInterval || 1000,
+            electionTimeout: distConfig.electionTimeout || 5000,
+            nodes: distConfig.peers || []
+        });
+        // Start coordinator with HTTP transport adapter
+        const transportAdapter = {
+            send: (nodeId, message) => this.httpTransport.call(nodeId, 'coordinator', message),
+            broadcast: (message) => this.httpTransport.broadcast('coordinator', message),
+            on: (event, handler) => {
+                // Register handler for coordinator messages
+                if (event === 'message') {
+                    this.httpTransport.registerHandler('coordinator', handler);
+                }
+            }
+        };
+        await this.coordinator.start(transportAdapter);
+        // Initialize shard manager
+        this.shardManager = new ShardManager({
+            shardCount: distConfig.shardCount || 8,
+            replicationFactor: distConfig.replicationFactor || 3,
+            virtualNodes: distConfig.virtualNodes || 100,
+            autoRebalance: distConfig.autoRebalance !== false
+        });
+        // Initialize cache synchronization
+        this.cacheSync = new CacheSync({
+            nodeId,
+            syncInterval: distConfig.syncInterval || 5000
+        });
+        // Initialize read/write separation
+        // Note: This requires all components to be initialized
+        this.readWriteSeparation = new ReadWriteSeparation({
+            nodeId,
+            role: this.readOnly ? 'replica' : this.writeOnly ? 'primary' : 'auto',
+            primaryUrl: distConfig.primaryUrl || '',
+            replicaUrls: distConfig.replicaUrls || [],
+            syncInterval: distConfig.syncInterval || 5000,
+            readPreference: distConfig.readPreference || 'nearest',
+            consistencyLevel: distConfig.consistencyLevel || 'eventual'
+        }, this.coordinator, this.shardManager, this.cacheSync);
+        // Initialize query planner for distributed queries
+        const { DistributedQueryPlanner } = await import('./distributed/queryPlanner.js');
+        this.queryPlanner = new DistributedQueryPlanner(nodeId, this.coordinator, this.shardManager, this.httpTransport, this.storage, this._tripleEngine);
+        // Set up coordinator event handlers
+        this.coordinator.on('leaderElected', (leaderId) => {
+            if (leaderId === nodeId) {
+                console.log(`[${nodeId}] Became the leader`);
+                // Update read/write separation if needed
+                if (this.readWriteSeparation) {
+                    this.readWriteSeparation.setPrimary(true);
+                }
+            }
+            else {
+                console.log(`[${nodeId}] Following leader: ${leaderId}`);
+                if (this.readWriteSeparation) {
+                    this.readWriteSeparation.setPrimary(false);
+                }
+            }
+        });
+        // Set up distributed operation handlers
+        this.setupDistributedOperations();
+        if (this.loggingConfig?.verbose) {
+            console.log(`[${nodeId}] Distributed components initialized`);
+            console.log(`- Network: ${this.networkTransport ? 'Ready' : 'Not initialized'}`);
+            console.log(`- Coordinator: ${this.coordinator.getState().state}`);
+            console.log(`- Shards: ${this.shardManager.getTotalShards()} total`);
+        }
+    }
+    /**
+     * Execute a query on local shards
      */
-    getHealthStatus() {
-        return this.monitoring?.getHealthStatus() || null;
+    async executeLocalQuery(query, shards) {
+        // For now, just do a regular find
+        // In production, this would filter by shard assignments
+        const results = await this.find(query.text || query.query, query.k || 10);
+        return results;
     }
     /**
-     * Connect to a remote Brainy server for search operations
-     * @param serverUrl WebSocket URL of the remote Brainy server
-     * @param protocols Optional WebSocket protocols to use
-     * @returns The connection object
+     * Set up distributed operation handlers
      */
-    async connectToRemoteServer(serverUrl, protocols) {
-        await this.ensureInitialized();
-        try {
-            // Create server search augmentations
-            const { conduit, connection } = await createServerSearchAugmentations(serverUrl, {
-                protocols,
-                localDb: this
+    setupDistributedOperations() {
+        // Set up handlers for HTTP transport if available
+        if (this.httpTransport) {
+            // Handle distributed query requests
+            this.httpTransport.registerHandler('query', async (data) => {
+                const { query, shards } = data;
+                try {
+                    // Execute query on specified shards
+                    const results = await this.executeLocalQuery(query, shards);
+                    return { success: true, results, count: results.length };
+                }
+                catch (error) {
+                    return { success: false, error: error.message };
+                }
+            });
+            // Handle migration batch receives
+            this.httpTransport.registerHandler('receiveMigrationBatch', async (data) => {
+                if (this.shardMigrationManager) {
+                    await this.shardMigrationManager.receiveMigrationBatch(data);
+                }
+                return { success: true };
+            });
+            // Handle migration validation
+            this.httpTransport.registerHandler('validateMigration', async (data) => {
+                if (this.shardMigrationManager) {
+                    return await this.shardMigrationManager.validateMigration(data);
+                }
+                return { valid: false, error: 'No migration manager' };
             });
-            // TODO: Store conduit and connection (post-2.0.0 feature)
-            // this.serverSearchConduit = conduit
-            // this.serverConnection = connection
-            return connection;
         }
-        catch (error) {
-            console.error('Failed to connect to remote server:', error);
-            throw new Error(`Failed to connect to remote server: ${error}`);
+        // Set up legacy NetworkTransport handlers if still used
+        if (this.networkTransport) {
+            const handlers = this.networkTransport.messageHandlers;
+            // Handle distributed add requests
+            handlers.set('DISTRIBUTED_ADD', async (msg) => {
+                const { data, type, metadata } = msg.data;
+                try {
+                    const id = await this.localAdd(data, type, metadata);
+                    return { success: true, id };
+                }
+                catch (error) {
+                    return { success: false, error: error.message };
+                }
+            });
+            // Handle distributed find requests
+            handlers.set('DISTRIBUTED_FIND', async (msg) => {
+                const { query, shardId } = msg.data;
+                try {
+                    const results = await this.localFind(query, shardId);
+                    return { success: true, results };
+                }
+                catch (error) {
+                    return { success: false, error: error.message };
+                }
+            });
+            // Handle cache sync messages
+            handlers.set('CACHE_SYNC', async (msg) => {
+                if (this.cacheSync) {
+                    await this.cacheSync.handleSyncMessage(msg.data);
+                }
+                return { success: true };
+            });
         }
     }
-    // REMOVED: addItem() - Use addNoun() instead (cleaner 2.0 API)
-    // REMOVED: addToBoth() - Remote server functionality moved to post-2.0.0
     /**
-     * Add a vector to the remote server
-     * @param id ID of the vector to add
-     * @param vector Vector to add
-     * @param metadata Optional metadata to associate with the vector
-     * @returns True if successful, false otherwise
-     * @private
+     * Local add operation (without distribution)
      */
-    async addToRemote(id, vector, metadata) {
-        if (!this.isConnectedToRemoteServer()) {
-            return false;
-        }
+    async localAdd(data, type, metadata) {
+        // Use the existing addNoun method but bypass distribution checks
+        const tempDistributed = this.shardManager;
+        this.shardManager = null; // Temporarily disable to prevent recursion
         try {
-            // TODO: Remote server operations (post-2.0.0 feature)
-            // if (!this.serverSearchConduit || !this.serverConnection) {
-            //   throw new Error(
-            //     'Server search conduit or connection is not initialized'
-            //   )
-            // }
-            // TODO: Add to remote server
-            // const addResult = await this.serverSearchConduit.addToBoth(
-            //   this.serverConnection.connectionId,
-            //   vector,
-            //   metadata
-            // )
-            throw new Error('Remote server functionality not yet implemented in Brainy 2.0.0');
-            // TODO: Handle remote add result (post-2.0.0 feature)
-            // if (!addResult.success) {
-            //   throw new Error(`Remote add failed: ${addResult.error}`)
-            // }
-            return true;
+            const id = await this.addNoun(data, type, metadata);
+            return id;
         }
-        catch (error) {
-            console.error('Failed to add to remote server:', error);
-            throw new Error(`Failed to add to remote server: ${error}`);
+        finally {
+            this.shardManager = tempDistributed; // Restore
         }
     }
+    /**
+     * Local find operation (potentially for a specific shard)
+     */
+    async localFind(query, shardId) {
+        // TODO: Implement shard-aware find
+        // For now, just use regular find
+        return this.find(query);
+    }
+    /**
+     * Get distributed health status
+     * @returns Health status if distributed mode is enabled
+     */
+    getHealthStatus() {
+        return this.monitoring?.getHealthStatus() || null;
+    }
+    // REMOVED: addItem() - Use addNoun() instead (cleaner 2.0 API)
+    // REMOVED: addToBoth() - Remote server functionality moved to post-2.0.0
     /**
      * Add multiple vectors or data items to the database
      * @param items Array of items to add
@@ -1387,20 +1530,6 @@ export class BrainyData {
             throw new Error(`Failed to add batch of items: ${error}`);
         }
     }
-    /**
-     * Add multiple vectors or data items to both local and remote databases
-     * @param items Array of items to add (with required types)
-     * @param options Additional options
-     * @returns Array of IDs for the added items
-     */
-    async addBatchToBoth(items, options = {}) {
-        // Check if connected to a remote server
-        if (!this.isConnectedToRemoteServer()) {
-            throw new Error('Not connected to a remote server. Call connectToRemoteServer() first.');
-        }
-        // Add to local with addToRemote option
-        return this.addNouns(items, { ...options, addToRemote: true });
-    }
     /**
      * Filter search results by service
      * @param results Search results to filter
@@ -1679,6 +1808,12 @@ export class BrainyData {
      * @param options Simple search options (metadata filters only)
      * @returns Vector search results
      */
+    // ================================================================
+    // SECTION 3: SEARCH & DISCOVERY
+    // ================================================================
+    // Vector search, semantic search, and content discovery operations
+    // Includes find(), search(), searchLocal(), findSimilar() and related methods
+    // Powers the Triple Intelligence search combining vector + graph + field queries
     /**
      * 🔍 Simple Vector Similarity Search - Clean wrapper around find()
      *
@@ -1700,19 +1835,22 @@ export class BrainyData {
      * })
      */
     async search(queryVectorOrData, options = {}) {
-        // Build metadata filter from options
-        const metadataFilter = { ...options.metadata };
+        // Comprehensive input validation
+        const validatedQuery = validateSearchQuery(queryVectorOrData, 'queryVectorOrData');
+        const validatedOptions = validateSearchOptions(options, 'options');
+        // Build metadata filter from validated options
+        const metadataFilter = { ...validatedOptions.metadata };
         // Add noun type filtering
-        if (options.nounTypes && options.nounTypes.length > 0) {
-            metadataFilter.nounType = { in: options.nounTypes };
+        if (validatedOptions.nounTypes && validatedOptions.nounTypes.length > 0) {
+            metadataFilter.nounType = { in: validatedOptions.nounTypes };
         }
         // Add item ID filtering
-        if (options.itemIds && options.itemIds.length > 0) {
-            metadataFilter.id = { in: options.itemIds };
+        if (validatedOptions.itemIds && validatedOptions.itemIds.length > 0) {
+            metadataFilter.id = { in: validatedOptions.itemIds };
         }
         // Build simple TripleQuery for vector similarity
         const tripleQuery = {
-            like: queryVectorOrData
+            like: validatedQuery
         };
         // Add metadata filter if we have conditions
         if (Object.keys(metadataFilter).length > 0) {
@@ -1720,17 +1858,17 @@ export class BrainyData {
         }
         // Extract find() options
         const findOptions = {
-            limit: options.limit,
-            offset: options.offset,
-            cursor: options.cursor,
-            excludeDeleted: options.excludeDeleted,
-            timeout: options.timeout
+            limit: validatedOptions.limit,
+            offset: validatedOptions.offset,
+            cursor: validatedOptions.cursor,
+            excludeDeleted: validatedOptions.excludeDeleted,
+            timeout: validatedOptions.timeout
         };
         // Call find() with structured query - this is the key simplification!
         let results = await this.find(tripleQuery, findOptions);
         // Apply threshold filtering if specified
-        if (options.threshold !== undefined) {
-            results = results.filter(r => (r.fusionScore || r.score || 0) >= options.threshold);
+        if (validatedOptions.threshold !== undefined) {
+            results = results.filter(r => (r.fusionScore || r.score || 0) >= validatedOptions.threshold);
         }
         // Convert to SearchResult format
         return results.map(r => ({
@@ -1843,11 +1981,9 @@ export class BrainyData {
         if (options.searchMode === 'local') {
             return this.searchLocal(queryVectorOrData, k, options);
         }
-        else if (options.searchMode === 'remote') {
-            return this.searchRemote(queryVectorOrData, k, options);
-        }
-        else if (options.searchMode === 'combined') {
-            return this.searchCombined(queryVectorOrData, k, options);
+        else if (options.searchMode === 'remote' || options.searchMode === 'combined') {
+            // Remote and combined search modes not supported - fall back to local
+            return this.searchLocal(queryVectorOrData, k, options);
         }
         // Generate deduplication key for concurrent request handling
         const dedupeKey = RequestDeduplicator.getSearchKey(typeof queryVectorOrData === 'string' ? queryVectorOrData : JSON.stringify(queryVectorOrData), k, options);
@@ -3135,6 +3271,12 @@ export class BrainyData {
             searchMemoryUsage: this.cache?.getMemoryUsage() || 0
         };
     }
+    // ================================================================
+    // SECTION 8: CACHE & PERFORMANCE
+    // ================================================================
+    // Caching, performance optimization, and memory management
+    // Includes clearCache(), performance tuning, and optimization utilities
+    // Cache management for improved response times and resource efficiency
     /**
      * Clear search cache manually (useful for testing or memory management)
      */
@@ -3303,6 +3445,12 @@ export class BrainyData {
             // Ignore errors in size calculation
         }
     }
+    // ================================================================
+    // SECTION 7: STATISTICS & MONITORING
+    // ================================================================
+    // Analytics, monitoring, and metrics collection operations
+    // Includes getStatistics(), getServiceStatistics(), performance monitoring
+    // System health reporting and data analytics for optimization
     /**
      * Get statistics about the current state of the database
      * @param options Additional options for retrieving statistics
@@ -4065,102 +4213,6 @@ export class BrainyData {
             throw new Error(`Failed to search with text query: ${error}`);
         }
     }
-    /**
-     * Search a remote Brainy server for similar vectors
-     * @param queryVectorOrData Query vector or data to search for
-     * @param k Number of results to return
-     * @param options Additional options
-     * @returns Array of search results
-     */
-    async searchRemote(queryVectorOrData, k = 10, options = {}) {
-        // TODO: Remote server search will be implemented in post-2.0.0 release
-        await this.ensureInitialized();
-        this.checkWriteOnly();
-        throw new Error('Remote server search functionality not yet implemented in Brainy 2.0.0');
-    }
-    /**
-     * Search both local and remote Brainy instances, combining the results
-     * @param queryVectorOrData Query vector or data to search for
-     * @param k Number of results to return
-     * @param options Additional options
-     * @returns Array of search results
-     */
-    async searchCombined(queryVectorOrData, k = 10, options = {}) {
-        await this.ensureInitialized();
-        // Check if database is in write-only mode
-        this.checkWriteOnly();
-        // Check if connected to a remote server
-        if (!this.isConnectedToRemoteServer()) {
-            // If not connected to a remote server, just search locally
-            return this.searchLocal(queryVectorOrData, k, options);
-        }
-        try {
-            // Default to searching local first
-            const localFirst = options.localFirst !== false;
-            if (localFirst) {
-                // Search local first
-                const localResults = await this.searchLocal(queryVectorOrData, k, options);
-                // If we have enough local results, return them
-                if (localResults.length >= k) {
-                    return localResults;
-                }
-                // Otherwise, search remote for additional results
-                const remoteResults = await this.searchRemote(queryVectorOrData, k - localResults.length, { ...options, storeResults: true });
-                // Combine results, removing duplicates
-                const combinedResults = [...localResults];
-                const localIds = new Set(localResults.map((r) => r.id));
-                for (const result of remoteResults) {
-                    if (!localIds.has(result.id)) {
-                        combinedResults.push(result);
-                    }
-                }
-                return combinedResults;
-            }
-            else {
-                // Search remote first
-                const remoteResults = await this.searchRemote(queryVectorOrData, k, {
-                    ...options,
-                    storeResults: true
-                });
-                // If we have enough remote results, return them
-                if (remoteResults.length >= k) {
-                    return remoteResults;
-                }
-                // Otherwise, search local for additional results
-                const localResults = await this.searchLocal(queryVectorOrData, k - remoteResults.length, options);
-                // Combine results, removing duplicates
-                const combinedResults = [...remoteResults];
-                const remoteIds = new Set(remoteResults.map((r) => r.id));
-                for (const result of localResults) {
-                    if (!remoteIds.has(result.id)) {
-                        combinedResults.push(result);
-                    }
-                }
-                return combinedResults;
-            }
-        }
-        catch (error) {
-            console.error('Failed to perform combined search:', error);
-            throw new Error(`Failed to perform combined search: ${error}`);
-        }
-    }
-    /**
-     * Check if the instance is connected to a remote server
-     * @returns True if connected to a remote server, false otherwise
-     */
-    isConnectedToRemoteServer() {
-        // TODO: Remote server connections will be implemented in post-2.0.0 release
-        return false;
-    }
-    /**
-     * Disconnect from the remote server
-     * @returns True if successfully disconnected, false if not connected
-     */
-    async disconnectFromRemoteServer() {
-        // TODO: Remote server disconnection will be implemented in post-2.0.0 release
-        console.warn('disconnectFromRemoteServer: Remote server functionality not yet implemented in Brainy 2.0.0');
-        return false;
-    }
     /**
      * Ensure the database is initialized
      */
@@ -4289,10 +4341,6 @@ export class BrainyData {
                     // Continue with shutdown even if statistics flush fails
                 }
             }
-            // Disconnect from remote server if connected
-            if (this.isConnectedToRemoteServer()) {
-                await this.disconnectFromRemoteServer();
-            }
             // Clean up worker pools to release resources
             cleanupWorkerPools();
             // Additional cleanup could be added here in the future
@@ -4853,6 +4901,12 @@ export class BrainyData {
     // 6. update() - Update noun data/metadata with index sync
     // 7. delete() - Smart delete with soft delete default (enhanced original)
     // ========================================
+    // ================================================================
+    // SECTION 6: IMPORT & DATA INGESTION
+    // ================================================================
+    // Bulk data import and processing operations
+    // Includes import(), importSparseData() with neural type detection
+    // Smart parsing of CSV, JSON, YAML and automatic relationship extraction
     /**
      * Neural Import - Smart bulk data import with semantic type detection
      * Uses transformer embeddings to automatically detect and classify data types
@@ -4906,6 +4960,12 @@ export class BrainyData {
      * @param metadata Additional metadata
      * @returns Created noun ID
      */
+    // ================================================================
+    // SECTION 4: NOUN OPERATIONS (CRUD)
+    // ================================================================
+    // Create, Read, Update, Delete operations for noun entities
+    // Includes addNoun(), addNouns(), getNoun(), getNouns(), updateNoun(), deleteNoun()
+    // Core data storage and retrieval operations with metadata management
     /**
      * Add a noun to the database with required type
      * Clean 2.0 API - primary method for adding data
@@ -4926,33 +4986,62 @@ export class BrainyData {
         await this.ensureInitialized();
         // Check if database is in read-only mode
         this.checkReadOnly();
-        // Validate input is not null or undefined
-        if (vectorOrData === null || vectorOrData === undefined) {
-            throw new Error('Input cannot be null or undefined');
+        // Distributed routing for add operations
+        if (this.shardManager && this.networkTransport && !options.id) {
+            // Generate ID early for consistent shard routing
+            const id = uuidv4();
+            // Determine which shard this data belongs to
+            const shardAssignment = this.shardManager.getShardForKey(id);
+            // If this shard is owned by another node, forward the request
+            if (shardAssignment && shardAssignment.nodeId &&
+                shardAssignment.nodeId !== this.networkTransport.nodeId) {
+                try {
+                    const response = await this.networkTransport.sendToNode(shardAssignment.nodeId, 'DISTRIBUTED_ADD', {
+                        data: vectorOrData,
+                        type: nounType,
+                        metadata: enrichedMetadata,
+                        id // Use the generated ID
+                    });
+                    if (response.success) {
+                        return response.id;
+                    }
+                    else {
+                        throw new Error(`Remote add failed: ${response.error}`);
+                    }
+                }
+                catch (error) {
+                    console.warn(`Failed to forward add to ${shardAssignment.nodeId}, falling back to local add:`, error);
+                    // Fall through to local add
+                }
+            }
+            // If we're here, either we own the shard or forwarding failed
+            options.id = id; // Use the generated ID for local add
         }
+        // Comprehensive input validation
+        const validatedData = validateDataInput(vectorOrData, 'vectorOrData');
         try {
             let vector;
-            if (Array.isArray(vectorOrData)) {
-                for (let i = 0; i < vectorOrData.length; i++) {
-                    if (typeof vectorOrData[i] !== 'number') {
+            if (Array.isArray(validatedData)) {
+                for (let i = 0; i < validatedData.length; i++) {
+                    if (typeof validatedData[i] !== 'number') {
                         throw new Error('Vector contains non-numeric values');
                     }
                 }
             }
             // Check if input is already a vector
-            if (Array.isArray(vectorOrData) && !options.forceEmbed) {
+            if (Array.isArray(validatedData) && !options.forceEmbed) {
                 // Input is already a vector (and we've validated it contains only numbers)
-                vector = vectorOrData;
+                vector = validatedData;
             }
             else {
                 // Input needs to be vectorized
                 try {
                     // Check if input is a JSON object and process it specially
-                    if (typeof vectorOrData === 'object' &&
-                        vectorOrData !== null &&
-                        !Array.isArray(vectorOrData)) {
+                    if (typeof validatedData === 'object' &&
+                        validatedData !== null &&
+                        !Array.isArray(validatedData)) {
                         // Process JSON object for better vectorization
-                        const preparedText = prepareJsonForVectorization(vectorOrData, {
+                        const preparedText = prepareJsonForVectorization(validatedData, {
                             // Prioritize common name/title fields if they exist
                             priorityFields: [
                                 'name',
@@ -4971,17 +5060,17 @@ export class BrainyData {
                         // 2. addNoun(object) - vectorize object text, store object as metadata
                         // 3. addNoun(object, metadata) - vectorize object text, store provided metadata
                         if (!enrichedMetadata || Object.keys(enrichedMetadata).length === 1) { // Only has 'noun' key
-                            enrichedMetadata = { ...vectorOrData, noun: validatedType };
+                            enrichedMetadata = { ...validatedData, noun: validatedType };
                         }
                         // Track field names for this JSON document
                         const service = this.getServiceName(options);
                         if (this.storage) {
-                            await this.storage.trackFieldNames(vectorOrData, service);
+                            await this.storage.trackFieldNames(validatedData, service);
                         }
                     }
                     else {
                         // Use standard embedding for non-JSON data
-                        vector = await this.embeddingFunction(vectorOrData);
+                        vector = await this.embeddingFunction(validatedData);
                     }
                 }
                 catch (embedError) {
@@ -5166,15 +5255,6 @@ export class BrainyData {
                 const vectorCount = await this.getNounCount();
                 this.monitoring.updateVectorCount(vectorCount);
             }
-            // If addToRemote is true and we're connected to a remote server, add to remote as well
-            if (options.addToRemote && this.isConnectedToRemoteServer()) {
-                try {
-                    await this.addToRemote(id, vector, enrichedMetadata);
-                }
-                catch (remoteError) {
-                    console.warn(`Failed to add to remote server: ${remoteError}. Continuing with local add.`);
-                }
-            }
             // Invalidate search cache since data has changed
             this.cache?.invalidateOnDataChange('add');
             // Determine processing mode
@@ -5214,6 +5294,12 @@ export class BrainyData {
             throw new Error(`Failed to add vector: ${error}`);
         }
     }
+    // ================================================================
+    // SECTION 5: GRAPH OPERATIONS (VERBS)
+    // ================================================================
+    // Relationship and graph operations for verb entities
+    // Includes addVerb(), addVerbs(), getVerb(), getVerbs(), relationship queries
+    // Manages typed connections between nouns with metadata and weights
     /**
      * Add Verb - Unified relationship creation between nouns
      * Creates typed relationships with proper vector embeddings from metadata
@@ -5240,10 +5326,10 @@ export class BrainyData {
             const sourceNoun = this.index.getNouns().get(sourceId);
             const targetNoun = this.index.getNouns().get(targetId);
             if (!sourceNoun) {
-                throw new Error(`Source noun with ID ${sourceId} does not exist`);
+                throw BrainyError.notFound(`Source noun with ID ${sourceId}`);
             }
             if (!targetNoun) {
-                throw new Error(`Target noun with ID ${targetId} does not exist`);
+                throw BrainyError.notFound(`Target noun with ID ${targetId}`);
             }
             // Create embeddable text from verb type and metadata for searchability
             let embeddingText = `${verbType} relationship`;
@@ -5809,7 +5895,7 @@ export class BrainyData {
                 // For data updates, we need to regenerate the vector
                 const existingNoun = this.index.getNouns().get(id);
                 if (!existingNoun) {
-                    throw new Error(`Noun with ID ${id} does not exist`);
+                    throw BrainyError.notFound(`Noun with ID ${id}`);
                 }
                 // Get existing metadata from storage (not just from index)
                 const existingMetadata = await this.storage.getMetadata(id) || {};
@@ -5898,7 +5984,7 @@ export class BrainyData {
             // Check if a vector exists
             const noun = this.index.getNouns().get(id);
             if (!noun) {
-                throw new Error(`Vector with ID ${id} does not exist`);
+                throw BrainyError.notFound(`Vector with ID ${id}`);
             }
             // Get existing metadata to preserve namespaces
             const existing = await this.storage.getMetadata(id) || {};
@@ -6028,60 +6114,9 @@ export class BrainyData {
         const { limit = 10, offset = 0, cursor, mode = 'auto', maxDepth = 2, parallel = true, timeout, excludeDeleted = true } = options || {};
         // Validate and cap limit for safety
         const safeLimit = Math.min(limit, 10000);
-        if (!this._tripleEngine) {
-            this._tripleEngine = new TripleIntelligenceEngine(this);
-        }
-        // 🎆 NATURAL LANGUAGE AUTO-BREAKDOWN
-        // If query is a string, auto-convert to structured Triple Intelligence query
-        let processedQuery;
-        if (typeof query === 'string') {
-            // Use Brainy's sophisticated natural language processing
-            processedQuery = await this.processNaturalLanguage(query);
-        }
-        else {
-            processedQuery = query;
-        }
-        // Apply pagination options
-        processedQuery.limit = safeLimit;
-        // Handle cursor-based pagination
-        if (cursor) {
-            const decodedCursor = this.decodeCursor(cursor);
-            processedQuery.offset = decodedCursor.offset;
-        }
-        else if (offset > 0) {
-            processedQuery.offset = offset;
-        }
-        // Add soft-delete filter using POSITIVE match (O(1) hash lookup)
-        // We use _brainy.deleted to avoid conflicts with user metadata
-        if (excludeDeleted) {
-            if (!processedQuery.where) {
-                processedQuery.where = {};
-            }
-            // Use namespaced field for O(1) hash lookup in metadata index
-            processedQuery.where['_brainy.deleted'] = false; // or { equals: false }
-        }
-        // Apply mode-specific optimizations
-        if (mode !== 'auto') {
-            processedQuery.mode = mode;
-        }
-        // Apply graph traversal depth limit
-        if (processedQuery.connected) {
-            processedQuery.connected.maxDepth = Math.min(processedQuery.connected.maxDepth || maxDepth, maxDepth);
-        }
-        // Execute with Triple Intelligence engine
-        const results = await this._tripleEngine.find(processedQuery);
-        // Generate next cursor if we hit the limit
-        if (results.length === safeLimit) {
-            const nextCursor = this.encodeCursor({
-                offset: (offset || 0) + safeLimit,
-                timestamp: Date.now()
-            });
-            // Attach cursor to last result for convenience
-            if (results.length > 0) {
-                results[results.length - 1].nextCursor = nextCursor;
-            }
-        }
-        return results;
+        // Triple Intelligence now requires Brainy, not BrainyData
+        // BrainyData is deprecated - use Brainy.find() instead
+        throw new Error('Triple Intelligence search is not available in BrainyData. Please use Brainy.find() instead.');
     }
     /**
      * 🧠 NATURAL LANGUAGE PROCESSING - Auto-breakdown using all Brainy features
@@ -6231,7 +6266,7 @@ export class BrainyData {
      * @returns The BrainyData instance for chaining
      */
     unregister(name) {
-        augmentationPipeline.unregister(name);
+        // Deprecated: use brain.augmentations instead
         return this;
     }
     /**
@@ -6242,7 +6277,8 @@ export class BrainyData {
      * @returns True if augmentation was found and enabled
      */
     enableAugmentation(name) {
-        return augmentationPipeline.enableAugmentation(name);
+        // Deprecated: use brain.augmentations instead
+        return false;
     }
     /**
      * Disable an augmentation by name
@@ -6252,7 +6288,8 @@ export class BrainyData {
      * @returns True if augmentation was found and disabled
      */
     disableAugmentation(name) {
-        return augmentationPipeline.disableAugmentation(name);
+        // Deprecated: use brain.augmentations instead
+        return false;
     }
     /**
      * Check if an augmentation is enabled
@@ -6261,7 +6298,8 @@ export class BrainyData {
      * @returns True if augmentation is found and enabled, false otherwise
      */
     isAugmentationEnabled(name) {
-        return augmentationPipeline.isAugmentationEnabled(name);
+        // Deprecated: use brain.augmentations instead
+        return false;
     }
     /**
      * Get all augmentations with their enabled status
@@ -6285,7 +6323,8 @@ export class BrainyData {
      * @returns Number of augmentations enabled
      */
     enableAugmentationType(type) {
-        return augmentationPipeline.enableAugmentationType(type);
+        // Deprecated: use brain.augmentations instead
+        return 0;
     }
     /**
      * Disable all augmentations of a specific type
@@ -6294,7 +6333,8 @@ export class BrainyData {
      * @returns Number of augmentations disabled
      */
     disableAugmentationType(type) {
-        return augmentationPipeline.disableAugmentationType(type);
+        // Deprecated: use brain.augmentations instead
+        return 0;
     }
     // ===== Enhanced Clear Methods (2.0.0 API) =====
     /**
@@ -6385,6 +6425,12 @@ export class BrainyData {
             throw new Error(`Failed to clear all data: ${error}`);
         }
     }
+    // ================================================================
+    // SECTION 9: UTILITIES & CLEANUP  
+    // ================================================================
+    // System utilities, cleanup operations, and maintenance functions
+    // Includes clearAll(), database maintenance, and system shutdown operations
+    // Essential maintenance and administrative functionality
     /**
      * Clear all data from the database (alias for clear)
      * @param options Options including force flag to skip confirmation