@soulcraft/brainy 0.37.0 → 0.39.0

package/dist/unified.js

@@ -15417,6 +15417,1699 @@ class AugmentationPipeline {
 // Create and export a default instance of the pipeline
 const augmentationPipeline$1 = new AugmentationPipeline();
 
+/**
+ * Distributed Configuration Manager
+ * Manages shared configuration in S3 for distributed Brainy instances
+ */
+class DistributedConfigManager {
+    constructor(storage, distributedConfig, brainyMode) {
+        this.config = null;
+        this.lastConfigVersion = 0;
+        this.storage = storage;
+        this.instanceId = distributedConfig?.instanceId || `instance-${v4()}`;
+        this.configPath = distributedConfig?.configPath || '_brainy/config.json';
+        this.heartbeatInterval = distributedConfig?.heartbeatInterval || 30000;
+        this.configCheckInterval = distributedConfig?.configCheckInterval || 10000;
+        this.instanceTimeout = distributedConfig?.instanceTimeout || 60000;
+        // Set role from distributed config if provided
+        if (distributedConfig?.role) {
+            this.role = distributedConfig.role;
+        }
+        // Infer role from Brainy's read/write mode if not explicitly set
+        else if (brainyMode) {
+            if (brainyMode.writeOnly) {
+                this.role = 'writer';
+            }
+            else if (brainyMode.readOnly) {
+                this.role = 'reader';
+            }
+            // If neither readOnly nor writeOnly, role must be explicitly set
+        }
+    }
+    /**
+     * Initialize the distributed configuration
+     */
+    async initialize() {
+        // Load or create configuration
+        this.config = await this.loadOrCreateConfig();
+        // Determine role if not explicitly set
+        if (!this.role) {
+            this.role = await this.determineRole();
+        }
+        // Register this instance
+        await this.registerInstance();
+        // Start heartbeat and config watching
+        this.startHeartbeat();
+        this.startConfigWatch();
+        return this.config;
+    }
+    /**
+     * Load existing config or create new one
+     */
+    async loadOrCreateConfig() {
+        try {
+            // Use metadata storage with a special ID for config
+            const configData = await this.storage.getMetadata('_distributed_config');
+            if (configData) {
+                this.lastConfigVersion = configData.version;
+                return configData;
+            }
+        }
+        catch (error) {
+            // Config doesn't exist yet
+        }
+        // Create default config
+        const newConfig = {
+            version: 1,
+            updated: new Date().toISOString(),
+            settings: {
+                partitionStrategy: 'hash',
+                partitionCount: 100,
+                embeddingModel: 'text-embedding-ada-002',
+                dimensions: 1536,
+                distanceMetric: 'cosine',
+                hnswParams: {
+                    M: 16,
+                    efConstruction: 200
+                }
+            },
+            instances: {}
+        };
+        await this.saveConfig(newConfig);
+        return newConfig;
+    }
+    /**
+     * Determine role based on configuration
+     * IMPORTANT: Role must be explicitly set - no automatic assignment based on order
+     */
+    async determineRole() {
+        // Check environment variable first
+        if (process.env.BRAINY_ROLE) {
+            const role = process.env.BRAINY_ROLE.toLowerCase();
+            if (role === 'writer' || role === 'reader' || role === 'hybrid') {
+                return role;
+            }
+            throw new Error(`Invalid BRAINY_ROLE: ${process.env.BRAINY_ROLE}. Must be 'writer', 'reader', or 'hybrid'`);
+        }
+        // Check if explicitly passed in distributed config
+        if (this.role) {
+            return this.role;
+        }
+        // DO NOT auto-assign roles based on deployment order or existing instances
+        // This is dangerous and can lead to data corruption or loss
+        throw new Error('Distributed mode requires explicit role configuration. ' +
+            'Set BRAINY_ROLE environment variable or pass role in distributed config. ' +
+            'Valid roles: "writer", "reader", "hybrid"');
+    }
+    /**
+     * Check if an instance is still alive
+     */
+    isInstanceAlive(instance) {
+        const lastSeen = new Date(instance.lastHeartbeat).getTime();
+        const now = Date.now();
+        return (now - lastSeen) < this.instanceTimeout;
+    }
+    /**
+     * Register this instance in the shared config
+     */
+    async registerInstance() {
+        if (!this.config)
+            return;
+        // Role must be set by this point
+        if (!this.role) {
+            throw new Error('Cannot register instance without a role');
+        }
+        const instanceInfo = {
+            role: this.role,
+            status: 'active',
+            lastHeartbeat: new Date().toISOString(),
+            metrics: {
+                memoryUsage: process.memoryUsage().heapUsed
+            }
+        };
+        // Add endpoint if available
+        if (process.env.SERVICE_ENDPOINT) {
+            instanceInfo.endpoint = process.env.SERVICE_ENDPOINT;
+        }
+        this.config.instances[this.instanceId] = instanceInfo;
+        await this.saveConfig(this.config);
+    }
+    /**
+     * Save configuration with version increment
+     */
+    async saveConfig(config) {
+        config.version++;
+        config.updated = new Date().toISOString();
+        this.lastConfigVersion = config.version;
+        // Use metadata storage with a special ID for config
+        await this.storage.saveMetadata('_distributed_config', config);
+        this.config = config;
+    }
+    /**
+     * Start heartbeat to keep instance alive in config
+     */
+    startHeartbeat() {
+        this.heartbeatTimer = setInterval(async () => {
+            await this.updateHeartbeat();
+        }, this.heartbeatInterval);
+    }
+    /**
+     * Update heartbeat and clean stale instances
+     */
+    async updateHeartbeat() {
+        if (!this.config)
+            return;
+        // Reload config to get latest state
+        try {
+            const latestConfig = await this.loadConfig();
+            if (latestConfig) {
+                this.config = latestConfig;
+            }
+        }
+        catch (error) {
+            console.error('Failed to reload config:', error);
+        }
+        // Update our heartbeat
+        if (this.config.instances[this.instanceId]) {
+            this.config.instances[this.instanceId].lastHeartbeat = new Date().toISOString();
+            this.config.instances[this.instanceId].status = 'active';
+            // Update metrics if available
+            this.config.instances[this.instanceId].metrics = {
+                memoryUsage: process.memoryUsage().heapUsed
+            };
+        }
+        else {
+            // Re-register if we were removed
+            await this.registerInstance();
+            return;
+        }
+        // Clean up stale instances
+        const now = Date.now();
+        let hasChanges = false;
+        for (const [id, instance] of Object.entries(this.config.instances)) {
+            if (id === this.instanceId)
+                continue;
+            const lastSeen = new Date(instance.lastHeartbeat).getTime();
+            if (now - lastSeen > this.instanceTimeout) {
+                delete this.config.instances[id];
+                hasChanges = true;
+            }
+        }
+        // Save if there were changes
+        if (hasChanges) {
+            await this.saveConfig(this.config);
+        }
+        else {
+            // Just update our heartbeat without version increment
+            await this.storage.saveMetadata('_distributed_config', this.config);
+        }
+    }
+    /**
+     * Start watching for config changes
+     */
+    startConfigWatch() {
+        this.configWatchTimer = setInterval(async () => {
+            await this.checkForConfigUpdates();
+        }, this.configCheckInterval);
+    }
+    /**
+     * Check for configuration updates
+     */
+    async checkForConfigUpdates() {
+        try {
+            const latestConfig = await this.loadConfig();
+            if (!latestConfig)
+                return;
+            if (latestConfig.version > this.lastConfigVersion) {
+                this.config = latestConfig;
+                this.lastConfigVersion = latestConfig.version;
+                // Notify listeners of config update
+                if (this.onConfigUpdate) {
+                    this.onConfigUpdate(latestConfig);
+                }
+            }
+        }
+        catch (error) {
+            console.error('Failed to check config updates:', error);
+        }
+    }
+    /**
+     * Load configuration from storage
+     */
+    async loadConfig() {
+        try {
+            const configData = await this.storage.getMetadata('_distributed_config');
+            if (configData) {
+                return configData;
+            }
+        }
+        catch (error) {
+            console.error('Failed to load config:', error);
+        }
+        return null;
+    }
+    /**
+     * Get current configuration
+     */
+    getConfig() {
+        return this.config;
+    }
+    /**
+     * Get instance role
+     */
+    getRole() {
+        if (!this.role) {
+            throw new Error('Role not initialized');
+        }
+        return this.role;
+    }
+    /**
+     * Get instance ID
+     */
+    getInstanceId() {
+        return this.instanceId;
+    }
+    /**
+     * Set config update callback
+     */
+    setOnConfigUpdate(callback) {
+        this.onConfigUpdate = callback;
+    }
+    /**
+     * Get all active instances of a specific role
+     */
+    getInstancesByRole(role) {
+        if (!this.config)
+            return [];
+        return Object.entries(this.config.instances)
+            .filter(([_, instance]) => instance.role === role &&
+            this.isInstanceAlive(instance))
+            .map(([_, instance]) => instance);
+    }
+    /**
+     * Update instance metrics
+     */
+    async updateMetrics(metrics) {
+        if (!this.config || !this.config.instances[this.instanceId])
+            return;
+        this.config.instances[this.instanceId].metrics = {
+            ...this.config.instances[this.instanceId].metrics,
+            ...metrics
+        };
+        // Don't increment version for metric updates
+        await this.storage.saveMetadata('_distributed_config', this.config);
+    }
+    /**
+     * Cleanup resources
+     */
+    async cleanup() {
+        // Stop timers
+        if (this.heartbeatTimer) {
+            clearInterval(this.heartbeatTimer);
+        }
+        if (this.configWatchTimer) {
+            clearInterval(this.configWatchTimer);
+        }
+        // Mark instance as inactive
+        if (this.config && this.config.instances[this.instanceId]) {
+            this.config.instances[this.instanceId].status = 'inactive';
+            await this.saveConfig(this.config);
+        }
+    }
+}
+
+/**
+ * Cross-platform crypto utilities
+ * Provides hashing functions that work in both Node.js and browser environments
+ */
+/**
+ * Simple string hash function that works in all environments
+ * Uses djb2 algorithm - fast and good distribution
+ * @param str - String to hash
+ * @returns Positive integer hash
+ */
+function hashString(str) {
+    let hash = 5381;
+    for (let i = 0; i < str.length; i++) {
+        const char = str.charCodeAt(i);
+        hash = ((hash << 5) + hash) + char; // hash * 33 + char
+    }
+    // Ensure positive number
+    return Math.abs(hash);
+}
+/**
+ * Generate a deterministic hash for partitioning
+ * Uses the most appropriate algorithm for the environment
+ * @param input - Input string to hash
+ * @returns Positive integer hash suitable for modulo operations
+ */
+function getPartitionHash(input) {
+    // Use djb2 by default as it's fast and has good distribution
+    // This ensures consistent partitioning across all environments
+    return hashString(input);
+}
+
+/**
+ * Hash-based Partitioner
+ * Provides deterministic partitioning for distributed writes
+ */
+class HashPartitioner {
+    constructor(config) {
+        this.partitionPrefix = 'vectors/p';
+        this.partitionCount = config.settings.partitionCount || 100;
+    }
+    /**
+     * Get partition for a given vector ID using deterministic hashing
+     * @param vectorId - The unique identifier of the vector
+     * @returns The partition path
+     */
+    getPartition(vectorId) {
+        const hash = this.hashString(vectorId);
+        const partitionIndex = hash % this.partitionCount;
+        return `${this.partitionPrefix}${partitionIndex.toString().padStart(3, '0')}`;
+    }
+    /**
+     * Get partition with domain metadata (domain stored as metadata, not in path)
+     * @param vectorId - The unique identifier of the vector
+     * @param domain - The domain identifier (for metadata only)
+     * @returns The partition path
+     */
+    getPartitionWithDomain(vectorId, domain) {
+        // Domain doesn't affect partitioning - it's just metadata
+        return this.getPartition(vectorId);
+    }
+    /**
+     * Get all partition paths
+     * @returns Array of all partition paths
+     */
+    getAllPartitions() {
+        const partitions = [];
+        for (let i = 0; i < this.partitionCount; i++) {
+            partitions.push(`${this.partitionPrefix}${i.toString().padStart(3, '0')}`);
+        }
+        return partitions;
+    }
+    /**
+     * Get partition index from partition path
+     * @param partitionPath - The partition path
+     * @returns The partition index
+     */
+    getPartitionIndex(partitionPath) {
+        const match = partitionPath.match(/p(\d+)$/);
+        if (match) {
+            return parseInt(match[1], 10);
+        }
+        throw new Error(`Invalid partition path: ${partitionPath}`);
+    }
+    /**
+     * Hash a string to a number for consistent partitioning
+     * @param str - The string to hash
+     * @returns A positive integer hash
+     */
+    hashString(str) {
+        // Use our cross-platform hash function
+        return getPartitionHash(str);
+    }
+    /**
+     * Get partitions for batch operations
+     * Groups vector IDs by their target partition
+     * @param vectorIds - Array of vector IDs
+     * @returns Map of partition to vector IDs
+     */
+    getPartitionsForBatch(vectorIds) {
+        const partitionMap = new Map();
+        for (const id of vectorIds) {
+            const partition = this.getPartition(id);
+            if (!partitionMap.has(partition)) {
+                partitionMap.set(partition, []);
+            }
+            partitionMap.get(partition).push(id);
+        }
+        return partitionMap;
+    }
+}
+
+/**
+ * Operational Modes for Distributed Brainy
+ * Defines different modes with optimized caching strategies
+ */
+/**
+ * Base operational mode
+ */
+class BaseOperationalMode {
+    /**
+     * Validate operation is allowed in this mode
+     */
+    validateOperation(operation) {
+        switch (operation) {
+            case 'read':
+                if (!this.canRead) {
+                    throw new Error('Read operations are not allowed in write-only mode');
+                }
+                break;
+            case 'write':
+                if (!this.canWrite) {
+                    throw new Error('Write operations are not allowed in read-only mode');
+                }
+                break;
+            case 'delete':
+                if (!this.canDelete) {
+                    throw new Error('Delete operations are not allowed in this mode');
+                }
+                break;
+        }
+    }
+}
+/**
+ * Read-only mode optimized for query performance
+ */
+class ReaderMode extends BaseOperationalMode {
+    constructor() {
+        super(...arguments);
+        this.canRead = true;
+        this.canWrite = false;
+        this.canDelete = false;
+        this.cacheStrategy = {
+            hotCacheRatio: 0.8, // 80% of memory for read cache
+            prefetchAggressive: true, // Aggressively prefetch related vectors
+            ttl: 3600000, // 1 hour cache TTL
+            compressionEnabled: true, // Trade CPU for more cache capacity
+            writeBufferSize: 0, // No write buffer needed
+            batchWrites: false, // No writes
+            adaptive: true // Adapt to query patterns
+        };
+    }
+    /**
+     * Get optimized cache configuration for readers
+     */
+    getCacheConfig() {
+        return {
+            hotCacheMaxSize: 1000000, // Large hot cache
+            hotCacheEvictionThreshold: 0.9, // Keep cache full
+            warmCacheTTL: 3600000, // 1 hour warm cache
+            batchSize: 100, // Large batch reads
+            autoTune: true, // Auto-tune for read patterns
+            autoTuneInterval: 60000, // Tune every minute
+            readOnly: true // Enable read-only optimizations
+        };
+    }
+}
+/**
+ * Write-only mode optimized for ingestion
+ */
+class WriterMode extends BaseOperationalMode {
+    constructor() {
+        super(...arguments);
+        this.canRead = false;
+        this.canWrite = true;
+        this.canDelete = true;
+        this.cacheStrategy = {
+            hotCacheRatio: 0.2, // Only 20% for cache, rest for write buffer
+            prefetchAggressive: false, // No prefetching needed
+            ttl: 60000, // Short TTL (1 minute)
+            compressionEnabled: false, // Speed over memory efficiency
+            writeBufferSize: 10000, // Large write buffer for batching
+            batchWrites: true, // Enable write batching
+            adaptive: false // Fixed strategy for consistent writes
+        };
+    }
+    /**
+     * Get optimized cache configuration for writers
+     */
+    getCacheConfig() {
+        return {
+            hotCacheMaxSize: 100000, // Small hot cache
+            hotCacheEvictionThreshold: 0.5, // Aggressive eviction
+            warmCacheTTL: 60000, // 1 minute warm cache
+            batchSize: 1000, // Large batch writes
+            autoTune: false, // Fixed configuration
+            writeOnly: true // Enable write-only optimizations
+        };
+    }
+}
+/**
+ * Hybrid mode that can both read and write
+ */
+class HybridMode extends BaseOperationalMode {
+    constructor() {
+        super(...arguments);
+        this.canRead = true;
+        this.canWrite = true;
+        this.canDelete = true;
+        this.cacheStrategy = {
+            hotCacheRatio: 0.5, // Balanced cache/buffer allocation
+            prefetchAggressive: false, // Moderate prefetching
+            ttl: 600000, // 10 minute TTL
+            compressionEnabled: true, // Compress when beneficial
+            writeBufferSize: 5000, // Moderate write buffer
+            batchWrites: true, // Batch writes when possible
+            adaptive: true // Adapt to workload mix
+        };
+        this.readWriteRatio = 0.5; // Track read/write ratio
+    }
+    /**
+     * Get balanced cache configuration
+     */
+    getCacheConfig() {
+        return {
+            hotCacheMaxSize: 500000, // Medium cache size
+            hotCacheEvictionThreshold: 0.7, // Balanced eviction
+            warmCacheTTL: 600000, // 10 minute warm cache
+            batchSize: 500, // Medium batch size
+            autoTune: true, // Auto-tune based on workload
+            autoTuneInterval: 300000 // Tune every 5 minutes
+        };
+    }
+    /**
+     * Update cache strategy based on workload
+     * @param readCount - Number of recent reads
+     * @param writeCount - Number of recent writes
+     */
+    updateWorkloadBalance(readCount, writeCount) {
+        const total = readCount + writeCount;
+        if (total === 0)
+            return;
+        this.readWriteRatio = readCount / total;
+        // Adjust cache strategy based on workload
+        if (this.readWriteRatio > 0.8) {
+            // Read-heavy workload
+            this.cacheStrategy.hotCacheRatio = 0.7;
+            this.cacheStrategy.prefetchAggressive = true;
+            this.cacheStrategy.writeBufferSize = 2000;
+        }
+        else if (this.readWriteRatio < 0.2) {
+            // Write-heavy workload
+            this.cacheStrategy.hotCacheRatio = 0.3;
+            this.cacheStrategy.prefetchAggressive = false;
+            this.cacheStrategy.writeBufferSize = 8000;
+        }
+        else {
+            // Balanced workload
+            this.cacheStrategy.hotCacheRatio = 0.5;
+            this.cacheStrategy.prefetchAggressive = false;
+            this.cacheStrategy.writeBufferSize = 5000;
+        }
+    }
+}
+/**
+ * Factory for creating operational modes
+ */
+class OperationalModeFactory {
+    /**
+     * Create operational mode based on role
+     * @param role - The instance role
+     * @returns The appropriate operational mode
+     */
+    static createMode(role) {
+        switch (role) {
+            case 'reader':
+                return new ReaderMode();
+            case 'writer':
+                return new WriterMode();
+            case 'hybrid':
+                return new HybridMode();
+            default:
+                // Default to reader for safety
+                return new ReaderMode();
+        }
+    }
+    /**
+     * Create mode with custom cache strategy
+     * @param role - The instance role
+     * @param customStrategy - Custom cache strategy overrides
+     * @returns The operational mode with custom strategy
+     */
+    static createModeWithStrategy(role, customStrategy) {
+        const mode = this.createMode(role);
+        // Apply custom strategy overrides
+        mode.cacheStrategy = {
+            ...mode.cacheStrategy,
+            ...customStrategy
+        };
+        return mode;
+    }
+}
+
+/**
+ * Domain Detector
+ * Automatically detects and manages data domains for logical separation
+ */
+class DomainDetector {
+    constructor() {
+        this.domainPatterns = [
+            {
+                domain: 'medical',
+                patterns: {
+                    fields: ['symptoms', 'diagnosis', 'treatment', 'medication', 'patient'],
+                    keywords: ['medical', 'health', 'disease', 'symptom', 'treatment', 'doctor', 'patient']
+                },
+                priority: 1
+            },
+            {
+                domain: 'legal',
+                patterns: {
+                    fields: ['contract', 'clause', 'litigation', 'statute', 'jurisdiction'],
+                    keywords: ['legal', 'law', 'contract', 'court', 'attorney', 'litigation', 'statute']
+                },
+                priority: 1
+            },
+            {
+                domain: 'product',
+                patterns: {
+                    fields: ['price', 'sku', 'inventory', 'category', 'brand'],
+                    keywords: ['product', 'price', 'sale', 'inventory', 'catalog', 'item', 'sku']
+                },
+                priority: 1
+            },
+            {
+                domain: 'customer',
+                patterns: {
+                    fields: ['customerId', 'email', 'phone', 'address', 'orders'],
+                    keywords: ['customer', 'client', 'user', 'account', 'profile', 'contact']
+                },
+                priority: 1
+            },
+            {
+                domain: 'financial',
+                patterns: {
+                    fields: ['amount', 'currency', 'transaction', 'balance', 'account'],
+                    keywords: ['financial', 'money', 'payment', 'transaction', 'bank', 'credit', 'debit']
+                },
+                priority: 1
+            },
+            {
+                domain: 'technical',
+                patterns: {
+                    fields: ['code', 'function', 'error', 'stack', 'api'],
+                    keywords: ['code', 'software', 'api', 'error', 'debug', 'function', 'class', 'method']
+                },
+                priority: 2
+            }
+        ];
+        this.customPatterns = [];
+        this.domainStats = new Map();
+    }
+    /**
+     * Detect domain from data object
+     * @param data - The data object to analyze
+     * @returns The detected domain and metadata
+     */
+    detectDomain(data) {
+        if (!data || typeof data !== 'object') {
+            return { domain: 'general' };
+        }
+        // Check for explicit domain field
+        if (data.domain && typeof data.domain === 'string') {
+            this.updateStats(data.domain);
+            return {
+                domain: data.domain,
+                domainMetadata: this.extractDomainMetadata(data, data.domain)
+            };
+        }
+        // Score each domain pattern
+        const scores = new Map();
+        // Check custom patterns first (higher priority)
+        for (const pattern of this.customPatterns) {
+            const score = this.scorePattern(data, pattern);
+            if (score > 0) {
+                scores.set(pattern.domain, score * (pattern.priority || 1));
+            }
+        }
+        // Check default patterns
+        for (const pattern of this.domainPatterns) {
+            const score = this.scorePattern(data, pattern);
+            if (score > 0) {
+                const currentScore = scores.get(pattern.domain) || 0;
+                scores.set(pattern.domain, currentScore + score * (pattern.priority || 1));
+            }
+        }
+        // Find highest scoring domain
+        let bestDomain = 'general';
+        let bestScore = 0;
+        for (const [domain, score] of scores.entries()) {
+            if (score > bestScore) {
+                bestDomain = domain;
+                bestScore = score;
+            }
+        }
+        this.updateStats(bestDomain);
+        return {
+            domain: bestDomain,
+            domainMetadata: this.extractDomainMetadata(data, bestDomain)
+        };
+    }
+    /**
+     * Score a data object against a domain pattern
+     */
+    scorePattern(data, pattern) {
+        let score = 0;
+        // Check field matches
+        if (pattern.patterns.fields) {
+            const dataKeys = Object.keys(data);
+            for (const field of pattern.patterns.fields) {
+                if (dataKeys.some(key => key.toLowerCase().includes(field.toLowerCase()))) {
+                    score += 2; // Field match is strong signal
+                }
+            }
+        }
+        // Check keyword matches in values
+        if (pattern.patterns.keywords) {
+            const dataStr = JSON.stringify(data).toLowerCase();
+            for (const keyword of pattern.patterns.keywords) {
+                if (dataStr.includes(keyword.toLowerCase())) {
+                    score += 1;
+                }
+            }
+        }
+        // Check regex patterns
+        if (pattern.patterns.regex) {
+            const dataStr = JSON.stringify(data);
+            if (pattern.patterns.regex.test(dataStr)) {
+                score += 3; // Regex match is very specific
+            }
+        }
+        return score;
+    }
+    /**
+     * Extract domain-specific metadata
+     */
+    extractDomainMetadata(data, domain) {
+        const metadata = {};
+        switch (domain) {
+            case 'medical':
+                if (data.patientId)
+                    metadata.patientId = data.patientId;
+                if (data.condition)
+                    metadata.condition = data.condition;
+                if (data.severity)
+                    metadata.severity = data.severity;
+                break;
+            case 'legal':
+                if (data.caseId)
+                    metadata.caseId = data.caseId;
+                if (data.jurisdiction)
+                    metadata.jurisdiction = data.jurisdiction;
+                if (data.documentType)
+                    metadata.documentType = data.documentType;
+                break;
+            case 'product':
+                if (data.sku)
+                    metadata.sku = data.sku;
+                if (data.category)
+                    metadata.category = data.category;
+                if (data.brand)
+                    metadata.brand = data.brand;
+                if (data.price)
+                    metadata.priceRange = this.getPriceRange(data.price);
+                break;
+            case 'customer':
+                if (data.customerId)
+                    metadata.customerId = data.customerId;
+                if (data.segment)
+                    metadata.segment = data.segment;
+                if (data.lifetime_value)
+                    metadata.valueCategory = this.getValueCategory(data.lifetime_value);
+                break;
+            case 'financial':
+                if (data.accountId)
+                    metadata.accountId = data.accountId;
+                if (data.transactionType)
+                    metadata.transactionType = data.transactionType;
+                if (data.amount)
+                    metadata.amountRange = this.getAmountRange(data.amount);
+                break;
+            case 'technical':
+                if (data.service)
+                    metadata.service = data.service;
+                if (data.environment)
+                    metadata.environment = data.environment;
+                if (data.severity)
+                    metadata.severity = data.severity;
+                break;
+        }
+        // Add detection confidence
+        metadata.detectionConfidence = this.calculateConfidence(data, domain);
+        return metadata;
+    }
+    /**
+     * Calculate detection confidence
+     */
+    calculateConfidence(data, domain) {
+        // If domain was explicitly specified
+        if (data.domain === domain)
+            return 'high';
+        // Check how many patterns matched
+        const pattern = [...this.customPatterns, ...this.domainPatterns]
+            .find(p => p.domain === domain);
+        if (!pattern)
+            return 'low';
+        const score = this.scorePattern(data, pattern);
+        if (score >= 5)
+            return 'high';
+        if (score >= 2)
+            return 'medium';
+        return 'low';
+    }
+    /**
+     * Categorize price ranges
+     */
+    getPriceRange(price) {
+        if (price < 10)
+            return 'low';
+        if (price < 100)
+            return 'medium';
+        if (price < 1000)
+            return 'high';
+        return 'premium';
+    }
+    /**
+     * Categorize customer value
+     */
+    getValueCategory(value) {
+        if (value < 100)
+            return 'low';
+        if (value < 1000)
+            return 'medium';
+        if (value < 10000)
+            return 'high';
+        return 'vip';
+    }
+    /**
+     * Categorize amount ranges
+     */
+    getAmountRange(amount) {
+        if (amount < 100)
+            return 'micro';
+        if (amount < 1000)
+            return 'small';
+        if (amount < 10000)
+            return 'medium';
+        if (amount < 100000)
+            return 'large';
+        return 'enterprise';
+    }
+    /**
+     * Add custom domain pattern
+     * @param pattern - Custom domain pattern to add
+     */
+    addCustomPattern(pattern) {
+        // Remove existing pattern for same domain if exists
+        this.customPatterns = this.customPatterns.filter(p => p.domain !== pattern.domain);
+        this.customPatterns.push(pattern);
+    }
+    /**
+     * Remove custom domain pattern
+     * @param domain - Domain to remove pattern for
+     */
+    removeCustomPattern(domain) {
+        this.customPatterns = this.customPatterns.filter(p => p.domain !== domain);
+    }
+    /**
+     * Update domain statistics
+     */
+    updateStats(domain) {
+        const count = this.domainStats.get(domain) || 0;
+        this.domainStats.set(domain, count + 1);
+    }
+    /**
+     * Get domain statistics
+     * @returns Map of domain to count
+     */
+    getDomainStats() {
+        return new Map(this.domainStats);
+    }
+    /**
+     * Clear domain statistics
+     */
+    clearStats() {
+        this.domainStats.clear();
+    }
+    /**
+     * Get all configured domains
+     * @returns Array of domain names
+     */
+    getConfiguredDomains() {
+        const domains = new Set();
+        for (const pattern of [...this.domainPatterns, ...this.customPatterns]) {
+            domains.add(pattern.domain);
+        }
+        return Array.from(domains).sort();
+    }
+}
+
+/**
+ * Health Monitor
+ * Monitors and reports instance health in distributed deployments
+ */
+class HealthMonitor {
+    constructor(configManager) {
+        this.requestCount = 0;
+        this.errorCount = 0;
+        this.totalLatency = 0;
+        this.cacheHits = 0;
+        this.cacheMisses = 0;
+        this.vectorCount = 0;
+        this.checkInterval = 30000; // 30 seconds
+        this.metricsWindow = []; // Sliding window for RPS calculation
+        this.latencyWindow = []; // Sliding window for latency
+        this.windowSize = 60000; // 1 minute window
+        this.configManager = configManager;
+        this.startTime = Date.now();
+    }
+    /**
+     * Start health monitoring
+     */
+    start() {
+        // Initial health update
+        this.updateHealth();
+        // Schedule periodic health checks
+        this.healthCheckTimer = setInterval(() => {
+            this.updateHealth();
+        }, this.checkInterval);
+    }
+    /**
+     * Stop health monitoring
+     */
+    stop() {
+        if (this.healthCheckTimer) {
+            clearInterval(this.healthCheckTimer);
+            this.healthCheckTimer = undefined;
+        }
+    }
+    /**
+     * Update health status and metrics
+     */
+    async updateHealth() {
+        const metrics = this.collectMetrics();
+        // Update config with latest metrics
+        await this.configManager.updateMetrics({
+            vectorCount: metrics.vectorCount,
+            cacheHitRate: metrics.cacheHitRate,
+            memoryUsage: metrics.memoryUsage,
+            cpuUsage: metrics.cpuUsage
+        });
+        // Clean sliding windows
+        this.cleanWindows();
+    }
+    /**
+     * Collect current metrics
+     */
+    collectMetrics() {
+        const memUsage = process.memoryUsage();
+        return {
+            vectorCount: this.vectorCount,
+            cacheHitRate: this.calculateCacheHitRate(),
+            memoryUsage: memUsage.heapUsed,
+            cpuUsage: this.getCPUUsage(),
+            requestsPerSecond: this.calculateRPS(),
+            averageLatency: this.calculateAverageLatency(),
+            errorRate: this.calculateErrorRate()
+        };
+    }
+    /**
+     * Calculate cache hit rate
+     */
+    calculateCacheHitRate() {
+        const total = this.cacheHits + this.cacheMisses;
+        if (total === 0)
+            return 0;
+        return this.cacheHits / total;
+    }
+    /**
+     * Calculate requests per second
+     */
+    calculateRPS() {
+        const now = Date.now();
+        const recentRequests = this.metricsWindow.filter(timestamp => now - timestamp < this.windowSize);
+        return recentRequests.length / (this.windowSize / 1000);
+    }
+    /**
+     * Calculate average latency
+     */
+    calculateAverageLatency() {
+        if (this.latencyWindow.length === 0)
+            return 0;
+        const sum = this.latencyWindow.reduce((a, b) => a + b, 0);
+        return sum / this.latencyWindow.length;
+    }
+    /**
+     * Calculate error rate
+     */
+    calculateErrorRate() {
+        if (this.requestCount === 0)
+            return 0;
+        return this.errorCount / this.requestCount;
+    }
+    /**
+     * Get CPU usage (simplified)
+     */
+    getCPUUsage() {
+        // Simplified CPU usage based on process time
+        const usage = process.cpuUsage();
+        const total = usage.user + usage.system;
+        const seconds = (Date.now() - this.startTime) / 1000;
+        return Math.min(100, (total / 1000000 / seconds) * 100);
+    }
+    /**
+     * Clean old entries from sliding windows
+     */
+    cleanWindows() {
+        const now = Date.now();
+        const cutoff = now - this.windowSize;
+        this.metricsWindow = this.metricsWindow.filter(t => t > cutoff);
+        // Keep only recent latency measurements
+        if (this.latencyWindow.length > 100) {
+            this.latencyWindow = this.latencyWindow.slice(-100);
+        }
+    }
+    /**
+     * Record a request
+     * @param latency - Request latency in milliseconds
+     * @param error - Whether the request resulted in an error
+     */
+    recordRequest(latency, error = false) {
+        this.requestCount++;
+        this.metricsWindow.push(Date.now());
+        this.latencyWindow.push(latency);
+        if (error) {
+            this.errorCount++;
+        }
+    }
+    /**
+     * Record cache access
+     * @param hit - Whether it was a cache hit
+     */
+    recordCacheAccess(hit) {
+        if (hit) {
+            this.cacheHits++;
+        }
+        else {
+            this.cacheMisses++;
+        }
+    }
+    /**
+     * Update vector count
+     * @param count - New vector count
+     */
+    updateVectorCount(count) {
+        this.vectorCount = count;
+    }
+    /**
+     * Get current health status
+     * @returns Health status object
+     */
+    getHealthStatus() {
+        const metrics = this.collectMetrics();
+        const uptime = Date.now() - this.startTime;
+        const warnings = [];
+        const errors = [];
+        // Check for warnings
+        if (metrics.memoryUsage > 1024 * 1024 * 1024) { // > 1GB
+            warnings.push('High memory usage detected');
+        }
+        if (metrics.cacheHitRate < 0.5) {
+            warnings.push('Low cache hit rate');
+        }
+        if (metrics.errorRate && metrics.errorRate > 0.05) {
+            warnings.push('High error rate detected');
+        }
+        if (metrics.averageLatency && metrics.averageLatency > 1000) {
+            warnings.push('High latency detected');
+        }
+        // Check for errors
+        if (metrics.memoryUsage > 2 * 1024 * 1024 * 1024) { // > 2GB
+            errors.push('Critical memory usage');
+        }
+        if (metrics.errorRate && metrics.errorRate > 0.2) {
+            errors.push('Critical error rate');
+        }
+        // Determine overall status
+        let status = 'healthy';
+        if (errors.length > 0) {
+            status = 'unhealthy';
+        }
+        else if (warnings.length > 0) {
+            status = 'degraded';
+        }
+        return {
+            status,
+            instanceId: this.configManager.getInstanceId(),
+            role: this.configManager.getRole(),
+            uptime,
+            lastCheck: new Date().toISOString(),
+            metrics,
+            warnings: warnings.length > 0 ? warnings : undefined,
+            errors: errors.length > 0 ? errors : undefined
+        };
+    }
+    /**
+     * Get health check endpoint data
+     * @returns JSON-serializable health data
+     */
+    getHealthEndpointData() {
+        const status = this.getHealthStatus();
+        return {
+            status: status.status,
+            instanceId: status.instanceId,
+            role: status.role,
+            uptime: Math.floor(status.uptime / 1000), // Convert to seconds
+            lastCheck: status.lastCheck,
+            metrics: {
+                vectorCount: status.metrics.vectorCount,
+                cacheHitRate: Math.round(status.metrics.cacheHitRate * 100) / 100,
+                memoryUsageMB: Math.round(status.metrics.memoryUsage / 1024 / 1024),
+                cpuUsagePercent: Math.round(status.metrics.cpuUsage || 0),
+                requestsPerSecond: Math.round(status.metrics.requestsPerSecond || 0),
+                averageLatencyMs: Math.round(status.metrics.averageLatency || 0),
+                errorRate: Math.round((status.metrics.errorRate || 0) * 100) / 100
+            },
+            warnings: status.warnings,
+            errors: status.errors
+        };
+    }
+    /**
+     * Reset metrics (useful for testing)
+     */
+    resetMetrics() {
+        this.requestCount = 0;
+        this.errorCount = 0;
+        this.totalLatency = 0;
+        this.cacheHits = 0;
+        this.cacheMisses = 0;
+        this.metricsWindow = [];
+        this.latencyWindow = [];
+    }
+}
+
+/**
+ * SearchCache - Caches search results for improved performance
+ */
+class SearchCache {
+    constructor(config = {}) {
+        this.cache = new Map();
+        // Cache statistics
+        this.hits = 0;
+        this.misses = 0;
+        this.evictions = 0;
+        this.maxAge = config.maxAge ?? 5 * 60 * 1000; // 5 minutes
+        this.maxSize = config.maxSize ?? 100;
+        this.enabled = config.enabled ?? true;
+        this.hitCountWeight = config.hitCountWeight ?? 0.3;
+    }
+    /**
+     * Generate cache key from search parameters
+     */
+    getCacheKey(query, k, options = {}) {
+        // Create a normalized key that ignores order of options
+        const normalizedOptions = Object.keys(options)
+            .sort()
+            .reduce((acc, key) => {
+            // Skip cache-related options
+            if (key === 'skipCache' || key === 'useStreaming')
+                return acc;
+            acc[key] = options[key];
+            return acc;
+        }, {});
+        return JSON.stringify({
+            query: typeof query === 'object' ? JSON.stringify(query) : query,
+            k,
+            ...normalizedOptions
+        });
+    }
+    /**
+     * Get cached results if available and not expired
+     */
+    get(key) {
+        if (!this.enabled)
+            return null;
+        const entry = this.cache.get(key);
+        if (!entry) {
+            this.misses++;
+            return null;
+        }
+        // Check if expired
+        if (Date.now() - entry.timestamp > this.maxAge) {
+            this.cache.delete(key);
+            this.misses++;
+            return null;
+        }
+        // Update hit count and statistics
+        entry.hits++;
+        this.hits++;
+        return entry.results;
+    }
+    /**
+     * Cache search results
+     */
+    set(key, results) {
+        if (!this.enabled)
+            return;
+        // Evict if cache is full
+        if (this.cache.size >= this.maxSize) {
+            this.evictOldest();
+        }
+        this.cache.set(key, {
+            results: [...results], // Deep copy to prevent mutations
+            timestamp: Date.now(),
+            hits: 0
+        });
+    }
+    /**
+     * Evict the oldest entry based on timestamp and hit count
+     */
+    evictOldest() {
+        let oldestKey = null;
+        let oldestScore = Infinity;
+        const now = Date.now();
+        for (const [key, entry] of this.cache.entries()) {
+            // Score combines age and inverse hit count
+            const age = now - entry.timestamp;
+            const hitScore = entry.hits > 0 ? 1 / entry.hits : 1;
+            const score = age + (hitScore * this.hitCountWeight * this.maxAge);
+            if (score < oldestScore) {
+                oldestScore = score;
+                oldestKey = key;
+            }
+        }
+        if (oldestKey) {
+            this.cache.delete(oldestKey);
+            this.evictions++;
+        }
+    }
+    /**
+     * Clear all cached results
+     */
+    clear() {
+        this.cache.clear();
+        this.hits = 0;
+        this.misses = 0;
+        this.evictions = 0;
+    }
+    /**
+     * Invalidate cache entries that might be affected by data changes
+     */
+    invalidate(pattern) {
+        if (!pattern) {
+            this.clear();
+            return;
+        }
+        const keysToDelete = [];
+        for (const key of this.cache.keys()) {
+            const shouldDelete = typeof pattern === 'string'
+                ? key.includes(pattern)
+                : pattern.test(key);
+            if (shouldDelete) {
+                keysToDelete.push(key);
+            }
+        }
+        keysToDelete.forEach(key => this.cache.delete(key));
+    }
+    /**
+     * Smart invalidation for real-time data updates
+     * Only clears cache if it's getting stale or if data changes significantly
+     */
+    invalidateOnDataChange(changeType) {
+        // For now, clear all caches on data changes to ensure consistency
+        // In the future, we could implement more sophisticated invalidation
+        // based on the type of change and affected data
+        this.clear();
+    }
+    /**
+     * Check if cache entries have expired and remove them
+     * This is especially important in distributed scenarios where
+     * real-time updates might be delayed or missed
+     */
+    cleanupExpiredEntries() {
+        const now = Date.now();
+        const keysToDelete = [];
+        for (const [key, entry] of this.cache.entries()) {
+            if (now - entry.timestamp > this.maxAge) {
+                keysToDelete.push(key);
+            }
+        }
+        keysToDelete.forEach(key => this.cache.delete(key));
+        return keysToDelete.length;
+    }
+    /**
+     * Get cache statistics
+     */
+    getStats() {
+        const total = this.hits + this.misses;
+        return {
+            hits: this.hits,
+            misses: this.misses,
+            evictions: this.evictions,
+            hitRate: total > 0 ? this.hits / total : 0,
+            size: this.cache.size,
+            maxSize: this.maxSize,
+            enabled: this.enabled
+        };
+    }
+    /**
+     * Enable or disable caching
+     */
+    setEnabled(enabled) {
+        Object.defineProperty(this, 'enabled', { value: enabled, writable: false });
+        if (!enabled) {
+            this.clear();
+        }
+    }
+    /**
+     * Get memory usage estimate in bytes
+     */
+    getMemoryUsage() {
+        let totalSize = 0;
+        for (const [key, entry] of this.cache.entries()) {
+            // Estimate key size
+            totalSize += key.length * 2; // UTF-16 characters
+            // Estimate entry size
+            totalSize += JSON.stringify(entry.results).length * 2;
+            totalSize += 16; // timestamp + hits (8 bytes each)
+        }
+        return totalSize;
+    }
+    /**
+     * Get current cache configuration
+     */
+    getConfig() {
+        return {
+            enabled: this.enabled,
+            maxSize: this.maxSize,
+            maxAge: this.maxAge,
+            hitCountWeight: this.hitCountWeight
+        };
+    }
+    /**
+     * Update cache configuration dynamically
+     */
+    updateConfig(newConfig) {
+        if (newConfig.enabled !== undefined) {
+            this.enabled = newConfig.enabled;
+        }
+        if (newConfig.maxSize !== undefined) {
+            this.maxSize = newConfig.maxSize;
+            // Trigger eviction if current size exceeds new limit
+            this.evictIfNeeded();
+        }
+        if (newConfig.maxAge !== undefined) {
+            this.maxAge = newConfig.maxAge;
+            // Clean up entries that are now expired with new TTL
+            this.cleanupExpiredEntries();
+        }
+        if (newConfig.hitCountWeight !== undefined) {
+            this.hitCountWeight = newConfig.hitCountWeight;
+        }
+    }
+    /**
+     * Evict entries if cache exceeds maxSize
+     */
+    evictIfNeeded() {
+        if (this.cache.size <= this.maxSize) {
+            return;
+        }
+        // Calculate eviction score for each entry (same logic as existing eviction)
+        const entries = Array.from(this.cache.entries()).map(([key, entry]) => {
+            const age = Date.now() - entry.timestamp;
+            const hitCount = entry.hits;
+            // Eviction score: lower is more likely to be evicted
+            // Combines age and hit count (weighted by hitCountWeight)
+            const ageScore = age / this.maxAge;
+            const hitScore = 1 / (hitCount + 1); // Inverse of hits (more hits = lower score)
+            const score = ageScore * (1 - this.hitCountWeight) + hitScore * this.hitCountWeight;
+            return { key, entry, score };
+        });
+        // Sort by score (lowest first - these will be evicted)
+        entries.sort((a, b) => a.score - b.score);
+        // Evict entries until we're under the limit
+        const toEvict = entries.slice(0, this.cache.size - this.maxSize);
+        toEvict.forEach(({ key }) => {
+            this.cache.delete(key);
+            this.evictions++;
+        });
+    }
+}
+
+/**
+ * Intelligent cache auto-configuration system
+ * Adapts cache settings based on environment, usage patterns, and storage type
+ */
+class CacheAutoConfigurator {
+    constructor() {
+        this.stats = {
+            totalQueries: 0,
+            repeatQueries: 0,
+            avgQueryTime: 50,
+            memoryPressure: 0,
+            storageType: 'memory',
+            isDistributed: false,
+            changeFrequency: 0,
+            readWriteRatio: 10,
+        };
+        this.configHistory = [];
+        this.lastOptimization = 0;
+    }
+    /**
+     * Auto-detect optimal cache configuration based on current conditions
+     */
+    autoDetectOptimalConfig(storageConfig, currentStats) {
+        // Update stats with current information
+        if (currentStats) {
+            this.stats = { ...this.stats, ...currentStats };
+        }
+        // Detect environment characteristics
+        this.detectEnvironment(storageConfig);
+        // Generate optimal configuration
+        const result = this.generateOptimalConfig();
+        // Store for learning
+        this.configHistory.push(result);
+        this.lastOptimization = Date.now();
+        return result;
+    }
+    /**
+     * Dynamically adjust configuration based on runtime performance
+     */
+    adaptConfiguration(currentConfig, performanceMetrics) {
+        const reasoning = [];
+        let needsUpdate = false;
+        // Check if we should update (don't over-optimize)
+        if (Date.now() - this.lastOptimization < 60000) {
+            return null; // Wait at least 1 minute between optimizations
+        }
+        // Analyze performance patterns
+        const adaptations = {};
+        // Low hit rate → adjust cache size or TTL
+        if (performanceMetrics.hitRate < 0.3) {
+            if (performanceMetrics.externalChangesDetected > 5) {
+                // Too many external changes → shorter TTL
+                adaptations.maxAge = Math.max(60000, currentConfig.maxAge * 0.7);
+                reasoning.push('Reduced cache TTL due to frequent external changes');
+                needsUpdate = true;
+            }
+            else {
+                // Expand cache size for better hit rate
+                adaptations.maxSize = Math.min(500, (currentConfig.maxSize || 100) * 1.5);
+                reasoning.push('Increased cache size due to low hit rate');
+                needsUpdate = true;
+            }
+        }
+        // High hit rate but slow responses → might need cache warming
+        if (performanceMetrics.hitRate > 0.8 && performanceMetrics.avgResponseTime > 100) {
+            reasoning.push('High hit rate but slow responses - consider cache warming');
+        }
+        // Memory pressure → reduce cache size
+        if (performanceMetrics.memoryUsage > 100 * 1024 * 1024) { // 100MB
+            adaptations.maxSize = Math.max(20, (currentConfig.maxSize || 100) * 0.7);
+            reasoning.push('Reduced cache size due to memory pressure');
+            needsUpdate = true;
+        }
+        // Recent external changes → adaptive TTL
+        if (performanceMetrics.timeSinceLastChange < 30000) { // 30 seconds
+            adaptations.maxAge = Math.max(30000, currentConfig.maxAge * 0.8);
+            reasoning.push('Shortened TTL due to recent external changes');
+            needsUpdate = true;
+        }
+        if (!needsUpdate) {
+            return null;
+        }
+        const newCacheConfig = {
+            ...currentConfig,
+            ...adaptations
+        };
+        const newRealtimeConfig = this.calculateRealtimeConfig();
+        return {
+            cacheConfig: newCacheConfig,
+            realtimeConfig: newRealtimeConfig,
+            reasoning
+        };
+    }
+    /**
+     * Get recommended configuration for specific use case
+     */
+    getRecommendedConfig(useCase) {
+        const configs = {
+            'high-consistency': {
+                cache: { maxAge: 120000, maxSize: 50 },
+                realtime: { interval: 15000, enabled: true },
+                reasoning: ['Optimized for data consistency and real-time updates']
+            },
+            'balanced': {
+                cache: { maxAge: 300000, maxSize: 100 },
+                realtime: { interval: 30000, enabled: true },
+                reasoning: ['Balanced performance and consistency']
+            },
+            'performance-first': {
+                cache: { maxAge: 600000, maxSize: 200 },
+                realtime: { interval: 60000, enabled: true },
+                reasoning: ['Optimized for maximum cache performance']
+            }
+        };
+        const config = configs[useCase];
+        return {
+            cacheConfig: {
+                enabled: true,
+                ...config.cache
+            },
+            realtimeConfig: {
+                updateIndex: true,
+                updateStatistics: true,
+                ...config.realtime
+            },
+            reasoning: config.reasoning
+        };
+    }
+    /**
+     * Learn from usage patterns and improve recommendations
+     */
+    learnFromUsage(usageData) {
+        // Update internal stats for better future recommendations
+        this.stats.totalQueries += usageData.totalQueries;
+        this.stats.repeatQueries += usageData.cacheHits;
+        this.stats.avgQueryTime = (this.stats.avgQueryTime + usageData.responseTime) / 2;
+        this.stats.changeFrequency = usageData.dataChanges / (usageData.timeWindow / 60000);
+        // Calculate read/write ratio
+        const writes = usageData.dataChanges;
+        const reads = usageData.totalQueries;
+        this.stats.readWriteRatio = reads > 0 ? reads / Math.max(writes, 1) : 10;
+    }
+    detectEnvironment(storageConfig) {
+        // Detect storage type
+        if (storageConfig?.s3Storage || storageConfig?.customS3Storage) {
+            this.stats.storageType = 's3';
+            this.stats.isDistributed = true;
+        }
+        else if (storageConfig?.forceFileSystemStorage) {
+            this.stats.storageType = 'filesystem';
+        }
+        else if (storageConfig?.forceMemoryStorage) {
+            this.stats.storageType = 'memory';
+        }
+        else {
+            // Auto-detect browser vs Node.js
+            this.stats.storageType = typeof window !== 'undefined' ? 'opfs' : 'filesystem';
+        }
+        // Detect distributed mode indicators
+        this.stats.isDistributed = this.stats.isDistributed ||
+            Boolean(storageConfig?.s3Storage || storageConfig?.customS3Storage);
+    }
+    generateOptimalConfig() {
+        const reasoning = [];
+        // Base configuration
+        let cacheConfig = {
+            enabled: true,
+            maxSize: 100,
+            maxAge: 300000, // 5 minutes
+            hitCountWeight: 0.3
+        };
+        let realtimeConfig = {
+            enabled: false,
+            interval: 60000,
+            updateIndex: true,
+            updateStatistics: true
+        };
+        // Adjust for storage type
+        if (this.stats.storageType === 's3' || this.stats.isDistributed) {
+            cacheConfig.maxAge = 180000; // 3 minutes for distributed
+            realtimeConfig.enabled = true;
+            realtimeConfig.interval = 30000; // 30 seconds
+            reasoning.push('Distributed storage detected - enabled real-time updates');
+            reasoning.push('Reduced cache TTL for distributed consistency');
+        }
+        // Adjust for read/write patterns
+        if (this.stats.readWriteRatio > 20) {
+            // Read-heavy workload
+            cacheConfig.maxSize = Math.min(300, (cacheConfig.maxSize || 100) * 2);
+            cacheConfig.maxAge = Math.min(900000, (cacheConfig.maxAge || 300000) * 1.5); // Up to 15 minutes
+            reasoning.push('Read-heavy workload detected - increased cache size and TTL');
+        }
+        else if (this.stats.readWriteRatio < 5) {
+            // Write-heavy workload
+            cacheConfig.maxSize = Math.max(50, (cacheConfig.maxSize || 100) * 0.7);
+            cacheConfig.maxAge = Math.max(60000, (cacheConfig.maxAge || 300000) * 0.6);
+            reasoning.push('Write-heavy workload detected - reduced cache size and TTL');
+        }
+        // Adjust for change frequency
+        if (this.stats.changeFrequency > 10) { // More than 10 changes per minute
+            realtimeConfig.interval = Math.max(10000, realtimeConfig.interval * 0.5);
+            cacheConfig.maxAge = Math.max(30000, (cacheConfig.maxAge || 300000) * 0.5);
+            reasoning.push('High change frequency detected - increased update frequency');
+        }
+        // Memory constraints
+        if (this.detectMemoryConstraints()) {
+            cacheConfig.maxSize = Math.max(20, (cacheConfig.maxSize || 100) * 0.6);
+            reasoning.push('Memory constraints detected - reduced cache size');
+        }
+        // Performance optimization
+        if (this.stats.avgQueryTime > 200) {
+            cacheConfig.maxSize = Math.min(500, (cacheConfig.maxSize || 100) * 1.5);
+            reasoning.push('Slow queries detected - increased cache size');
+        }
+        return {
+            cacheConfig,
+            realtimeConfig,
+            reasoning
+        };
+    }
+    calculateRealtimeConfig() {
+        return {
+            enabled: this.stats.isDistributed || this.stats.changeFrequency > 1,
+            interval: this.stats.isDistributed ? 30000 : 60000,
+            updateIndex: true,
+            updateStatistics: true
+        };
+    }
+    detectMemoryConstraints() {
+        // Simple heuristic for memory constraints
+        try {
+            if (typeof performance !== 'undefined' && 'memory' in performance) {
+                const memInfo = performance.memory;
+                return memInfo.usedJSHeapSize > memInfo.jsHeapSizeLimit * 0.8;
+            }
+        }
+        catch (e) {
+            // Ignore errors
+        }
+        // Default assumption for constrained environments
+        return false;
+    }
+    /**
+     * Get human-readable explanation of current configuration
+     */
+    getConfigExplanation(config) {
+        const lines = [
+            '🤖 Brainy Auto-Configuration:',
+            '',
+            `📊 Cache: ${config.cacheConfig.maxSize} queries, ${config.cacheConfig.maxAge / 1000}s TTL`,
+            `🔄 Updates: ${config.realtimeConfig.enabled ? `Every ${(config.realtimeConfig.interval || 30000) / 1000}s` : 'Disabled'}`,
+            '',
+            '🎯 Optimizations applied:'
+        ];
+        config.reasoning.forEach(reason => {
+            lines.push(`  • ${reason}`);
+        });
+        return lines.join('\n');
+    }
+}
+
 /**
  * BrainyData
  * Main class that provides the vector database functionality
@@ -15470,6 +17163,13 @@ class BrainyData {
         this.remoteServerConfig = null;
         this.serverSearchConduit = null;
         this.serverConnection = null;
+        // Distributed mode properties
+        this.distributedConfig = null;
+        this.configManager = null;
+        this.partitioner = null;
+        this.operationalMode = null;
+        this.domainDetector = null;
+        this.healthMonitor = null;
         // Set dimensions to fixed value of 512 (Universal Sentence Encoder dimension)
         this._dimensions = 512;
         // Set distance function
@@ -15554,6 +17254,39 @@ class BrainyData {
                 ...config.cache
             };
         }
+        // Store distributed configuration
+        if (config.distributed) {
+            if (typeof config.distributed === 'boolean') {
+                // Auto-mode enabled
+                this.distributedConfig = {
+                    enabled: true
+                };
+            }
+            else {
+                // Explicit configuration
+                this.distributedConfig = config.distributed;
+            }
+        }
+        // Initialize cache auto-configurator first
+        this.cacheAutoConfigurator = new CacheAutoConfigurator();
+        // Auto-detect optimal cache configuration if not explicitly provided
+        let finalSearchCacheConfig = config.searchCache;
+        if (!config.searchCache || Object.keys(config.searchCache).length === 0) {
+            const autoConfig = this.cacheAutoConfigurator.autoDetectOptimalConfig(config.storage);
+            finalSearchCacheConfig = autoConfig.cacheConfig;
+            // Apply auto-detected real-time update configuration if not explicitly set
+            if (!config.realtimeUpdates && autoConfig.realtimeConfig.enabled) {
+                this.realtimeUpdateConfig = {
+                    ...this.realtimeUpdateConfig,
+                    ...autoConfig.realtimeConfig
+                };
+            }
+            if (this.loggingConfig?.verbose) {
+                console.log(this.cacheAutoConfigurator.getConfigExplanation(autoConfig));
+            }
+        }
+        // Initialize search cache with final configuration
+        this.searchCache = new SearchCache(finalSearchCacheConfig);
     }
     /**
      * Check if the database is in read-only mode and throw an error if it is
@@ -15693,6 +17426,17 @@ class BrainyData {
                     await this.applyChangesFromFullScan();
                 }
             }
+            // Cleanup expired cache entries (defensive mechanism for distributed scenarios)
+            const expiredCount = this.searchCache.cleanupExpiredEntries();
+            if (expiredCount > 0 && this.loggingConfig?.verbose) {
+                console.log(`Cleaned up ${expiredCount} expired cache entries`);
+            }
+            // Adapt cache configuration based on performance (every few updates)
+            // Only adapt every 5th update to avoid over-optimization
+            const updateCount = Math.floor((Date.now() - (this.lastUpdateTime || 0)) / this.realtimeUpdateConfig.interval);
+            if (updateCount % 5 === 0) {
+                this.adaptCacheConfiguration();
+            }
             // Update the last update time
             this.lastUpdateTime = Date.now();
             if (this.loggingConfig?.verbose) {
@@ -15767,6 +17511,13 @@ class BrainyData {
                 (addedCount > 0 || updatedCount > 0 || deletedCount > 0)) {
                 console.log(`Real-time update: Added ${addedCount}, updated ${updatedCount}, deleted ${deletedCount} nouns using change log`);
             }
+            // Invalidate search cache if any external changes were detected
+            if (addedCount > 0 || updatedCount > 0 || deletedCount > 0) {
+                this.searchCache.invalidateOnDataChange('update');
+                if (this.loggingConfig?.verbose) {
+                    console.log('Search cache invalidated due to external data changes');
+                }
+            }
             // Update the last known noun count
             this.lastKnownNounCount = await this.getNounCount();
         }
@@ -15810,6 +17561,13 @@ class BrainyData {
                 }
                 // Update the last known noun count
                 this.lastKnownNounCount = currentCount;
+                // Invalidate search cache if new nouns were detected
+                if (newNouns.length > 0) {
+                    this.searchCache.invalidateOnDataChange('add');
+                    if (this.loggingConfig?.verbose) {
+                        console.log('Search cache invalidated due to external data changes');
+                    }
+                }
                 if (this.loggingConfig?.verbose && newNouns.length > 0) {
                     console.log(`Real-time update: Added ${newNouns.length} new nouns to index using full scan`);
                 }
@@ -15941,6 +17699,10 @@ class BrainyData {
             }
             // Initialize storage
             await this.storage.init();
+            // Initialize distributed mode if configured
+            if (this.distributedConfig) {
+                await this.initializeDistributedMode();
+            }
             // If using optimized index, set the storage adapter
             if (this.useOptimizedIndex && this.index instanceof HNSWIndexOptimized) {
                 this.index.setStorage(this.storage);
@@ -16000,6 +17762,97 @@ class BrainyData {
             throw new Error(`Failed to initialize BrainyData: ${error}`);
         }
     }
+    /**
+     * Initialize distributed mode
+     * Sets up configuration management, partitioning, and operational modes
+     */
+    async initializeDistributedMode() {
+        if (!this.storage) {
+            throw new Error('Storage must be initialized before distributed mode');
+        }
+        // Create configuration manager with mode hints
+        this.configManager = new DistributedConfigManager(this.storage, this.distributedConfig || undefined, { readOnly: this.readOnly, writeOnly: this.writeOnly });
+        // Initialize configuration
+        const sharedConfig = await this.configManager.initialize();
+        // Create partitioner based on strategy
+        if (sharedConfig.settings.partitionStrategy === 'hash') {
+            this.partitioner = new HashPartitioner(sharedConfig);
+        }
+        else {
+            // Default to hash partitioner for now
+            this.partitioner = new HashPartitioner(sharedConfig);
+        }
+        // Create operational mode based on role
+        const role = this.configManager.getRole();
+        this.operationalMode = OperationalModeFactory.createMode(role);
+        // Validate that role matches the configured mode
+        // Don't override explicitly set readOnly/writeOnly
+        if (role === 'reader' && !this.readOnly) {
+            console.warn('Distributed role is "reader" but readOnly is not set. Setting readOnly=true for consistency.');
+            this.readOnly = true;
+            this.writeOnly = false;
+        }
+        else if (role === 'writer' && !this.writeOnly) {
+            console.warn('Distributed role is "writer" but writeOnly is not set. Setting writeOnly=true for consistency.');
+            this.readOnly = false;
+            this.writeOnly = true;
+        }
+        else if (role === 'hybrid' && (this.readOnly || this.writeOnly)) {
+            console.warn('Distributed role is "hybrid" but readOnly or writeOnly is set. Clearing both for hybrid mode.');
+            this.readOnly = false;
+            this.writeOnly = false;
+        }
+        // Apply cache configuration from operational mode
+        const modeCache = this.operationalMode.cacheStrategy;
+        if (modeCache) {
+            this.cacheConfig = {
+                ...this.cacheConfig,
+                hotCacheMaxSize: modeCache.hotCacheRatio * 1000000, // Convert ratio to size
+                hotCacheEvictionThreshold: modeCache.hotCacheRatio,
+                warmCacheTTL: modeCache.ttl,
+                batchSize: modeCache.writeBufferSize || 100
+            };
+            // Update storage cache config if it supports it
+            if (this.storage && 'updateCacheConfig' in this.storage) {
+                this.storage.updateCacheConfig(this.cacheConfig);
+            }
+        }
+        // Initialize domain detector
+        this.domainDetector = new DomainDetector();
+        // Initialize health monitor
+        this.healthMonitor = new HealthMonitor(this.configManager);
+        this.healthMonitor.start();
+        // Set up config update listener
+        this.configManager.setOnConfigUpdate((config) => {
+            this.handleDistributedConfigUpdate(config);
+        });
+        if (this.loggingConfig?.verbose) {
+            console.log(`Distributed mode initialized as ${role} with ${sharedConfig.settings.partitionStrategy} partitioning`);
+        }
+    }
+    /**
+     * Handle distributed configuration updates
+     */
+    handleDistributedConfigUpdate(config) {
+        // Update partitioner if needed
+        if (this.partitioner && config.settings) {
+            this.partitioner = new HashPartitioner(config);
+        }
+        // Log configuration update
+        if (this.loggingConfig?.verbose) {
+            console.log('Distributed configuration updated:', config.version);
+        }
+    }
+    /**
+     * Get distributed health status
+     * @returns Health status if distributed mode is enabled
+     */
+    getHealthStatus() {
+        if (this.healthMonitor) {
+            return this.healthMonitor.getHealthEndpointData();
+        }
+        return null;
+    }
     /**
      * Connect to a remote Brainy server for search operations
      * @param serverUrl WebSocket URL of the remote Brainy server
@@ -16216,6 +18069,33 @@ class BrainyData {
                     if (metadata && typeof metadata === 'object') {
                         // Always make a copy without adding the ID
                         metadataToSave = { ...metadata };
+                        // Add domain metadata if distributed mode is enabled
+                        if (this.domainDetector) {
+                            // First check if domain is already in metadata
+                            if (metadataToSave.domain) {
+                                // Domain already specified, keep it
+                                const domainInfo = this.domainDetector.detectDomain(metadataToSave);
+                                if (domainInfo.domainMetadata) {
+                                    metadataToSave.domainMetadata = domainInfo.domainMetadata;
+                                }
+                            }
+                            else {
+                                // Try to detect domain from the data
+                                const dataToAnalyze = Array.isArray(vectorOrData) ? metadata : vectorOrData;
+                                const domainInfo = this.domainDetector.detectDomain(dataToAnalyze);
+                                if (domainInfo.domain) {
+                                    metadataToSave.domain = domainInfo.domain;
+                                    if (domainInfo.domainMetadata) {
+                                        metadataToSave.domainMetadata = domainInfo.domainMetadata;
+                                    }
+                                }
+                            }
+                        }
+                        // Add partition information if distributed mode is enabled
+                        if (this.partitioner) {
+                            const partition = this.partitioner.getPartition(id);
+                            metadataToSave.partition = partition;
+                        }
                     }
                     await this.storage.saveMetadata(id, metadataToSave);
                     // Track metadata statistics
@@ -16225,6 +18105,11 @@ class BrainyData {
             }
             // Update HNSW index size (excluding verbs)
             await this.storage.updateHnswIndexSize(await this.getNounCount());
+            // Update health metrics if in distributed mode
+            if (this.healthMonitor) {
+                const vectorCount = await this.getNounCount();
+                this.healthMonitor.updateVectorCount(vectorCount);
+            }
             // If addToRemote is true and we're connected to a remote server, add to remote as well
             if (options.addToRemote && this.isConnectedToRemoteServer()) {
                 try {
@@ -16234,10 +18119,16 @@ class BrainyData {
                     console.warn(`Failed to add to remote server: ${remoteError}. Continuing with local add.`);
                 }
             }
+            // Invalidate search cache since data has changed
+            this.searchCache.invalidateOnDataChange('add');
             return id;
         }
         catch (error) {
             console.error('Failed to add vector:', error);
+            // Track error in health monitor
+            if (this.healthMonitor) {
+                this.healthMonitor.recordRequest(0, true);
+            }
             throw new Error(`Failed to add vector: ${error}`);
         }
     }
@@ -16487,11 +18378,16 @@ class BrainyData {
                         console.log(`Lazy loading mode: Added ${limitedNouns.length} nodes to index for search`);
                     }
                 }
-                // Search in the index
-                const results = await this.index.search(queryVector, k);
+                // When using offset, we need to fetch more results and then slice
+                const offset = options.offset || 0;
+                const totalNeeded = k + offset;
+                // Search in the index for totalNeeded results
+                const results = await this.index.search(queryVector, totalNeeded);
+                // Skip the offset number of results
+                const paginatedResults = results.slice(offset, offset + k);
                 // Get metadata for each result
                 const searchResults = [];
-                for (const [id, score] of results) {
+                for (const [id, score] of paginatedResults) {
                     const noun = this.index.getNouns().get(id);
                     if (!noun) {
                         continue;
@@ -16532,8 +18428,9 @@ class BrainyData {
                 }
                 // Sort by distance (ascending)
                 results.sort((a, b) => a[1] - b[1]);
-                // Take top k results
-                const topResults = results.slice(0, k);
+                // Apply offset and take k results
+                const offset = options.offset || 0;
+                const topResults = results.slice(offset, offset + k);
                 // Get metadata for each result
                 const searchResults = [];
                 for (const [id, score] of topResults) {
@@ -16574,6 +18471,7 @@ class BrainyData {
      * @returns Array of search results
      */
     async search(queryVectorOrData, k = 10, options = {}) {
+        const startTime = Date.now();
         // Validate input is not null or undefined
         if (queryVectorOrData === null || queryVectorOrData === undefined) {
             throw new Error('Query cannot be null or undefined');
@@ -16625,7 +18523,93 @@ class BrainyData {
             return this.searchCombined(queryVectorOrData, k, options);
         }
         // Default behavior (backward compatible): search locally
-        return this.searchLocal(queryVectorOrData, k, options);
+        try {
+            // Check cache first (transparent to user)
+            const cacheKey = this.searchCache.getCacheKey(queryVectorOrData, k, options);
+            const cachedResults = this.searchCache.get(cacheKey);
+            if (cachedResults) {
+                // Track cache hit in health monitor
+                if (this.healthMonitor) {
+                    const latency = Date.now() - startTime;
+                    this.healthMonitor.recordRequest(latency, false);
+                    this.healthMonitor.recordCacheAccess(true);
+                }
+                return cachedResults;
+            }
+            // Cache miss - perform actual search
+            const results = await this.searchLocal(queryVectorOrData, k, options);
+            // Cache results for future queries (unless explicitly disabled)
+            if (!options.skipCache) {
+                this.searchCache.set(cacheKey, results);
+            }
+            // Track successful search in health monitor
+            if (this.healthMonitor) {
+                const latency = Date.now() - startTime;
+                this.healthMonitor.recordRequest(latency, false);
+                this.healthMonitor.recordCacheAccess(false);
+            }
+            return results;
+        }
+        catch (error) {
+            // Track error in health monitor
+            if (this.healthMonitor) {
+                const latency = Date.now() - startTime;
+                this.healthMonitor.recordRequest(latency, true);
+            }
+            throw error;
+        }
+    }
+    /**
+     * Search with cursor-based pagination for better performance on large datasets
+     * @param queryVectorOrData Query vector or data to search for
+     * @param k Number of results to return
+     * @param options Additional options including cursor for pagination
+     * @returns Paginated search results with cursor for next page
+     */
+    async searchWithCursor(queryVectorOrData, k = 10, options = {}) {
+        // For cursor-based search, we need to fetch more results and filter
+        const searchK = options.cursor ? k + 20 : k; // Get extra results for filtering
+        // Perform regular search
+        const allResults = await this.search(queryVectorOrData, searchK, {
+            ...options,
+            skipCache: options.skipCache
+        });
+        let results = allResults;
+        let startIndex = 0;
+        // If cursor provided, find starting position
+        if (options.cursor) {
+            startIndex = allResults.findIndex(r => r.id === options.cursor.lastId &&
+                Math.abs(r.score - options.cursor.lastScore) < 0.0001);
+            if (startIndex >= 0) {
+                startIndex += 1; // Start after the cursor position
+                results = allResults.slice(startIndex, startIndex + k);
+            }
+            else {
+                // Cursor not found, might be stale - return from beginning
+                results = allResults.slice(0, k);
+                startIndex = 0;
+            }
+        }
+        else {
+            results = allResults.slice(0, k);
+        }
+        // Create cursor for next page
+        let nextCursor;
+        const hasMoreResults = (startIndex + results.length) < allResults.length || allResults.length >= searchK;
+        if (results.length > 0 && hasMoreResults) {
+            const lastResult = results[results.length - 1];
+            nextCursor = {
+                lastId: lastResult.id,
+                lastScore: lastResult.score,
+                position: startIndex + results.length
+            };
+        }
+        return {
+            results,
+            cursor: nextCursor,
+            hasMore: !!nextCursor,
+            totalEstimate: allResults.length > searchK ? undefined : allResults.length
+        };
     }
     /**
      * Search the local database for similar vectors
@@ -16682,14 +18666,16 @@ class BrainyData {
         if (options.nounTypes && options.nounTypes.length > 0) {
             searchResults = await this.searchByNounTypes(queryToUse, k, options.nounTypes, {
                 forceEmbed: options.forceEmbed,
-                service: options.service
+                service: options.service,
+                offset: options.offset
             });
         }
         else {
             // Otherwise, search all GraphNouns
             searchResults = await this.searchByNounTypes(queryToUse, k, null, {
                 forceEmbed: options.forceEmbed,
-                service: options.service
+                service: options.service,
+                offset: options.offset
             });
         }
         // Filter out placeholder nouns from search results
@@ -16697,7 +18683,15 @@ class BrainyData {
             if (result.metadata && typeof result.metadata === 'object') {
                 const metadata = result.metadata;
                 // Exclude placeholder nouns from search results
-                return !metadata.isPlaceholder;
+                if (metadata.isPlaceholder) {
+                    return false;
+                }
+                // Apply domain filter if specified
+                if (options.filter?.domain) {
+                    if (metadata.domain !== options.filter.domain) {
+                        return false;
+                    }
+                }
             }
             return true;
         });
@@ -17015,6 +19009,8 @@ class BrainyData {
             catch (error) {
                 // Ignore
             }
+            // Invalidate search cache since data has changed
+            this.searchCache.invalidateOnDataChange('delete');
             return true;
         }
         catch (error) {
@@ -17098,6 +19094,8 @@ class BrainyData {
             // Track metadata statistics
             const service = this.getServiceName(options);
             await this.storage.incrementStatistic('metadata', service);
+            // Invalidate search cache since metadata has changed
+            this.searchCache.invalidateOnDataChange('update');
             return true;
         }
         catch (error) {
@@ -17452,6 +19450,8 @@ class BrainyData {
             await this.storage.incrementStatistic('verb', serviceForStats);
             // Update HNSW index size (excluding verbs)
             await this.storage.updateHnswIndexSize(await this.getNounCount());
+            // Invalidate search cache since verb data has changed
+            this.searchCache.invalidateOnDataChange('add');
             return id;
         }
         catch (error) {
@@ -17674,6 +19674,8 @@ class BrainyData {
             await this.index.clear();
             // Clear storage
             await this.storage.clear();
+            // Clear search cache since all data has been removed
+            this.searchCache.invalidateOnDataChange('delete');
         }
         catch (error) {
             console.error('Failed to clear vector database:', error);
@@ -17686,6 +19688,66 @@ class BrainyData {
     size() {
         return this.index.size();
     }
+    /**
+     * Get search cache statistics for performance monitoring
+     * @returns Cache statistics including hit rate and memory usage
+     */
+    getCacheStats() {
+        return {
+            search: this.searchCache.getStats(),
+            searchMemoryUsage: this.searchCache.getMemoryUsage()
+        };
+    }
+    /**
+     * Clear search cache manually (useful for testing or memory management)
+     */
+    clearCache() {
+        this.searchCache.clear();
+    }
+    /**
+     * Adapt cache configuration based on current performance metrics
+     * This method analyzes usage patterns and automatically optimizes cache settings
+     * @private
+     */
+    adaptCacheConfiguration() {
+        const stats = this.searchCache.getStats();
+        const memoryUsage = this.searchCache.getMemoryUsage();
+        const currentConfig = this.searchCache.getConfig();
+        // Prepare performance metrics for adaptation
+        const performanceMetrics = {
+            hitRate: stats.hitRate,
+            avgResponseTime: 50, // Would be measured in real implementation
+            memoryUsage: memoryUsage,
+            externalChangesDetected: 0, // Would be tracked from real-time updates
+            timeSinceLastChange: Date.now() - this.lastUpdateTime
+        };
+        // Try to adapt configuration
+        const newConfig = this.cacheAutoConfigurator.adaptConfiguration(currentConfig, performanceMetrics);
+        if (newConfig) {
+            // Apply new cache configuration
+            this.searchCache.updateConfig(newConfig.cacheConfig);
+            // Apply new real-time update configuration if needed
+            if (newConfig.realtimeConfig.enabled !== this.realtimeUpdateConfig.enabled ||
+                newConfig.realtimeConfig.interval !== this.realtimeUpdateConfig.interval) {
+                const wasEnabled = this.realtimeUpdateConfig.enabled;
+                this.realtimeUpdateConfig = {
+                    ...this.realtimeUpdateConfig,
+                    ...newConfig.realtimeConfig
+                };
+                // Restart real-time updates with new configuration
+                if (wasEnabled) {
+                    this.stopRealtimeUpdates();
+                }
+                if (this.realtimeUpdateConfig.enabled && this.isInitialized) {
+                    this.startRealtimeUpdates();
+                }
+            }
+            if (this.loggingConfig?.verbose) {
+                console.log('🔧 Auto-adapted cache configuration:');
+                console.log(this.cacheAutoConfigurator.getConfigExplanation(newConfig));
+            }
+        }
+    }
     /**
      * Get the number of nouns in the database (excluding verbs)
      * This is used for statistics reporting to match the expected behavior in tests
@@ -18232,12 +20294,17 @@ class BrainyData {
             if (!this.serverSearchConduit || !this.serverConnection) {
                 throw new Error('Server search conduit or connection is not initialized');
             }
-            // Search the remote server
-            const searchResult = await this.serverSearchConduit.searchServer(this.serverConnection.connectionId, query, k);
+            // When using offset, fetch more results and slice
+            const offset = options.offset || 0;
+            const totalNeeded = k + offset;
+            // Search the remote server for totalNeeded results
+            const searchResult = await this.serverSearchConduit.searchServer(this.serverConnection.connectionId, query, totalNeeded);
             if (!searchResult.success) {
                 throw new Error(`Remote search failed: ${searchResult.error}`);
             }
-            return searchResult.data;
+            // Apply offset to remote results
+            const allResults = searchResult.data;
+            return allResults.slice(offset, offset + k);
         }
         catch (error) {
             console.error('Failed to search remote server:', error);
@@ -18889,6 +20956,26 @@ class BrainyData {
         // Sort by score and limit to k results
         return allResults.sort((a, b) => b.score - a.score).slice(0, k);
     }
+    /**
+     * Cleanup distributed resources
+     * Should be called when shutting down the instance
+     */
+    async cleanup() {
+        // Stop real-time updates
+        if (this.updateTimerId) {
+            clearInterval(this.updateTimerId);
+            this.updateTimerId = null;
+        }
+        // Clean up distributed mode resources
+        if (this.healthMonitor) {
+            this.healthMonitor.stop();
+        }
+        if (this.configManager) {
+            await this.configManager.cleanup();
+        }
+        // Clean up worker pools
+        await cleanupWorkerPools();
+    }
 }
 
 /**