@soulcraft/brainy 0.38.0 → 0.40.0

package/dist/unified.js

@@ -4939,7 +4939,8 @@ class HNSWIndex {
         const noun = {
             id,
             vector,
-            connections: new Map()
+            connections: new Map(),
+            level: nounLevel
         };
         // Initialize empty connection sets for each level
         for (let level = 0; level <= nounLevel; level++) {
@@ -5264,6 +5265,29 @@ class HNSWIndex {
     getConfig() {
         return { ...this.config };
     }
+    /**
+     * Get index health metrics
+     */
+    getIndexHealth() {
+        let totalConnections = 0;
+        const layerCounts = new Array(this.maxLevel + 1).fill(0);
+        // Count connections and layer distribution
+        this.nouns.forEach(noun => {
+            // Count connections at each layer
+            for (let level = 0; level <= noun.level; level++) {
+                totalConnections += noun.connections.get(level)?.size || 0;
+                layerCounts[level]++;
+            }
+        });
+        const totalNodes = this.nouns.size;
+        const averageConnections = totalNodes > 0 ? totalConnections / totalNodes : 0;
+        return {
+            averageConnections,
+            layerDistribution: layerCounts,
+            maxLayer: this.maxLevel,
+            totalNodes
+        };
+    }
     /**
      * Search within a specific layer
      * Returns a map of noun IDs to distances, sorted by distance
@@ -5721,7 +5745,8 @@ class HNSWIndexOptimized extends HNSWIndex {
             const noun = {
                 id,
                 vector,
-                connections: new Map()
+                connections: new Map(),
+                level: 0
             };
             // Store the noun
             this.storage.saveNoun(noun).catch((error) => {
@@ -6869,7 +6894,8 @@ class MemoryStorage extends BaseStorage {
         const nounCopy = {
             id: noun.id,
             vector: [...noun.vector],
-            connections: new Map()
+            connections: new Map(),
+            level: noun.level || 0
         };
         // Copy connections
         for (const [level, connections] of noun.connections.entries()) {
@@ -6892,7 +6918,8 @@ class MemoryStorage extends BaseStorage {
         const nounCopy = {
             id: noun.id,
             vector: [...noun.vector],
-            connections: new Map()
+            connections: new Map(),
+            level: noun.level || 0
         };
         // Copy connections
         for (const [level, connections] of noun.connections.entries()) {
@@ -6911,7 +6938,8 @@ class MemoryStorage extends BaseStorage {
             const nounCopy = {
                 id: noun.id,
                 vector: [...noun.vector],
-                connections: new Map()
+                connections: new Map(),
+                level: noun.level || 0
             };
             // Copy connections
             for (const [level, connections] of noun.connections.entries()) {
@@ -6986,7 +7014,8 @@ class MemoryStorage extends BaseStorage {
             const nounCopy = {
                 id: noun.id,
                 vector: [...noun.vector],
-                connections: new Map()
+                connections: new Map(),
+                level: noun.level || 0
             };
             // Copy connections
             for (const [level, connections] of noun.connections.entries()) {
@@ -7294,6 +7323,9 @@ class MemoryStorage extends BaseStorage {
         this.nounMetadata.clear();
         this.verbMetadata.clear();
         this.statistics = null;
+        // Clear the statistics cache
+        this.statisticsCache = null;
+        this.statisticsModified = false;
     }
     /**
      * Get information about storage usage and capacity
@@ -7527,7 +7559,8 @@ class OPFSStorage extends BaseStorage {
             return {
                 id: data.id,
                 vector: data.vector,
-                connections
+                connections,
+                level: data.level || 0
             };
         }
         catch (error) {
@@ -7558,7 +7591,8 @@ class OPFSStorage extends BaseStorage {
                         allNouns.push({
                             id: data.id,
                             vector: data.vector,
-                            connections
+                            connections,
+                            level: data.level || 0
                         });
                     }
                     catch (error) {
@@ -7609,7 +7643,8 @@ class OPFSStorage extends BaseStorage {
                             nodes.push({
                                 id: data.id,
                                 vector: data.vector,
-                                connections
+                                connections,
+                                level: data.level || 0
                             });
                         }
                     }
@@ -7979,6 +8014,9 @@ class OPFSStorage extends BaseStorage {
             await removeDirectoryContents(this.verbMetadataDir);
             // Remove all files in the index directory
             await removeDirectoryContents(this.indexDir);
+            // Clear the statistics cache
+            this.statisticsCache = null;
+            this.statisticsModified = false;
         }
         catch (error) {
             console.error('Error clearing storage:', error);
@@ -10306,7 +10344,8 @@ class S3CompatibleStorage extends BaseStorage {
                 const node = {
                     id: parsedNode.id,
                     vector: parsedNode.vector,
-                    connections
+                    connections,
+                    level: parsedNode.level || 0
                 };
                 console.log(`Successfully retrieved node ${id}:`, node);
                 return node;
@@ -11204,6 +11243,9 @@ class S3CompatibleStorage extends BaseStorage {
             await deleteObjectsWithPrefix(this.metadataPrefix);
             // Delete all objects in the index directory
             await deleteObjectsWithPrefix(this.indexPrefix);
+            // Clear the statistics cache
+            this.statisticsCache = null;
+            this.statisticsModified = false;
         }
         catch (error) {
             console.error('Failed to clear storage:', error);
@@ -12080,7 +12122,8 @@ class FileSystemStorage extends BaseStorage {
             return {
                 id: parsedNode.id,
                 vector: parsedNode.vector,
-                connections
+                connections,
+                level: parsedNode.level || 0
             };
         }
         catch (error) {
@@ -12111,7 +12154,8 @@ class FileSystemStorage extends BaseStorage {
                     allNodes.push({
                         id: parsedNode.id,
                         vector: parsedNode.vector,
-                        connections
+                        connections,
+                        level: parsedNode.level || 0
                     });
                 }
             }
@@ -12150,7 +12194,8 @@ class FileSystemStorage extends BaseStorage {
                         nouns.push({
                             id: parsedNode.id,
                             vector: parsedNode.vector,
-                            connections
+                            connections,
+                            level: parsedNode.level || 0
                         });
                     }
                 }
@@ -12415,6 +12460,9 @@ class FileSystemStorage extends BaseStorage {
         await removeDirectoryContents(this.verbMetadataDir);
         // Remove all files in the index directory
         await removeDirectoryContents(this.indexDir);
+        // Clear the statistics cache
+        this.statisticsCache = null;
+        this.statisticsModified = false;
     }
     /**
      * Get information about storage usage and capacity
@@ -16601,6 +16649,739 @@ class HealthMonitor {
     }
 }
 
+/**
+ * SearchCache - Caches search results for improved performance
+ */
+class SearchCache {
+    constructor(config = {}) {
+        this.cache = new Map();
+        // Cache statistics
+        this.hits = 0;
+        this.misses = 0;
+        this.evictions = 0;
+        this.maxAge = config.maxAge ?? 5 * 60 * 1000; // 5 minutes
+        this.maxSize = config.maxSize ?? 100;
+        this.enabled = config.enabled ?? true;
+        this.hitCountWeight = config.hitCountWeight ?? 0.3;
+    }
+    /**
+     * Generate cache key from search parameters
+     */
+    getCacheKey(query, k, options = {}) {
+        // Create a normalized key that ignores order of options
+        const normalizedOptions = Object.keys(options)
+            .sort()
+            .reduce((acc, key) => {
+            // Skip cache-related options
+            if (key === 'skipCache' || key === 'useStreaming')
+                return acc;
+            acc[key] = options[key];
+            return acc;
+        }, {});
+        return JSON.stringify({
+            query: typeof query === 'object' ? JSON.stringify(query) : query,
+            k,
+            ...normalizedOptions
+        });
+    }
+    /**
+     * Get cached results if available and not expired
+     */
+    get(key) {
+        if (!this.enabled)
+            return null;
+        const entry = this.cache.get(key);
+        if (!entry) {
+            this.misses++;
+            return null;
+        }
+        // Check if expired
+        if (Date.now() - entry.timestamp > this.maxAge) {
+            this.cache.delete(key);
+            this.misses++;
+            return null;
+        }
+        // Update hit count and statistics
+        entry.hits++;
+        this.hits++;
+        return entry.results;
+    }
+    /**
+     * Cache search results
+     */
+    set(key, results) {
+        if (!this.enabled)
+            return;
+        // Evict if cache is full
+        if (this.cache.size >= this.maxSize) {
+            this.evictOldest();
+        }
+        this.cache.set(key, {
+            results: [...results], // Deep copy to prevent mutations
+            timestamp: Date.now(),
+            hits: 0
+        });
+    }
+    /**
+     * Evict the oldest entry based on timestamp and hit count
+     */
+    evictOldest() {
+        let oldestKey = null;
+        let oldestScore = Infinity;
+        const now = Date.now();
+        for (const [key, entry] of this.cache.entries()) {
+            // Score combines age and inverse hit count
+            const age = now - entry.timestamp;
+            const hitScore = entry.hits > 0 ? 1 / entry.hits : 1;
+            const score = age + (hitScore * this.hitCountWeight * this.maxAge);
+            if (score < oldestScore) {
+                oldestScore = score;
+                oldestKey = key;
+            }
+        }
+        if (oldestKey) {
+            this.cache.delete(oldestKey);
+            this.evictions++;
+        }
+    }
+    /**
+     * Clear all cached results
+     */
+    clear() {
+        this.cache.clear();
+        this.hits = 0;
+        this.misses = 0;
+        this.evictions = 0;
+    }
+    /**
+     * Invalidate cache entries that might be affected by data changes
+     */
+    invalidate(pattern) {
+        if (!pattern) {
+            this.clear();
+            return;
+        }
+        const keysToDelete = [];
+        for (const key of this.cache.keys()) {
+            const shouldDelete = typeof pattern === 'string'
+                ? key.includes(pattern)
+                : pattern.test(key);
+            if (shouldDelete) {
+                keysToDelete.push(key);
+            }
+        }
+        keysToDelete.forEach(key => this.cache.delete(key));
+    }
+    /**
+     * Smart invalidation for real-time data updates
+     * Only clears cache if it's getting stale or if data changes significantly
+     */
+    invalidateOnDataChange(changeType) {
+        // For now, clear all caches on data changes to ensure consistency
+        // In the future, we could implement more sophisticated invalidation
+        // based on the type of change and affected data
+        this.clear();
+    }
+    /**
+     * Check if cache entries have expired and remove them
+     * This is especially important in distributed scenarios where
+     * real-time updates might be delayed or missed
+     */
+    cleanupExpiredEntries() {
+        const now = Date.now();
+        const keysToDelete = [];
+        for (const [key, entry] of this.cache.entries()) {
+            if (now - entry.timestamp > this.maxAge) {
+                keysToDelete.push(key);
+            }
+        }
+        keysToDelete.forEach(key => this.cache.delete(key));
+        return keysToDelete.length;
+    }
+    /**
+     * Get cache statistics
+     */
+    getStats() {
+        const total = this.hits + this.misses;
+        return {
+            hits: this.hits,
+            misses: this.misses,
+            evictions: this.evictions,
+            hitRate: total > 0 ? this.hits / total : 0,
+            size: this.cache.size,
+            maxSize: this.maxSize,
+            enabled: this.enabled
+        };
+    }
+    /**
+     * Enable or disable caching
+     */
+    setEnabled(enabled) {
+        Object.defineProperty(this, 'enabled', { value: enabled, writable: false });
+        if (!enabled) {
+            this.clear();
+        }
+    }
+    /**
+     * Get memory usage estimate in bytes
+     */
+    getMemoryUsage() {
+        let totalSize = 0;
+        for (const [key, entry] of this.cache.entries()) {
+            // Estimate key size
+            totalSize += key.length * 2; // UTF-16 characters
+            // Estimate entry size
+            totalSize += JSON.stringify(entry.results).length * 2;
+            totalSize += 16; // timestamp + hits (8 bytes each)
+        }
+        return totalSize;
+    }
+    /**
+     * Get current cache configuration
+     */
+    getConfig() {
+        return {
+            enabled: this.enabled,
+            maxSize: this.maxSize,
+            maxAge: this.maxAge,
+            hitCountWeight: this.hitCountWeight
+        };
+    }
+    /**
+     * Update cache configuration dynamically
+     */
+    updateConfig(newConfig) {
+        if (newConfig.enabled !== undefined) {
+            this.enabled = newConfig.enabled;
+        }
+        if (newConfig.maxSize !== undefined) {
+            this.maxSize = newConfig.maxSize;
+            // Trigger eviction if current size exceeds new limit
+            this.evictIfNeeded();
+        }
+        if (newConfig.maxAge !== undefined) {
+            this.maxAge = newConfig.maxAge;
+            // Clean up entries that are now expired with new TTL
+            this.cleanupExpiredEntries();
+        }
+        if (newConfig.hitCountWeight !== undefined) {
+            this.hitCountWeight = newConfig.hitCountWeight;
+        }
+    }
+    /**
+     * Evict entries if cache exceeds maxSize
+     */
+    evictIfNeeded() {
+        if (this.cache.size <= this.maxSize) {
+            return;
+        }
+        // Calculate eviction score for each entry (same logic as existing eviction)
+        const entries = Array.from(this.cache.entries()).map(([key, entry]) => {
+            const age = Date.now() - entry.timestamp;
+            const hitCount = entry.hits;
+            // Eviction score: lower is more likely to be evicted
+            // Combines age and hit count (weighted by hitCountWeight)
+            const ageScore = age / this.maxAge;
+            const hitScore = 1 / (hitCount + 1); // Inverse of hits (more hits = lower score)
+            const score = ageScore * (1 - this.hitCountWeight) + hitScore * this.hitCountWeight;
+            return { key, entry, score };
+        });
+        // Sort by score (lowest first - these will be evicted)
+        entries.sort((a, b) => a.score - b.score);
+        // Evict entries until we're under the limit
+        const toEvict = entries.slice(0, this.cache.size - this.maxSize);
+        toEvict.forEach(({ key }) => {
+            this.cache.delete(key);
+            this.evictions++;
+        });
+    }
+}
+
+/**
+ * Intelligent cache auto-configuration system
+ * Adapts cache settings based on environment, usage patterns, and storage type
+ */
+class CacheAutoConfigurator {
+    constructor() {
+        this.stats = {
+            totalQueries: 0,
+            repeatQueries: 0,
+            avgQueryTime: 50,
+            memoryPressure: 0,
+            storageType: 'memory',
+            isDistributed: false,
+            changeFrequency: 0,
+            readWriteRatio: 10,
+        };
+        this.configHistory = [];
+        this.lastOptimization = 0;
+    }
+    /**
+     * Auto-detect optimal cache configuration based on current conditions
+     */
+    autoDetectOptimalConfig(storageConfig, currentStats) {
+        // Update stats with current information
+        if (currentStats) {
+            this.stats = { ...this.stats, ...currentStats };
+        }
+        // Detect environment characteristics
+        this.detectEnvironment(storageConfig);
+        // Generate optimal configuration
+        const result = this.generateOptimalConfig();
+        // Store for learning
+        this.configHistory.push(result);
+        this.lastOptimization = Date.now();
+        return result;
+    }
+    /**
+     * Dynamically adjust configuration based on runtime performance
+     */
+    adaptConfiguration(currentConfig, performanceMetrics) {
+        const reasoning = [];
+        let needsUpdate = false;
+        // Check if we should update (don't over-optimize)
+        if (Date.now() - this.lastOptimization < 60000) {
+            return null; // Wait at least 1 minute between optimizations
+        }
+        // Analyze performance patterns
+        const adaptations = {};
+        // Low hit rate → adjust cache size or TTL
+        if (performanceMetrics.hitRate < 0.3) {
+            if (performanceMetrics.externalChangesDetected > 5) {
+                // Too many external changes → shorter TTL
+                adaptations.maxAge = Math.max(60000, currentConfig.maxAge * 0.7);
+                reasoning.push('Reduced cache TTL due to frequent external changes');
+                needsUpdate = true;
+            }
+            else {
+                // Expand cache size for better hit rate
+                adaptations.maxSize = Math.min(500, (currentConfig.maxSize || 100) * 1.5);
+                reasoning.push('Increased cache size due to low hit rate');
+                needsUpdate = true;
+            }
+        }
+        // High hit rate but slow responses → might need cache warming
+        if (performanceMetrics.hitRate > 0.8 && performanceMetrics.avgResponseTime > 100) {
+            reasoning.push('High hit rate but slow responses - consider cache warming');
+        }
+        // Memory pressure → reduce cache size
+        if (performanceMetrics.memoryUsage > 100 * 1024 * 1024) { // 100MB
+            adaptations.maxSize = Math.max(20, (currentConfig.maxSize || 100) * 0.7);
+            reasoning.push('Reduced cache size due to memory pressure');
+            needsUpdate = true;
+        }
+        // Recent external changes → adaptive TTL
+        if (performanceMetrics.timeSinceLastChange < 30000) { // 30 seconds
+            adaptations.maxAge = Math.max(30000, currentConfig.maxAge * 0.8);
+            reasoning.push('Shortened TTL due to recent external changes');
+            needsUpdate = true;
+        }
+        if (!needsUpdate) {
+            return null;
+        }
+        const newCacheConfig = {
+            ...currentConfig,
+            ...adaptations
+        };
+        const newRealtimeConfig = this.calculateRealtimeConfig();
+        return {
+            cacheConfig: newCacheConfig,
+            realtimeConfig: newRealtimeConfig,
+            reasoning
+        };
+    }
+    /**
+     * Get recommended configuration for specific use case
+     */
+    getRecommendedConfig(useCase) {
+        const configs = {
+            'high-consistency': {
+                cache: { maxAge: 120000, maxSize: 50 },
+                realtime: { interval: 15000, enabled: true },
+                reasoning: ['Optimized for data consistency and real-time updates']
+            },
+            'balanced': {
+                cache: { maxAge: 300000, maxSize: 100 },
+                realtime: { interval: 30000, enabled: true },
+                reasoning: ['Balanced performance and consistency']
+            },
+            'performance-first': {
+                cache: { maxAge: 600000, maxSize: 200 },
+                realtime: { interval: 60000, enabled: true },
+                reasoning: ['Optimized for maximum cache performance']
+            }
+        };
+        const config = configs[useCase];
+        return {
+            cacheConfig: {
+                enabled: true,
+                ...config.cache
+            },
+            realtimeConfig: {
+                updateIndex: true,
+                updateStatistics: true,
+                ...config.realtime
+            },
+            reasoning: config.reasoning
+        };
+    }
+    /**
+     * Learn from usage patterns and improve recommendations
+     */
+    learnFromUsage(usageData) {
+        // Update internal stats for better future recommendations
+        this.stats.totalQueries += usageData.totalQueries;
+        this.stats.repeatQueries += usageData.cacheHits;
+        this.stats.avgQueryTime = (this.stats.avgQueryTime + usageData.responseTime) / 2;
+        this.stats.changeFrequency = usageData.dataChanges / (usageData.timeWindow / 60000);
+        // Calculate read/write ratio
+        const writes = usageData.dataChanges;
+        const reads = usageData.totalQueries;
+        this.stats.readWriteRatio = reads > 0 ? reads / Math.max(writes, 1) : 10;
+    }
+    detectEnvironment(storageConfig) {
+        // Detect storage type
+        if (storageConfig?.s3Storage || storageConfig?.customS3Storage) {
+            this.stats.storageType = 's3';
+            this.stats.isDistributed = true;
+        }
+        else if (storageConfig?.forceFileSystemStorage) {
+            this.stats.storageType = 'filesystem';
+        }
+        else if (storageConfig?.forceMemoryStorage) {
+            this.stats.storageType = 'memory';
+        }
+        else {
+            // Auto-detect browser vs Node.js
+            this.stats.storageType = typeof window !== 'undefined' ? 'opfs' : 'filesystem';
+        }
+        // Detect distributed mode indicators
+        this.stats.isDistributed = this.stats.isDistributed ||
+            Boolean(storageConfig?.s3Storage || storageConfig?.customS3Storage);
+    }
+    generateOptimalConfig() {
+        const reasoning = [];
+        // Base configuration
+        let cacheConfig = {
+            enabled: true,
+            maxSize: 100,
+            maxAge: 300000, // 5 minutes
+            hitCountWeight: 0.3
+        };
+        let realtimeConfig = {
+            enabled: false,
+            interval: 60000,
+            updateIndex: true,
+            updateStatistics: true
+        };
+        // Adjust for storage type
+        if (this.stats.storageType === 's3' || this.stats.isDistributed) {
+            cacheConfig.maxAge = 180000; // 3 minutes for distributed
+            realtimeConfig.enabled = true;
+            realtimeConfig.interval = 30000; // 30 seconds
+            reasoning.push('Distributed storage detected - enabled real-time updates');
+            reasoning.push('Reduced cache TTL for distributed consistency');
+        }
+        // Adjust for read/write patterns
+        if (this.stats.readWriteRatio > 20) {
+            // Read-heavy workload
+            cacheConfig.maxSize = Math.min(300, (cacheConfig.maxSize || 100) * 2);
+            cacheConfig.maxAge = Math.min(900000, (cacheConfig.maxAge || 300000) * 1.5); // Up to 15 minutes
+            reasoning.push('Read-heavy workload detected - increased cache size and TTL');
+        }
+        else if (this.stats.readWriteRatio < 5) {
+            // Write-heavy workload
+            cacheConfig.maxSize = Math.max(50, (cacheConfig.maxSize || 100) * 0.7);
+            cacheConfig.maxAge = Math.max(60000, (cacheConfig.maxAge || 300000) * 0.6);
+            reasoning.push('Write-heavy workload detected - reduced cache size and TTL');
+        }
+        // Adjust for change frequency
+        if (this.stats.changeFrequency > 10) { // More than 10 changes per minute
+            realtimeConfig.interval = Math.max(10000, realtimeConfig.interval * 0.5);
+            cacheConfig.maxAge = Math.max(30000, (cacheConfig.maxAge || 300000) * 0.5);
+            reasoning.push('High change frequency detected - increased update frequency');
+        }
+        // Memory constraints
+        if (this.detectMemoryConstraints()) {
+            cacheConfig.maxSize = Math.max(20, (cacheConfig.maxSize || 100) * 0.6);
+            reasoning.push('Memory constraints detected - reduced cache size');
+        }
+        // Performance optimization
+        if (this.stats.avgQueryTime > 200) {
+            cacheConfig.maxSize = Math.min(500, (cacheConfig.maxSize || 100) * 1.5);
+            reasoning.push('Slow queries detected - increased cache size');
+        }
+        return {
+            cacheConfig,
+            realtimeConfig,
+            reasoning
+        };
+    }
+    calculateRealtimeConfig() {
+        return {
+            enabled: this.stats.isDistributed || this.stats.changeFrequency > 1,
+            interval: this.stats.isDistributed ? 30000 : 60000,
+            updateIndex: true,
+            updateStatistics: true
+        };
+    }
+    detectMemoryConstraints() {
+        // Simple heuristic for memory constraints
+        try {
+            if (typeof performance !== 'undefined' && 'memory' in performance) {
+                const memInfo = performance.memory;
+                return memInfo.usedJSHeapSize > memInfo.jsHeapSizeLimit * 0.8;
+            }
+        }
+        catch (e) {
+            // Ignore errors
+        }
+        // Default assumption for constrained environments
+        return false;
+    }
+    /**
+     * Get human-readable explanation of current configuration
+     */
+    getConfigExplanation(config) {
+        const lines = [
+            '🤖 Brainy Auto-Configuration:',
+            '',
+            `📊 Cache: ${config.cacheConfig.maxSize} queries, ${config.cacheConfig.maxAge / 1000}s TTL`,
+            `🔄 Updates: ${config.realtimeConfig.enabled ? `Every ${(config.realtimeConfig.interval || 30000) / 1000}s` : 'Disabled'}`,
+            '',
+            '🎯 Optimizations applied:'
+        ];
+        config.reasoning.forEach(reason => {
+            lines.push(`  • ${reason}`);
+        });
+        return lines.join('\n');
+    }
+}
+
+/**
+ * Lightweight statistics collector for Brainy
+ * Designed to have minimal performance impact even with millions of entries
+ */
+class StatisticsCollector {
+    constructor() {
+        // Content type tracking (lightweight counters)
+        this.contentTypes = new Map();
+        // Data freshness tracking (only track timestamps, not full data)
+        this.oldestTimestamp = Date.now();
+        this.newestTimestamp = Date.now();
+        this.updateTimestamps = [];
+        // Search performance tracking (rolling window)
+        this.searchMetrics = {
+            totalSearches: 0,
+            totalSearchTimeMs: 0,
+            searchTimestamps: [],
+            topSearchTerms: new Map()
+        };
+        // Verb type tracking
+        this.verbTypes = new Map();
+        // Storage size estimates (updated periodically, not on every operation)
+        this.storageSizeCache = {
+            lastUpdated: 0,
+            sizes: {
+                nouns: 0,
+                verbs: 0,
+                metadata: 0,
+                index: 0
+            }
+        };
+        this.MAX_TIMESTAMPS = 1000; // Keep last 1000 timestamps
+        this.MAX_SEARCH_TERMS = 100; // Track top 100 search terms
+        this.SIZE_UPDATE_INTERVAL = 60000; // Update sizes every minute
+    }
+    /**
+     * Track content type (very lightweight)
+     */
+    trackContentType(type) {
+        this.contentTypes.set(type, (this.contentTypes.get(type) || 0) + 1);
+    }
+    /**
+     * Track data update timestamp (lightweight)
+     */
+    trackUpdate(timestamp) {
+        const ts = timestamp || Date.now();
+        // Update oldest/newest
+        if (ts < this.oldestTimestamp)
+            this.oldestTimestamp = ts;
+        if (ts > this.newestTimestamp)
+            this.newestTimestamp = ts;
+        // Add to rolling window
+        this.updateTimestamps.push({ timestamp: ts, count: 1 });
+        // Keep window size limited
+        if (this.updateTimestamps.length > this.MAX_TIMESTAMPS) {
+            this.updateTimestamps.shift();
+        }
+    }
+    /**
+     * Track search performance (lightweight)
+     */
+    trackSearch(searchTerm, durationMs) {
+        this.searchMetrics.totalSearches++;
+        this.searchMetrics.totalSearchTimeMs += durationMs;
+        // Add to rolling window
+        this.searchMetrics.searchTimestamps.push({
+            timestamp: Date.now(),
+            count: 1
+        });
+        // Keep window size limited
+        if (this.searchMetrics.searchTimestamps.length > this.MAX_TIMESTAMPS) {
+            this.searchMetrics.searchTimestamps.shift();
+        }
+        // Track search term (limit to top N)
+        const termCount = (this.searchMetrics.topSearchTerms.get(searchTerm) || 0) + 1;
+        this.searchMetrics.topSearchTerms.set(searchTerm, termCount);
+        // Prune if too many terms
+        if (this.searchMetrics.topSearchTerms.size > this.MAX_SEARCH_TERMS * 2) {
+            this.pruneSearchTerms();
+        }
+    }
+    /**
+     * Track verb type (lightweight)
+     */
+    trackVerbType(type) {
+        this.verbTypes.set(type, (this.verbTypes.get(type) || 0) + 1);
+    }
+    /**
+     * Update storage size estimates (called periodically, not on every operation)
+     */
+    updateStorageSizes(sizes) {
+        this.storageSizeCache = {
+            lastUpdated: Date.now(),
+            sizes
+        };
+    }
+    /**
+     * Get comprehensive statistics
+     */
+    getStatistics() {
+        const now = Date.now();
+        const hourAgo = now - 3600000;
+        const dayAgo = now - 86400000;
+        const weekAgo = now - 604800000;
+        const monthAgo = now - 2592000000;
+        // Calculate data freshness
+        const updatesLastHour = this.updateTimestamps.filter(t => t.timestamp > hourAgo).length;
+        const updatesLastDay = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length;
+        // Calculate age distribution
+        const ageDistribution = {
+            last24h: 0,
+            last7d: 0,
+            last30d: 0,
+            older: 0
+        };
+        // Estimate based on update patterns (not scanning all data)
+        const totalUpdates = this.updateTimestamps.length;
+        if (totalUpdates > 0) {
+            const recentUpdates = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length;
+            const weekUpdates = this.updateTimestamps.filter(t => t.timestamp > weekAgo).length;
+            const monthUpdates = this.updateTimestamps.filter(t => t.timestamp > monthAgo).length;
+            ageDistribution.last24h = Math.round((recentUpdates / totalUpdates) * 100);
+            ageDistribution.last7d = Math.round(((weekUpdates - recentUpdates) / totalUpdates) * 100);
+            ageDistribution.last30d = Math.round(((monthUpdates - weekUpdates) / totalUpdates) * 100);
+            ageDistribution.older = 100 - ageDistribution.last24h - ageDistribution.last7d - ageDistribution.last30d;
+        }
+        // Calculate search metrics
+        const searchesLastHour = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > hourAgo).length;
+        const searchesLastDay = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > dayAgo).length;
+        const avgSearchTime = this.searchMetrics.totalSearches > 0
+            ? this.searchMetrics.totalSearchTimeMs / this.searchMetrics.totalSearches
+            : 0;
+        // Get top search terms
+        const topSearchTerms = Array.from(this.searchMetrics.topSearchTerms.entries())
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, 10)
+            .map(([term]) => term);
+        // Calculate storage metrics
+        const totalSize = Object.values(this.storageSizeCache.sizes).reduce((a, b) => a + b, 0);
+        return {
+            contentTypes: Object.fromEntries(this.contentTypes),
+            dataFreshness: {
+                oldestEntry: new Date(this.oldestTimestamp).toISOString(),
+                newestEntry: new Date(this.newestTimestamp).toISOString(),
+                updatesLastHour,
+                updatesLastDay,
+                ageDistribution
+            },
+            storageMetrics: {
+                totalSizeBytes: totalSize,
+                nounsSizeBytes: this.storageSizeCache.sizes.nouns,
+                verbsSizeBytes: this.storageSizeCache.sizes.verbs,
+                metadataSizeBytes: this.storageSizeCache.sizes.metadata,
+                indexSizeBytes: this.storageSizeCache.sizes.index
+            },
+            searchMetrics: {
+                totalSearches: this.searchMetrics.totalSearches,
+                averageSearchTimeMs: avgSearchTime,
+                searchesLastHour,
+                searchesLastDay,
+                topSearchTerms
+            },
+            verbStatistics: {
+                totalVerbs: Array.from(this.verbTypes.values()).reduce((a, b) => a + b, 0),
+                verbTypes: Object.fromEntries(this.verbTypes),
+                averageConnectionsPerVerb: 2 // Verbs connect 2 nouns
+            }
+        };
+    }
+    /**
+     * Merge statistics from storage (for distributed systems)
+     */
+    mergeFromStorage(stored) {
+        // Merge content types
+        if (stored.contentTypes) {
+            for (const [type, count] of Object.entries(stored.contentTypes)) {
+                this.contentTypes.set(type, count);
+            }
+        }
+        // Merge verb types
+        if (stored.verbStatistics?.verbTypes) {
+            for (const [type, count] of Object.entries(stored.verbStatistics.verbTypes)) {
+                this.verbTypes.set(type, count);
+            }
+        }
+        // Merge search metrics
+        if (stored.searchMetrics) {
+            this.searchMetrics.totalSearches = stored.searchMetrics.totalSearches || 0;
+            this.searchMetrics.totalSearchTimeMs = (stored.searchMetrics.averageSearchTimeMs || 0) * this.searchMetrics.totalSearches;
+        }
+        // Merge data freshness
+        if (stored.dataFreshness) {
+            this.oldestTimestamp = new Date(stored.dataFreshness.oldestEntry).getTime();
+            this.newestTimestamp = new Date(stored.dataFreshness.newestEntry).getTime();
+        }
+    }
+    /**
+     * Reset statistics (for testing)
+     */
+    reset() {
+        this.contentTypes.clear();
+        this.verbTypes.clear();
+        this.updateTimestamps = [];
+        this.searchMetrics = {
+            totalSearches: 0,
+            totalSearchTimeMs: 0,
+            searchTimestamps: [],
+            topSearchTerms: new Map()
+        };
+        this.oldestTimestamp = Date.now();
+        this.newestTimestamp = Date.now();
+    }
+    pruneSearchTerms() {
+        // Keep only top N search terms
+        const sorted = Array.from(this.searchMetrics.topSearchTerms.entries())
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, this.MAX_SEARCH_TERMS);
+        this.searchMetrics.topSearchTerms.clear();
+        for (const [term, count] of sorted) {
+            this.searchMetrics.topSearchTerms.set(term, count);
+        }
+    }
+}
+
 /**
  * BrainyData
  * Main class that provides the vector database functionality
@@ -16661,6 +17442,8 @@ class BrainyData {
         this.operationalMode = null;
         this.domainDetector = null;
         this.healthMonitor = null;
+        // Statistics collector
+        this.statisticsCollector = new StatisticsCollector();
         // Set dimensions to fixed value of 512 (Universal Sentence Encoder dimension)
         this._dimensions = 512;
         // Set distance function
@@ -16758,6 +17541,26 @@ class BrainyData {
                 this.distributedConfig = config.distributed;
             }
         }
+        // Initialize cache auto-configurator first
+        this.cacheAutoConfigurator = new CacheAutoConfigurator();
+        // Auto-detect optimal cache configuration if not explicitly provided
+        let finalSearchCacheConfig = config.searchCache;
+        if (!config.searchCache || Object.keys(config.searchCache).length === 0) {
+            const autoConfig = this.cacheAutoConfigurator.autoDetectOptimalConfig(config.storage);
+            finalSearchCacheConfig = autoConfig.cacheConfig;
+            // Apply auto-detected real-time update configuration if not explicitly set
+            if (!config.realtimeUpdates && autoConfig.realtimeConfig.enabled) {
+                this.realtimeUpdateConfig = {
+                    ...this.realtimeUpdateConfig,
+                    ...autoConfig.realtimeConfig
+                };
+            }
+            if (this.loggingConfig?.verbose) {
+                console.log(this.cacheAutoConfigurator.getConfigExplanation(autoConfig));
+            }
+        }
+        // Initialize search cache with final configuration
+        this.searchCache = new SearchCache(finalSearchCacheConfig);
     }
     /**
      * Check if the database is in read-only mode and throw an error if it is
@@ -16897,6 +17700,18 @@ class BrainyData {
                     await this.applyChangesFromFullScan();
                 }
             }
+            // Cleanup expired cache entries (defensive mechanism for distributed scenarios)
+            const expiredCount = this.searchCache.cleanupExpiredEntries();
+            if (expiredCount > 0 && this.loggingConfig?.verbose) {
+                console.log(`Cleaned up ${expiredCount} expired cache entries`);
+            }
+            // Adapt cache configuration based on performance (every few updates)
+            // Only adapt every 5th update to avoid over-optimization
+            const updateCount = Math.floor((Date.now() - (this.lastUpdateTime || 0)) /
+                this.realtimeUpdateConfig.interval);
+            if (updateCount % 5 === 0) {
+                this.adaptCacheConfiguration();
+            }
             // Update the last update time
             this.lastUpdateTime = Date.now();
             if (this.loggingConfig?.verbose) {
@@ -16971,6 +17786,13 @@ class BrainyData {
                 (addedCount > 0 || updatedCount > 0 || deletedCount > 0)) {
                 console.log(`Real-time update: Added ${addedCount}, updated ${updatedCount}, deleted ${deletedCount} nouns using change log`);
             }
+            // Invalidate search cache if any external changes were detected
+            if (addedCount > 0 || updatedCount > 0 || deletedCount > 0) {
+                this.searchCache.invalidateOnDataChange('update');
+                if (this.loggingConfig?.verbose) {
+                    console.log('Search cache invalidated due to external data changes');
+                }
+            }
             // Update the last known noun count
             this.lastKnownNounCount = await this.getNounCount();
         }
@@ -17014,6 +17836,13 @@ class BrainyData {
                 }
                 // Update the last known noun count
                 this.lastKnownNounCount = currentCount;
+                // Invalidate search cache if new nouns were detected
+                if (newNouns.length > 0) {
+                    this.searchCache.invalidateOnDataChange('add');
+                    if (this.loggingConfig?.verbose) {
+                        console.log('Search cache invalidated due to external data changes');
+                    }
+                }
                 if (this.loggingConfig?.verbose && newNouns.length > 0) {
                     console.log(`Real-time update: Added ${newNouns.length} new nouns to index using full scan`);
                 }
@@ -17197,6 +18026,16 @@ class BrainyData {
                     // Continue initialization even if remote connection fails
                 }
             }
+            // Initialize statistics collector with existing data
+            try {
+                const existingStats = await this.storage.getStatistics();
+                if (existingStats) {
+                    this.statisticsCollector.mergeFromStorage(existingStats);
+                }
+            }
+            catch (e) {
+                // Ignore errors loading existing statistics
+            }
             this.isInitialized = true;
             this.isInitializing = false;
             // Start real-time updates if enabled
@@ -17408,13 +18247,15 @@ class BrainyData {
                 try {
                     if (this.writeOnly) {
                         // In write-only mode, check storage directly
-                        existingNoun = await this.storage.getNoun(options.id) ?? undefined;
+                        existingNoun =
+                            (await this.storage.getNoun(options.id)) ?? undefined;
                     }
                     else {
                         // In normal mode, check index first, then storage
                         existingNoun = this.index.getNouns().get(options.id);
                         if (!existingNoun) {
-                            existingNoun = await this.storage.getNoun(options.id) ?? undefined;
+                            existingNoun =
+                                (await this.storage.getNoun(options.id)) ?? undefined;
                         }
                     }
                     if (existingNoun) {
@@ -17449,6 +18290,7 @@ class BrainyData {
                     id,
                     vector,
                     connections: new Map(),
+                    level: 0, // Default level for new nodes
                     metadata: undefined // Will be set separately
                 };
             }
@@ -17522,17 +18364,24 @@ class BrainyData {
                                 // Domain already specified, keep it
                                 const domainInfo = this.domainDetector.detectDomain(metadataToSave);
                                 if (domainInfo.domainMetadata) {
-                                    metadataToSave.domainMetadata = domainInfo.domainMetadata;
+                                    ;
+                                    metadataToSave.domainMetadata =
+                                        domainInfo.domainMetadata;
                                 }
                             }
                             else {
                                 // Try to detect domain from the data
-                                const dataToAnalyze = Array.isArray(vectorOrData) ? metadata : vectorOrData;
+                                const dataToAnalyze = Array.isArray(vectorOrData)
+                                    ? metadata
+                                    : vectorOrData;
                                 const domainInfo = this.domainDetector.detectDomain(dataToAnalyze);
                                 if (domainInfo.domain) {
+                                    ;
                                     metadataToSave.domain = domainInfo.domain;
                                     if (domainInfo.domainMetadata) {
-                                        metadataToSave.domainMetadata = domainInfo.domainMetadata;
+                                        ;
+                                        metadataToSave.domainMetadata =
+                                            domainInfo.domainMetadata;
                                     }
                                 }
                             }
@@ -17547,10 +18396,19 @@ class BrainyData {
                     // Track metadata statistics
                     const metadataService = this.getServiceName(options);
                     await this.storage.incrementStatistic('metadata', metadataService);
+                    // Track content type if it's a GraphNoun
+                    if (metadataToSave &&
+                        typeof metadataToSave === 'object' &&
+                        'noun' in metadataToSave) {
+                        this.statisticsCollector.trackContentType(metadataToSave.noun);
+                    }
+                    // Track update timestamp
+                    this.statisticsCollector.trackUpdate();
                 }
             }
-            // Update HNSW index size (excluding verbs)
-            await this.storage.updateHnswIndexSize(await this.getNounCount());
+            // Update HNSW index size with actual index size
+            const indexSize = this.index.size();
+            await this.storage.updateHnswIndexSize(indexSize);
             // Update health metrics if in distributed mode
             if (this.healthMonitor) {
                 const vectorCount = await this.getNounCount();
@@ -17565,6 +18423,8 @@ class BrainyData {
                     console.warn(`Failed to add to remote server: ${remoteError}. Continuing with local add.`);
                 }
             }
+            // Invalidate search cache since data has changed
+            this.searchCache.invalidateOnDataChange('add');
             return id;
         }
         catch (error) {
@@ -17822,11 +18682,16 @@ class BrainyData {
                         console.log(`Lazy loading mode: Added ${limitedNouns.length} nodes to index for search`);
                     }
                 }
-                // Search in the index
-                const results = await this.index.search(queryVector, k);
+                // When using offset, we need to fetch more results and then slice
+                const offset = options.offset || 0;
+                const totalNeeded = k + offset;
+                // Search in the index for totalNeeded results
+                const results = await this.index.search(queryVector, totalNeeded);
+                // Skip the offset number of results
+                const paginatedResults = results.slice(offset, offset + k);
                 // Get metadata for each result
                 const searchResults = [];
-                for (const [id, score] of results) {
+                for (const [id, score] of paginatedResults) {
                     const noun = this.index.getNouns().get(id);
                     if (!noun) {
                         continue;
@@ -17867,8 +18732,9 @@ class BrainyData {
                 }
                 // Sort by distance (ascending)
                 results.sort((a, b) => a[1] - b[1]);
-                // Take top k results
-                const topResults = results.slice(0, k);
+                // Apply offset and take k results
+                const offset = options.offset || 0;
+                const topResults = results.slice(offset, offset + k);
                 // Get metadata for each result
                 const searchResults = [];
                 for (const [id, score] of topResults) {
@@ -17962,12 +18828,29 @@ class BrainyData {
         }
         // Default behavior (backward compatible): search locally
         try {
+            // Check cache first (transparent to user)
+            const cacheKey = this.searchCache.getCacheKey(queryVectorOrData, k, options);
+            const cachedResults = this.searchCache.get(cacheKey);
+            if (cachedResults) {
+                // Track cache hit in health monitor
+                if (this.healthMonitor) {
+                    const latency = Date.now() - startTime;
+                    this.healthMonitor.recordRequest(latency, false);
+                    this.healthMonitor.recordCacheAccess(true);
+                }
+                return cachedResults;
+            }
+            // Cache miss - perform actual search
             const results = await this.searchLocal(queryVectorOrData, k, options);
+            // Cache results for future queries (unless explicitly disabled)
+            if (!options.skipCache) {
+                this.searchCache.set(cacheKey, results);
+            }
             // Track successful search in health monitor
             if (this.healthMonitor) {
                 const latency = Date.now() - startTime;
                 this.healthMonitor.recordRequest(latency, false);
-                this.healthMonitor.recordCacheAccess(results.length > 0);
+                this.healthMonitor.recordCacheAccess(false);
             }
             return results;
         }
@@ -17980,6 +18863,59 @@ class BrainyData {
             throw error;
         }
     }
+    /**
+     * Search with cursor-based pagination for better performance on large datasets
+     * @param queryVectorOrData Query vector or data to search for
+     * @param k Number of results to return
+     * @param options Additional options including cursor for pagination
+     * @returns Paginated search results with cursor for next page
+     */
+    async searchWithCursor(queryVectorOrData, k = 10, options = {}) {
+        // For cursor-based search, we need to fetch more results and filter
+        const searchK = options.cursor ? k + 20 : k; // Get extra results for filtering
+        // Perform regular search
+        const allResults = await this.search(queryVectorOrData, searchK, {
+            ...options,
+            skipCache: options.skipCache
+        });
+        let results = allResults;
+        let startIndex = 0;
+        // If cursor provided, find starting position
+        if (options.cursor) {
+            startIndex = allResults.findIndex((r) => r.id === options.cursor.lastId &&
+                Math.abs(r.score - options.cursor.lastScore) < 0.0001);
+            if (startIndex >= 0) {
+                startIndex += 1; // Start after the cursor position
+                results = allResults.slice(startIndex, startIndex + k);
+            }
+            else {
+                // Cursor not found, might be stale - return from beginning
+                results = allResults.slice(0, k);
+                startIndex = 0;
+            }
+        }
+        else {
+            results = allResults.slice(0, k);
+        }
+        // Create cursor for next page
+        let nextCursor;
+        const hasMoreResults = startIndex + results.length < allResults.length ||
+            allResults.length >= searchK;
+        if (results.length > 0 && hasMoreResults) {
+            const lastResult = results[results.length - 1];
+            nextCursor = {
+                lastId: lastResult.id,
+                lastScore: lastResult.score,
+                position: startIndex + results.length
+            };
+        }
+        return {
+            results,
+            cursor: nextCursor,
+            hasMore: !!nextCursor,
+            totalEstimate: allResults.length > searchK ? undefined : allResults.length
+        };
+    }
     /**
      * Search the local database for similar vectors
      * @param queryVectorOrData Query vector or data to search for
@@ -18035,18 +18971,20 @@ class BrainyData {
         if (options.nounTypes && options.nounTypes.length > 0) {
             searchResults = await this.searchByNounTypes(queryToUse, k, options.nounTypes, {
                 forceEmbed: options.forceEmbed,
-                service: options.service
+                service: options.service,
+                offset: options.offset
             });
         }
         else {
             // Otherwise, search all GraphNouns
             searchResults = await this.searchByNounTypes(queryToUse, k, null, {
                 forceEmbed: options.forceEmbed,
-                service: options.service
+                service: options.service,
+                offset: options.offset
             });
         }
         // Filter out placeholder nouns from search results
-        searchResults = searchResults.filter(result => {
+        searchResults = searchResults.filter((result) => {
             if (result.metadata && typeof result.metadata === 'object') {
                 const metadata = result.metadata;
                 // Exclude placeholder nouns from search results
@@ -18154,7 +19092,7 @@ class BrainyData {
             // In write-only mode, query storage directly since index is not loaded
             if (this.writeOnly) {
                 try {
-                    noun = await this.storage.getNoun(id) ?? undefined;
+                    noun = (await this.storage.getNoun(id)) ?? undefined;
                 }
                 catch (storageError) {
                     // If storage lookup fails, return null (noun doesn't exist)
@@ -18167,7 +19105,7 @@ class BrainyData {
                 // If not found in index, fallback to storage (for race conditions)
                 if (!noun && this.storage) {
                     try {
-                        noun = await this.storage.getNoun(id) ?? undefined;
+                        noun = (await this.storage.getNoun(id)) ?? undefined;
                     }
                     catch (storageError) {
                         // Storage lookup failed, noun doesn't exist
@@ -18376,6 +19314,8 @@ class BrainyData {
             catch (error) {
                 // Ignore
             }
+            // Invalidate search cache since data has changed
+            this.searchCache.invalidateOnDataChange('delete');
             return true;
         }
         catch (error) {
@@ -18459,6 +19399,8 @@ class BrainyData {
             // Track metadata statistics
             const service = this.getServiceName(options);
             await this.storage.incrementStatistic('metadata', service);
+            // Invalidate search cache since metadata has changed
+            this.searchCache.invalidateOnDataChange('update');
             return true;
         }
         catch (error) {
@@ -18555,6 +19497,7 @@ class BrainyData {
                     id: sourceId,
                     vector: sourcePlaceholderVector,
                     connections: new Map(),
+                    level: 0,
                     metadata: sourceMetadata
                 };
                 // Create placeholder target noun
@@ -18575,6 +19518,7 @@ class BrainyData {
                     id: targetId,
                     vector: targetPlaceholderVector,
                     connections: new Map(),
+                    level: 0,
                     metadata: targetMetadata
                 };
                 // Save placeholder nouns to storage (but skip indexing for speed)
@@ -18811,8 +19755,13 @@ class BrainyData {
             // Track verb statistics
             const serviceForStats = this.getServiceName(options);
             await this.storage.incrementStatistic('verb', serviceForStats);
-            // Update HNSW index size (excluding verbs)
-            await this.storage.updateHnswIndexSize(await this.getNounCount());
+            // Track verb type
+            this.statisticsCollector.trackVerbType(verbMetadata.verb);
+            // Update HNSW index size with actual index size
+            const indexSize = this.index.size();
+            await this.storage.updateHnswIndexSize(indexSize);
+            // Invalidate search cache since verb data has changed
+            this.searchCache.invalidateOnDataChange('add');
             return id;
         }
         catch (error) {
@@ -19035,6 +19984,10 @@ class BrainyData {
             await this.index.clear();
             // Clear storage
             await this.storage.clear();
+            // Reset statistics collector
+            this.statisticsCollector = new StatisticsCollector();
+            // Clear search cache since all data has been removed
+            this.searchCache.invalidateOnDataChange('delete');
         }
         catch (error) {
             console.error('Failed to clear vector database:', error);
@@ -19047,6 +20000,67 @@ class BrainyData {
     size() {
         return this.index.size();
     }
+    /**
+     * Get search cache statistics for performance monitoring
+     * @returns Cache statistics including hit rate and memory usage
+     */
+    getCacheStats() {
+        return {
+            search: this.searchCache.getStats(),
+            searchMemoryUsage: this.searchCache.getMemoryUsage()
+        };
+    }
+    /**
+     * Clear search cache manually (useful for testing or memory management)
+     */
+    clearCache() {
+        this.searchCache.clear();
+    }
+    /**
+     * Adapt cache configuration based on current performance metrics
+     * This method analyzes usage patterns and automatically optimizes cache settings
+     * @private
+     */
+    adaptCacheConfiguration() {
+        const stats = this.searchCache.getStats();
+        const memoryUsage = this.searchCache.getMemoryUsage();
+        const currentConfig = this.searchCache.getConfig();
+        // Prepare performance metrics for adaptation
+        const performanceMetrics = {
+            hitRate: stats.hitRate,
+            avgResponseTime: 50, // Would be measured in real implementation
+            memoryUsage: memoryUsage,
+            externalChangesDetected: 0, // Would be tracked from real-time updates
+            timeSinceLastChange: Date.now() - this.lastUpdateTime
+        };
+        // Try to adapt configuration
+        const newConfig = this.cacheAutoConfigurator.adaptConfiguration(currentConfig, performanceMetrics);
+        if (newConfig) {
+            // Apply new cache configuration
+            this.searchCache.updateConfig(newConfig.cacheConfig);
+            // Apply new real-time update configuration if needed
+            if (newConfig.realtimeConfig.enabled !==
+                this.realtimeUpdateConfig.enabled ||
+                newConfig.realtimeConfig.interval !== this.realtimeUpdateConfig.interval) {
+                const wasEnabled = this.realtimeUpdateConfig.enabled;
+                this.realtimeUpdateConfig = {
+                    ...this.realtimeUpdateConfig,
+                    ...newConfig.realtimeConfig
+                };
+                // Restart real-time updates with new configuration
+                if (wasEnabled) {
+                    this.stopRealtimeUpdates();
+                }
+                if (this.realtimeUpdateConfig.enabled && this.isInitialized) {
+                    this.startRealtimeUpdates();
+                }
+            }
+            if (this.loggingConfig?.verbose) {
+                console.log('🔧 Auto-adapted cache configuration:');
+                console.log(this.cacheAutoConfigurator.getConfigExplanation(newConfig));
+            }
+        }
+    }
     /**
      * Get the number of nouns in the database (excluding verbs)
      * This is used for statistics reporting to match the expected behavior in tests
@@ -19116,6 +20130,41 @@ class BrainyData {
         // Call the flushStatisticsToStorage method on the storage adapter
         await this.storage.flushStatisticsToStorage();
     }
+    /**
+     * Update storage sizes if needed (called periodically for performance)
+     */
+    async updateStorageSizesIfNeeded() {
+        // Only update every minute to avoid performance impact
+        const now = Date.now();
+        const lastUpdate = this.lastStorageSizeUpdate || 0;
+        if (now - lastUpdate < 60000) {
+            return; // Skip if updated recently
+        }
+        this.lastStorageSizeUpdate = now;
+        try {
+            // Estimate sizes based on counts and average sizes
+            const stats = await this.storage.getStatistics();
+            if (stats) {
+                const avgNounSize = 2048; // ~2KB per noun (vector + metadata)
+                const avgVerbSize = 512; // ~0.5KB per verb
+                const avgMetadataSize = 256; // ~0.25KB per metadata entry
+                const avgIndexEntrySize = 128; // ~128 bytes per index entry
+                // Calculate total counts
+                const totalNouns = Object.values(stats.nounCount).reduce((a, b) => a + b, 0);
+                const totalVerbs = Object.values(stats.verbCount).reduce((a, b) => a + b, 0);
+                const totalMetadata = Object.values(stats.metadataCount).reduce((a, b) => a + b, 0);
+                this.statisticsCollector.updateStorageSizes({
+                    nouns: totalNouns * avgNounSize,
+                    verbs: totalVerbs * avgVerbSize,
+                    metadata: totalMetadata * avgMetadataSize,
+                    index: stats.hnswIndexSize * avgIndexEntrySize
+                });
+            }
+        }
+        catch (error) {
+            // Ignore errors in size calculation
+        }
+    }
     /**
      * Get statistics about the current state of the database
      * @param options Additional options for retrieving statistics
@@ -19190,6 +20239,40 @@ class BrainyData {
                     relate: result.verbCount,
                     total: result.nounCount + result.verbCount + result.metadataCount
                 };
+                // Add extended statistics if requested
+                if (true) {
+                    // Always include for now
+                    // Add index health metrics
+                    try {
+                        const indexHealth = this.index.getIndexHealth();
+                        result.indexHealth = indexHealth;
+                    }
+                    catch (e) {
+                        // Index health not available
+                    }
+                    // Add cache metrics
+                    try {
+                        const cacheStats = this.searchCache.getStats();
+                        result.cacheMetrics = cacheStats;
+                    }
+                    catch (e) {
+                        // Cache stats not available
+                    }
+                    // Add memory usage
+                    if (typeof process !== 'undefined' && process.memoryUsage) {
+                        ;
+                        result.memoryUsage = process.memoryUsage().heapUsed;
+                    }
+                    // Add last updated timestamp
+                    ;
+                    result.lastUpdated =
+                        stats.lastUpdated || new Date().toISOString();
+                    // Add enhanced statistics from collector
+                    const collectorStats = this.statisticsCollector.getStatistics();
+                    Object.assign(result, collectorStats);
+                    // Update storage sizes if needed (only periodically for performance)
+                    await this.updateStorageSizesIfNeeded();
+                }
                 return result;
             }
             // If statistics are not available, return zeros instead of calculating on-demand
@@ -19549,15 +20632,20 @@ class BrainyData {
         await this.ensureInitialized();
         // Check if database is in write-only mode
         this.checkWriteOnly();
+        const searchStartTime = Date.now();
         try {
             // Embed the query text
             const queryVector = await this.embed(query);
             // Search using the embedded vector
-            return await this.search(queryVector, k, {
+            const results = await this.search(queryVector, k, {
                 nounTypes: options.nounTypes,
                 includeVerbs: options.includeVerbs,
                 searchMode: options.searchMode
             });
+            // Track search performance
+            const duration = Date.now() - searchStartTime;
+            this.statisticsCollector.trackSearch(query, duration);
+            return results;
         }
         catch (error) {
             console.error('Failed to search with text query:', error);
@@ -19593,12 +20681,17 @@ class BrainyData {
             if (!this.serverSearchConduit || !this.serverConnection) {
                 throw new Error('Server search conduit or connection is not initialized');
             }
-            // Search the remote server
-            const searchResult = await this.serverSearchConduit.searchServer(this.serverConnection.connectionId, query, k);
+            // When using offset, fetch more results and slice
+            const offset = options.offset || 0;
+            const totalNeeded = k + offset;
+            // Search the remote server for totalNeeded results
+            const searchResult = await this.serverSearchConduit.searchServer(this.serverConnection.connectionId, query, totalNeeded);
             if (!searchResult.success) {
                 throw new Error(`Remote search failed: ${searchResult.error}`);
             }
-            return searchResult.data;
+            // Apply offset to remote results
+            const allResults = searchResult.data;
+            return allResults.slice(offset, offset + k);
         }
         catch (error) {
             console.error('Failed to search remote server:', error);