@soulcraft/brainy 3.34.0 → 3.36.0

package/dist/hnsw/hnswIndex.js

@@ -3,6 +3,8 @@
  * Based on the paper: "Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs"
  */
 import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js';
+import { getGlobalCache } from '../utils/unifiedCache.js';
+import { prodLog } from '../utils/logger.js';
 // Default HNSW parameters
 const DEFAULT_CONFIG = {
     M: 16, // Max number of connections per noun
@@ -11,6 +13,7 @@ const DEFAULT_CONFIG = {
     ml: 16 // Max level
 };
 export class HNSWIndex {
+    // Always-adaptive caching (v3.36.0+) - no "mode" concept, system adapts automatically
     constructor(config = {}, distanceFunction = euclideanDistance, options = {}) {
         this.nouns = new Map();
         this.entryPointId = null;
@@ -20,12 +23,16 @@ export class HNSWIndex {
         this.MAX_TRACKED_LEVELS = 10; // Only track top levels for memory efficiency
         this.dimension = null;
         this.useParallelization = true; // Whether to use parallelization for performance-critical operations
+        this.storage = null; // Storage adapter for HNSW persistence (v3.35.0+)
         this.config = { ...DEFAULT_CONFIG, ...config };
         this.distanceFunction = distanceFunction;
         this.useParallelization =
             options.useParallelization !== undefined
                 ? options.useParallelization
                 : true;
+        this.storage = options.storage || null;
+        // Use SAME UnifiedCache as Graph and Metadata for fair memory competition
+        this.unifiedCache = getGlobalCache();
     }
     /**
      * Set whether to use parallelization for performance-critical operations
@@ -136,7 +143,8 @@ export class HNSWIndex {
             return id;
         }
         let currObj = entryPoint;
-        let currDist = this.distanceFunction(vector, entryPoint.vector);
+        // Calculate distance to entry point (handles lazy loading + sync fast path)
+        let currDist = await Promise.resolve(this.distanceSafe(vector, entryPoint));
         // Traverse the graph from top to bottom to find the closest noun
         for (let level = this.maxLevel; level > nounLevel; level--) {
             let changed = true;
@@ -144,13 +152,17 @@ export class HNSWIndex {
                 changed = false;
                 // Check all neighbors at current level
                 const connections = currObj.connections.get(level) || new Set();
+                // OPTIMIZATION: Preload neighbor vectors for parallel loading
+                if (connections.size > 0) {
+                    await this.preloadVectors(Array.from(connections));
+                }
                 for (const neighborId of connections) {
                     const neighbor = this.nouns.get(neighborId);
                     if (!neighbor) {
                         // Skip neighbors that don't exist (expected during rapid additions/deletions)
                         continue;
                     }
-                    const distToNeighbor = this.distanceFunction(vector, neighbor.vector);
+                    const distToNeighbor = await Promise.resolve(this.distanceSafe(vector, neighbor));
                     if (distToNeighbor < currDist) {
                         currDist = distToNeighbor;
                         currObj = neighbor;
@@ -180,7 +192,20 @@ export class HNSWIndex {
                 neighbor.connections.get(level).add(id);
                 // Ensure neighbor doesn't have too many connections
                 if (neighbor.connections.get(level).size > this.config.M) {
-                    this.pruneConnections(neighbor, level);
+                    await this.pruneConnections(neighbor, level);
+                }
+                // Persist updated neighbor HNSW data (v3.35.0+)
+                if (this.storage) {
+                    const neighborConnectionsObj = {};
+                    for (const [lvl, nounIds] of neighbor.connections.entries()) {
+                        neighborConnectionsObj[lvl.toString()] = Array.from(nounIds);
+                    }
+                    this.storage.saveHNSWData(neighborId, {
+                        level: neighbor.level,
+                        connections: neighborConnectionsObj
+                    }).catch((error) => {
+                        console.error(`Failed to persist neighbor HNSW data for ${neighborId}:`, error);
+                    });
                 }
             }
             // Update entry point for the next level
@@ -213,6 +238,27 @@ export class HNSWIndex {
             }
             this.highLevelNodes.get(nounLevel).add(id);
         }
+        // Persist HNSW graph data to storage (v3.35.0+)
+        if (this.storage) {
+            // Convert connections Map to serializable format
+            const connectionsObj = {};
+            for (const [level, nounIds] of noun.connections.entries()) {
+                connectionsObj[level.toString()] = Array.from(nounIds);
+            }
+            await this.storage.saveHNSWData(id, {
+                level: nounLevel,
+                connections: connectionsObj
+            }).catch((error) => {
+                console.error(`Failed to persist HNSW data for ${id}:`, error);
+            });
+            // Persist system data (entry point and max level)
+            await this.storage.saveHNSWSystem({
+                entryPointId: this.entryPointId,
+                maxLevel: this.maxLevel
+            }).catch((error) => {
+                console.error('Failed to persist HNSW system data:', error);
+            });
+        }
         return id;
     }
     /**
@@ -240,7 +286,9 @@ export class HNSWIndex {
             return [];
         }
         let currObj = entryPoint;
-        let currDist = this.distanceFunction(queryVector, currObj.vector);
+        // OPTIMIZATION: Preload entry point vector
+        await this.preloadVectors([entryPoint.id]);
+        let currDist = await Promise.resolve(this.distanceSafe(queryVector, currObj));
         // Traverse the graph from top to bottom to find the closest noun
         for (let level = this.maxLevel; level > 0; level--) {
             let changed = true;
@@ -248,6 +296,10 @@ export class HNSWIndex {
                 changed = false;
                 // Check all neighbors at current level
                 const connections = currObj.connections.get(level) || new Set();
+                // OPTIMIZATION: Preload all neighbor vectors in parallel before distance calculations
+                if (connections.size > 0) {
+                    await this.preloadVectors(Array.from(connections));
+                }
                 // If we have enough connections, use parallel distance calculation
                 if (this.useParallelization && connections.size >= 10) {
                     // Prepare vectors for parallel calculation
@@ -256,7 +308,8 @@ export class HNSWIndex {
                         const neighbor = this.nouns.get(neighborId);
                         if (!neighbor)
                             continue;
-                        vectors.push({ id: neighborId, vector: neighbor.vector });
+                        const neighborVector = await this.getVectorSafe(neighbor);
+                        vectors.push({ id: neighborId, vector: neighborVector });
                     }
                     // Calculate distances in parallel
                     const distances = await this.calculateDistancesInParallel(queryVector, vectors);
@@ -280,7 +333,7 @@ export class HNSWIndex {
                             // Skip neighbors that don't exist (expected during rapid additions/deletions)
                             continue;
                         }
-                        const distToNeighbor = this.distanceFunction(queryVector, neighbor.vector);
+                        const distToNeighbor = await Promise.resolve(this.distanceSafe(queryVector, neighbor));
                         if (distToNeighbor < currDist) {
                             currDist = distToNeighbor;
                             currObj = neighbor;
@@ -300,7 +353,7 @@ export class HNSWIndex {
     /**
      * Remove an item from the index
      */
-    removeItem(id) {
+    async removeItem(id) {
         if (!this.nouns.has(id)) {
             return false;
         }
@@ -316,7 +369,7 @@ export class HNSWIndex {
                 if (neighbor.connections.has(level)) {
                     neighbor.connections.get(level).delete(id);
                     // Prune connections after removing this noun to ensure consistency
-                    this.pruneConnections(neighbor, level);
+                    await this.pruneConnections(neighbor, level);
                 }
             }
         }
@@ -328,7 +381,7 @@ export class HNSWIndex {
                 if (connections.has(id)) {
                     connections.delete(id);
                     // Prune connections after removing this reference
-                    this.pruneConnections(otherNoun, level);
+                    await this.pruneConnections(otherNoun, level);
                 }
             }
         }
@@ -437,6 +490,120 @@ export class HNSWIndex {
     getConfig() {
         return { ...this.config };
     }
+    /**
+     * Get vector safely (always uses adaptive caching via UnifiedCache)
+     *
+     * Production-grade adaptive caching (v3.36.0+):
+     * - Vector already loaded: Returns immediately (O(1))
+     * - Vector in cache: Loads from UnifiedCache (O(1) hash lookup)
+     * - Vector on disk: Loads from storage → UnifiedCache (O(disk))
+     * - Cost-aware caching: UnifiedCache manages memory competition
+     *
+     * @param noun The HNSW noun (may have empty vector if not yet loaded)
+     * @returns Promise<Vector> The vector (loaded on-demand if needed)
+     */
+    async getVectorSafe(noun) {
+        // Vector already in memory
+        if (noun.vector.length > 0) {
+            return noun.vector;
+        }
+        // Load from UnifiedCache with storage fallback
+        const cacheKey = `hnsw:vector:${noun.id}`;
+        const vector = await this.unifiedCache.get(cacheKey, async () => {
+            // Cache miss - load from storage
+            if (!this.storage) {
+                throw new Error('Storage not available for vector loading');
+            }
+            const loaded = await this.storage.getNounVector(noun.id);
+            if (!loaded) {
+                throw new Error(`Vector not found for noun ${noun.id}`);
+            }
+            // Add to UnifiedCache with cost-aware eviction
+            // This competes fairly with Graph and Metadata indexes
+            this.unifiedCache.set(cacheKey, loaded, 'hnsw', // Type for fairness monitoring
+            loaded.length * 4, // Size in bytes (float32)
+            50 // Rebuild cost in ms (moderate priority)
+            );
+            return loaded;
+        });
+        return vector;
+    }
+    /**
+     * Get vector synchronously if available in memory (v3.36.0+)
+     *
+     * Sync fast path optimization:
+     * - Vector in memory: Returns immediately (zero overhead)
+     * - Vector in cache: Returns from UnifiedCache synchronously
+     * - Returns null if vector not available (caller must handle async path)
+     *
+     * Use for sync fast path in distance calculations - eliminates async overhead
+     * when vectors are already cached.
+     *
+     * @param noun The HNSW noun
+     * @returns Vector | null - vector if in memory/cache, null if needs async load
+     */
+    getVectorSync(noun) {
+        // Vector already in memory
+        if (noun.vector.length > 0) {
+            return noun.vector;
+        }
+        // Try sync cache lookup
+        const cacheKey = `hnsw:vector:${noun.id}`;
+        const vector = this.unifiedCache.getSync(cacheKey);
+        return vector || null;
+    }
+    /**
+     * Preload multiple vectors in parallel via UnifiedCache
+     *
+     * Optimization for search operations:
+     * - Loads all candidate vectors before distance calculations
+     * - Reduces serial disk I/O (parallel loads are faster)
+     * - Uses UnifiedCache's request coalescing to prevent stampede
+     * - Always active (no "mode" check) for optimal performance
+     *
+     * @param nodeIds Array of node IDs to preload
+     */
+    async preloadVectors(nodeIds) {
+        if (nodeIds.length === 0)
+            return;
+        // Use UnifiedCache's request coalescing to prevent duplicate loads
+        const promises = nodeIds.map(async (id) => {
+            const cacheKey = `hnsw:vector:${id}`;
+            return this.unifiedCache.get(cacheKey, async () => {
+                if (!this.storage)
+                    return null;
+                const vector = await this.storage.getNounVector(id);
+                if (vector) {
+                    this.unifiedCache.set(cacheKey, vector, 'hnsw', vector.length * 4, 50);
+                }
+                return vector;
+            });
+        });
+        await Promise.all(promises);
+    }
+    /**
+     * Calculate distance with sync fast path (v3.36.0+)
+     *
+     * Eliminates async overhead when vectors are in memory:
+     * - Sync path: Vector in memory → returns number (zero overhead)
+     * - Async path: Vector needs loading → returns Promise<number>
+     *
+     * Callers must handle union type: `const dist = await Promise.resolve(distance)`
+     *
+     * @param queryVector The query vector
+     * @param noun The target noun (may have empty vector in lazy mode)
+     * @returns number | Promise<number> - sync when cached, async when needs load
+     */
+    distanceSafe(queryVector, noun) {
+        // Try sync fast path
+        const nounVector = this.getVectorSync(noun);
+        if (nounVector !== null) {
+            // SYNC PATH: Vector in memory - zero async overhead
+            return this.distanceFunction(queryVector, nounVector);
+        }
+        // ASYNC PATH: Vector needs loading from storage
+        return this.getVectorSafe(noun).then(loadedVector => this.distanceFunction(queryVector, loadedVector));
+    }
     /**
      * Get all nodes at a specific level for clustering
      * This enables O(n) clustering using HNSW's natural hierarchy
@@ -451,6 +618,126 @@ export class HNSWIndex {
         }
         return nodesAtLevel;
     }
+    /**
+     * Rebuild HNSW index from persisted graph data (v3.35.0+)
+     *
+     * This is a production-grade O(N) rebuild that restores the pre-computed graph structure
+     * from storage. Much faster than re-building which is O(N log N).
+     *
+     * Designed for millions of entities with:
+     * - Cursor-based pagination (no memory overflow)
+     * - Batch processing (configurable batch size)
+     * - Progress reporting (optional callback)
+     * - Error recovery (continues on partial failures)
+     * - Lazy mode support (memory-efficient for constrained environments)
+     *
+     * @param options Rebuild options
+     * @returns Promise that resolves when rebuild is complete
+     */
+    async rebuild(options = {}) {
+        if (!this.storage) {
+            prodLog.warn('HNSW rebuild skipped: no storage adapter configured');
+            return;
+        }
+        const batchSize = options.batchSize || 1000;
+        try {
+            // Step 1: Clear existing in-memory index
+            this.clear();
+            // Step 2: Load system data (entry point, max level)
+            const systemData = await this.storage.getHNSWSystem();
+            if (systemData) {
+                this.entryPointId = systemData.entryPointId;
+                this.maxLevel = systemData.maxLevel;
+            }
+            // Step 3: Determine preloading strategy (adaptive caching)
+            // Check if vectors should be preloaded at init or loaded on-demand
+            const stats = await this.storage.getStatistics();
+            const entityCount = stats?.totalNodes || 0;
+            // Estimate memory needed for all vectors (384 dims × 4 bytes = 1536 bytes/vector)
+            const vectorMemory = entityCount * 1536;
+            // Get available cache size (80% threshold - preload only if fits comfortably)
+            const cacheStats = this.unifiedCache.getStats();
+            const availableCache = cacheStats.maxSize * 0.80;
+            const shouldPreload = vectorMemory < availableCache;
+            if (shouldPreload) {
+                prodLog.info(`HNSW: Preloading ${entityCount.toLocaleString()} vectors at init ` +
+                    `(${(vectorMemory / 1024 / 1024).toFixed(1)}MB < ${(availableCache / 1024 / 1024).toFixed(1)}MB cache)`);
+            }
+            else {
+                prodLog.info(`HNSW: Adaptive caching for ${entityCount.toLocaleString()} vectors ` +
+                    `(${(vectorMemory / 1024 / 1024).toFixed(1)}MB > ${(availableCache / 1024 / 1024).toFixed(1)}MB cache) - loading on-demand`);
+            }
+            // Step 4: Paginate through all nouns and restore HNSW graph structure
+            let loadedCount = 0;
+            let totalCount = undefined;
+            let hasMore = true;
+            let cursor = undefined;
+            while (hasMore) {
+                // Fetch batch of nouns from storage (cast needed as method is not in base interface)
+                const result = await this.storage.getNounsWithPagination({
+                    limit: batchSize,
+                    cursor
+                });
+                // Set total count on first batch
+                if (totalCount === undefined && result.totalCount !== undefined) {
+                    totalCount = result.totalCount;
+                }
+                // Process each noun in the batch
+                for (const nounData of result.items) {
+                    try {
+                        // Load HNSW graph data for this entity
+                        const hnswData = await this.storage.getHNSWData(nounData.id);
+                        if (!hnswData) {
+                            // No HNSW data - skip (might be entity added before persistence)
+                            continue;
+                        }
+                        // Create noun object with restored connections
+                        const noun = {
+                            id: nounData.id,
+                            vector: shouldPreload ? nounData.vector : [], // Preload if dataset is small
+                            connections: new Map(),
+                            level: hnswData.level
+                        };
+                        // Restore connections from persisted data
+                        for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) {
+                            const level = parseInt(levelStr, 10);
+                            noun.connections.set(level, new Set(nounIds));
+                        }
+                        // Add to in-memory index
+                        this.nouns.set(nounData.id, noun);
+                        // Track high-level nodes for O(1) entry point selection
+                        if (noun.level >= 2 && noun.level <= this.MAX_TRACKED_LEVELS) {
+                            if (!this.highLevelNodes.has(noun.level)) {
+                                this.highLevelNodes.set(noun.level, new Set());
+                            }
+                            this.highLevelNodes.get(noun.level).add(nounData.id);
+                        }
+                        loadedCount++;
+                    }
+                    catch (error) {
+                        // Log error but continue (robust error recovery)
+                        console.error(`Failed to rebuild HNSW data for ${nounData.id}:`, error);
+                    }
+                }
+                // Report progress
+                if (options.onProgress && totalCount !== undefined) {
+                    options.onProgress(loadedCount, totalCount);
+                }
+                // Check for more data
+                hasMore = result.hasMore;
+                cursor = result.nextCursor;
+            }
+            const cacheInfo = shouldPreload
+                ? ` (vectors preloaded)`
+                : ` (adaptive caching - vectors loaded on-demand)`;
+            prodLog.info(`✅ HNSW index rebuilt: ${loadedCount.toLocaleString()} entities, ` +
+                `${this.maxLevel + 1} levels, entry point: ${this.entryPointId || 'none'}${cacheInfo}`);
+        }
+        catch (error) {
+            prodLog.error('HNSW rebuild failed:', error);
+            throw new Error(`Failed to rebuild HNSW index: ${error}`);
+        }
+    }
     /**
      * Get level statistics for understanding the hierarchy
      */
@@ -495,6 +782,97 @@ export class HNSWIndex {
             totalNodes
         };
     }
+    /**
+     * Get cache performance statistics for monitoring and diagnostics (v3.36.0+)
+     *
+     * Production-grade monitoring:
+     * - Adaptive caching strategy (preloading vs on-demand)
+     * - UnifiedCache performance (hits, misses, evictions)
+     * - HNSW-specific cache statistics
+     * - Fair competition metrics across all indexes
+     * - Actionable recommendations for tuning
+     *
+     * Use this to:
+     * - Diagnose performance issues (low hit rate = increase cache)
+     * - Monitor memory competition (fairness violations = adjust costs)
+     * - Verify adaptive caching decisions (memory estimates vs actual)
+     * - Track cache efficiency over time
+     *
+     * @returns Comprehensive caching and performance statistics
+     */
+    getCacheStats() {
+        // Get UnifiedCache stats
+        const cacheStats = this.unifiedCache.getStats();
+        // Calculate entity and memory estimates
+        const entityCount = this.nouns.size;
+        const vectorDimension = this.dimension || 384;
+        const bytesPerVector = vectorDimension * 4; // float32
+        const estimatedVectorMemoryMB = (entityCount * bytesPerVector) / (1024 * 1024);
+        const availableCacheMB = (cacheStats.maxSize * 0.8) / (1024 * 1024); // 80% threshold
+        // Calculate HNSW-specific cache stats
+        const vectorsInCache = cacheStats.typeCounts.hnsw || 0;
+        const hnswMemoryBytes = cacheStats.typeSizes.hnsw || 0;
+        // Calculate fairness metrics
+        const hnswAccessCount = cacheStats.typeAccessCounts.hnsw || 0;
+        const totalAccessCount = cacheStats.totalAccessCount;
+        const hnswAccessPercent = totalAccessCount > 0 ? (hnswAccessCount / totalAccessCount) * 100 : 0;
+        // Detect fairness violation (>90% cache with <10% access)
+        const hnswCachePercent = cacheStats.maxSize > 0 ? (hnswMemoryBytes / cacheStats.maxSize) * 100 : 0;
+        const fairnessViolation = hnswCachePercent > 90 && hnswAccessPercent < 10;
+        // Calculate hit rate from cache
+        const hitRatePercent = (cacheStats.hitRate * 100) || 0;
+        // Determine caching strategy (same logic as rebuild())
+        const cachingStrategy = estimatedVectorMemoryMB < availableCacheMB ? 'preloaded' : 'on-demand';
+        // Generate actionable recommendations
+        const recommendations = [];
+        if (cachingStrategy === 'on-demand' && hitRatePercent < 50) {
+            recommendations.push(`Low cache hit rate (${hitRatePercent.toFixed(1)}%). Consider increasing UnifiedCache size for better performance`);
+        }
+        if (cachingStrategy === 'preloaded' && estimatedVectorMemoryMB > availableCacheMB * 0.5) {
+            recommendations.push(`Dataset growing (${estimatedVectorMemoryMB.toFixed(1)}MB). May switch to on-demand caching as entities increase`);
+        }
+        if (fairnessViolation) {
+            recommendations.push(`Fairness violation: HNSW using ${hnswCachePercent.toFixed(1)}% cache with only ${hnswAccessPercent.toFixed(1)}% access`);
+        }
+        if (cacheStats.utilization > 0.95) {
+            recommendations.push(`Cache utilization high (${(cacheStats.utilization * 100).toFixed(1)}%). Consider increasing cache size`);
+        }
+        if (recommendations.length === 0) {
+            recommendations.push('All metrics healthy - no action needed');
+        }
+        return {
+            cachingStrategy,
+            autoDetection: {
+                entityCount,
+                estimatedVectorMemoryMB: parseFloat(estimatedVectorMemoryMB.toFixed(2)),
+                availableCacheMB: parseFloat(availableCacheMB.toFixed(2)),
+                threshold: 0.8, // 80% of UnifiedCache
+                rationale: cachingStrategy === 'preloaded'
+                    ? `Vectors preloaded at init (${estimatedVectorMemoryMB.toFixed(1)}MB < ${availableCacheMB.toFixed(1)}MB threshold)`
+                    : `Adaptive on-demand loading (${estimatedVectorMemoryMB.toFixed(1)}MB > ${availableCacheMB.toFixed(1)}MB threshold)`
+            },
+            unifiedCache: {
+                totalSize: cacheStats.totalSize,
+                maxSize: cacheStats.maxSize,
+                utilizationPercent: parseFloat((cacheStats.utilization * 100).toFixed(2)),
+                itemCount: cacheStats.itemCount,
+                hitRatePercent: parseFloat(hitRatePercent.toFixed(2)),
+                totalAccessCount: cacheStats.totalAccessCount
+            },
+            hnswCache: {
+                vectorsInCache,
+                cacheKeyPrefix: 'hnsw:vector:',
+                estimatedMemoryMB: parseFloat((hnswMemoryBytes / (1024 * 1024)).toFixed(2))
+            },
+            fairness: {
+                hnswAccessCount,
+                hnswAccessPercent: parseFloat(hnswAccessPercent.toFixed(2)),
+                totalAccessCount,
+                fairnessViolation
+            },
+            recommendations
+        };
+    }
     /**
      * Search within a specific layer
      * Returns a map of noun IDs to distances, sorted by distance
@@ -502,8 +880,10 @@ export class HNSWIndex {
     async searchLayer(queryVector, entryPoint, ef, level, filter) {
         // Set of visited nouns
         const visited = new Set([entryPoint.id]);
-        // Check if entry point passes filter
-        const entryPointDistance = this.distanceFunction(queryVector, entryPoint.vector);
+        // OPTIMIZATION: Preload entry point vector
+        await this.preloadVectors([entryPoint.id]);
+        // Check if entry point passes filter (with sync fast path)
+        const entryPointDistance = await Promise.resolve(this.distanceSafe(queryVector, entryPoint));
         const entryPointPasses = filter ? await filter(entryPoint.id) : true;
         // Priority queue of candidates (closest first)
         const candidates = new Map();
@@ -526,10 +906,17 @@ export class HNSWIndex {
             // Explore neighbors of the closest candidate
             const noun = this.nouns.get(closestId);
             if (!noun) {
-                console.error(`Noun with ID ${closestId} not found in searchLayer`);
+                prodLog.error(`Noun with ID ${closestId} not found in searchLayer`);
                 continue;
             }
             const connections = noun.connections.get(level) || new Set();
+            // OPTIMIZATION: Preload unvisited neighbor vectors in parallel
+            if (connections.size > 0) {
+                const unvisitedIds = Array.from(connections).filter(id => !visited.has(id));
+                if (unvisitedIds.length > 0) {
+                    await this.preloadVectors(unvisitedIds);
+                }
+            }
             // If we have enough connections and parallelization is enabled, use parallel distance calculation
             if (this.useParallelization && connections.size >= 10) {
                 // Collect unvisited neighbors
@@ -540,7 +927,8 @@ export class HNSWIndex {
                         const neighbor = this.nouns.get(neighborId);
                         if (!neighbor)
                             continue;
-                        unvisitedNeighbors.push({ id: neighborId, vector: neighbor.vector });
+                        const neighborVector = await this.getVectorSafe(neighbor);
+                        unvisitedNeighbors.push({ id: neighborId, vector: neighborVector });
                     }
                 }
                 if (unvisitedNeighbors.length > 0) {
@@ -580,7 +968,7 @@ export class HNSWIndex {
                             // Skip neighbors that don't exist (expected during rapid additions/deletions)
                             continue;
                         }
-                        const distToNeighbor = this.distanceFunction(queryVector, neighbor.vector);
+                        const distToNeighbor = await Promise.resolve(this.distanceSafe(queryVector, neighbor));
                         // Apply filter if provided
                         const passes = filter ? await filter(neighborId) : true;
                         // Always add to candidates for graph traversal
@@ -625,7 +1013,7 @@ export class HNSWIndex {
     /**
      * Ensure a noun doesn't have too many connections at a given level
      */
-    pruneConnections(noun, level) {
+    async pruneConnections(noun, level) {
         const connections = noun.connections.get(level);
         if (connections.size <= this.config.M) {
             return;
@@ -633,14 +1021,20 @@ export class HNSWIndex {
         // Calculate distances to all neighbors
         const distances = new Map();
         const validNeighborIds = new Set();
+        // OPTIMIZATION: Preload all neighbor vectors
+        if (connections.size > 0) {
+            await this.preloadVectors(Array.from(connections));
+        }
         for (const neighborId of connections) {
             const neighbor = this.nouns.get(neighborId);
             if (!neighbor) {
                 // Skip neighbors that don't exist (expected during rapid additions/deletions)
                 continue;
             }
-            // Only add valid neighbors to the distances map
-            distances.set(neighborId, this.distanceFunction(noun.vector, neighbor.vector));
+            // Only add valid neighbors to the distances map (handles lazy loading + sync fast path)
+            const nounVector = await this.getVectorSafe(noun);
+            const distance = await Promise.resolve(this.distanceSafe(nounVector, neighbor));
+            distances.set(neighborId, distance);
             validNeighborIds.add(neighborId);
         }
         // Only proceed if we have valid neighbors

package/dist/hnsw/hnswIndexOptimized.d.ts

@@ -5,7 +5,7 @@
  */
 import { DistanceFunction, HNSWConfig, Vector, VectorDocument } from '../coreTypes.js';
 import { HNSWIndex } from './hnswIndex.js';
-import { StorageAdapter } from '../coreTypes.js';
+import type { BaseStorage } from '../storage/baseStorage.js';
 export interface HNSWOptimizedConfig extends HNSWConfig {
     memoryThreshold?: number;
     productQuantization?: {
@@ -88,15 +88,13 @@ declare class ProductQuantizer {
 export declare class HNSWIndexOptimized extends HNSWIndex {
     private optimizedConfig;
     private productQuantizer;
-    private storage;
     private useDiskBasedIndex;
     private useProductQuantization;
     private quantizedVectors;
     private memoryUsage;
     private vectorCount;
     private memoryUpdateLock;
-    private unifiedCache;
-    constructor(config: Partial<HNSWOptimizedConfig>, distanceFunction: DistanceFunction, storage?: StorageAdapter | null);
+    constructor(config: Partial<HNSWOptimizedConfig>, distanceFunction: DistanceFunction, storage?: BaseStorage | null);
     /**
      * Thread-safe method to update memory usage
      * @param memoryDelta Change in memory usage (can be negative)
@@ -121,7 +119,7 @@ export declare class HNSWIndexOptimized extends HNSWIndex {
     /**
      * Remove an item from the index
      */
-    removeItem(id: string): boolean;
+    removeItem(id: string): Promise<boolean>;
     /**
      * Clear the index
      */
@@ -145,16 +143,6 @@ export declare class HNSWIndexOptimized extends HNSWIndex {
      * @returns Estimated memory usage in bytes
      */
     getMemoryUsage(): number;
-    /**
-     * Set the storage adapter
-     * @param storage Storage adapter
-     */
-    setStorage(storage: StorageAdapter): void;
-    /**
-     * Get the storage adapter
-     * @returns Storage adapter or null if not set
-     */
-    getStorage(): StorageAdapter | null;
     /**
      * Set whether to use disk-based index
      * @param useDiskBasedIndex Whether to use disk-based index