@soulcraft/brainy 3.47.0 → 3.47.1

package/dist/brainy.js

@@ -2523,6 +2523,14 @@ export class Brainy {
                 }
                 return;
             }
+            // OPTIMIZATION: Instant check - if index already has data, skip immediately
+            // This gives 0s startup for warm restarts (vs 50-100ms of async checks)
+            if (this.index.size() > 0) {
+                if (!this.config.silent) {
+                    console.log(`✅ Index already populated (${this.index.size().toLocaleString()} entities) - 0s startup!`);
+                }
+                return;
+            }
             // BUG #2 FIX: Don't trust counts - check actual storage instead
             // Counts can be lost/corrupted in container restarts
             const entities = await this.storage.getNouns({ pagination: { limit: 1 } });
@@ -2543,31 +2551,31 @@ export class Brainy {
                 graphIndexSize === 0 ||
                 this.config.disableAutoRebuild === false; // Explicitly enabled
             if (!needsRebuild) {
-                // All indexes populated, no rebuild needed
+                // All indexes already populated, no rebuild needed
                 return;
             }
             // Small dataset: Rebuild all indexes for best performance
             if (totalCount < AUTO_REBUILD_THRESHOLD || this.config.disableAutoRebuild === false) {
                 if (!this.config.silent) {
                     console.log(this.config.disableAutoRebuild === false
-                        ? '🔄 Auto-rebuild explicitly enabled - rebuilding all indexes...'
-                        : `🔄 Small dataset (${totalCount} items) - rebuilding all indexes...`);
+                        ? '🔄 Auto-rebuild explicitly enabled - rebuilding all indexes from persisted data...'
+                        : `🔄 Small dataset (${totalCount} items) - rebuilding all indexes from persisted data...`);
                 }
-                // BUG #1 FIX: Actually call graphIndex.rebuild()
-                // BUG #4 FIX: Actually call HNSW index.rebuild()
                 // Rebuild all 3 indexes in parallel for performance
-                const startTime = Date.now();
+                // Indexes load their data from storage (no recomputation)
+                const rebuildStartTime = Date.now();
                 await Promise.all([
                     metadataStats.totalEntries === 0 ? this.metadataIndex.rebuild() : Promise.resolve(),
                     hnswIndexSize === 0 ? this.index.rebuild() : Promise.resolve(),
                     graphIndexSize === 0 ? this.graphIndex.rebuild() : Promise.resolve()
                 ]);
-                const duration = Date.now() - startTime;
+                const rebuildDuration = Date.now() - rebuildStartTime;
                 if (!this.config.silent) {
-                    console.log(`✅ All indexes rebuilt in ${duration}ms:\n` +
+                    console.log(`✅ All indexes rebuilt in ${rebuildDuration}ms:\n` +
                         `   - Metadata: ${await this.metadataIndex.getStats().then(s => s.totalEntries)} entries\n` +
                         `   - HNSW Vector: ${this.index.size()} nodes\n` +
-                        `   - Graph Adjacency: ${await this.graphIndex.size()} relationships`);
+                        `   - Graph Adjacency: ${await this.graphIndex.size()} relationships\n` +
+                        `   💡 Indexes loaded from persisted storage (no recomputation)`);
                 }
             }
             else {

package/dist/hnsw/typeAwareHNSWIndex.js

@@ -292,65 +292,121 @@ export class TypeAwareHNSWIndex {
             prodLog.warn('TypeAwareHNSW rebuild skipped: no storage adapter');
             return;
         }
+        const batchSize = options.batchSize || 1000;
         // Determine which types to rebuild
         const typesToRebuild = options.types || this.getAllNounTypes();
-        prodLog.info(`Rebuilding ${typesToRebuild.length} type-aware HNSW indexes...`);
-        const errors = [];
-        // Rebuild each type's index with type-filtered pagination
+        prodLog.info(`Rebuilding ${typesToRebuild.length} type-aware HNSW indexes from persisted data...`);
+        // Clear all indexes we're rebuilding
         for (const type of typesToRebuild) {
-            try {
-                prodLog.info(`Rebuilding HNSW index for type: ${type}`);
-                const index = this.getIndexForType(type);
-                index.clear(); // Clear before rebuild
-                // Load ONLY entities of this type from storage using pagination
-                let cursor = undefined;
-                let hasMore = true;
-                let loaded = 0;
-                while (hasMore) {
-                    // CRITICAL: Use type filtering to load only this type's entities
-                    const result = await this.storage.getNounsWithPagination({
-                        limit: options.batchSize || 1000,
-                        cursor,
-                        filter: { nounType: type } // ← TYPE FILTER!
-                    });
-                    // Add each entity to this type's index
-                    for (const noun of result.items) {
-                        try {
-                            await index.addItem({
-                                id: noun.id,
-                                vector: noun.vector
-                            });
-                            loaded++;
-                            if (options.onProgress) {
-                                options.onProgress(type, loaded, result.totalCount || loaded);
-                            }
-                        }
-                        catch (error) {
-                            prodLog.error(`Failed to add entity ${noun.id} to ${type} index:`, error);
-                            // Continue with other entities
+            const index = this.getIndexForType(type);
+            index.nouns.clear();
+        }
+        // Determine preloading strategy (adaptive caching) for entire dataset
+        const stats = await this.storage.getStatistics();
+        const entityCount = stats?.totalNodes || 0;
+        const vectorMemory = entityCount * 1536; // 384 dims × 4 bytes
+        // Use first index's cache (they all share the same UnifiedCache)
+        const firstIndex = this.getIndexForType(typesToRebuild[0]);
+        const cacheStats = firstIndex.unifiedCache.getStats();
+        const availableCache = cacheStats.maxSize * 0.80;
+        const shouldPreload = vectorMemory < availableCache;
+        if (shouldPreload) {
+            prodLog.info(`HNSW: Preloading ${entityCount.toLocaleString()} vectors at init ` +
+                `(${(vectorMemory / 1024 / 1024).toFixed(1)}MB < ${(availableCache / 1024 / 1024).toFixed(1)}MB cache)`);
+        }
+        else {
+            prodLog.info(`HNSW: Adaptive caching for ${entityCount.toLocaleString()} vectors ` +
+                `(${(vectorMemory / 1024 / 1024).toFixed(1)}MB > ${(availableCache / 1024 / 1024).toFixed(1)}MB cache) - loading on-demand`);
+        }
+        // Load ALL nouns ONCE and route to correct type indexes
+        // This is O(N) instead of O(31*N) from the previous parallel approach
+        let cursor = undefined;
+        let hasMore = true;
+        let totalLoaded = 0;
+        const loadedByType = new Map();
+        while (hasMore) {
+            const result = await this.storage.getNounsWithPagination({
+                limit: batchSize,
+                cursor
+            });
+            // Route each noun to its type index
+            for (const nounData of result.items) {
+                try {
+                    // Determine noun type from multiple possible sources
+                    const nounType = nounData.nounType || nounData.metadata?.noun || nounData.metadata?.type;
+                    // Skip if type not in rebuild list
+                    if (!nounType || !typesToRebuild.includes(nounType)) {
+                        continue;
+                    }
+                    // Get the index for this type
+                    const index = this.getIndexForType(nounType);
+                    // Load HNSW graph data
+                    const hnswData = await this.storage.getHNSWData(nounData.id);
+                    if (!hnswData) {
+                        continue; // No HNSW data
+                    }
+                    // Create noun with restored connections
+                    const noun = {
+                        id: nounData.id,
+                        vector: shouldPreload ? nounData.vector : [],
+                        connections: new Map(),
+                        level: hnswData.level
+                    };
+                    // Restore connections from storage
+                    for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) {
+                        const level = parseInt(levelStr, 10);
+                        noun.connections.set(level, new Set(nounIds));
+                    }
+                    // Add to type-specific index
+                    ;
+                    index.nouns.set(nounData.id, noun);
+                    // Track high-level nodes
+                    if (noun.level >= 2 && noun.level <= index.MAX_TRACKED_LEVELS) {
+                        if (!index.highLevelNodes.has(noun.level)) {
+                            ;
+                            index.highLevelNodes.set(noun.level, new Set());
                         }
+                        ;
+                        index.highLevelNodes.get(noun.level).add(nounData.id);
                     }
-                    hasMore = result.hasMore;
-                    cursor = result.nextCursor;
+                    // Track progress
+                    loadedByType.set(nounType, (loadedByType.get(nounType) || 0) + 1);
+                    totalLoaded++;
+                    if (options.onProgress && totalLoaded % 100 === 0) {
+                        options.onProgress(nounType, loadedByType.get(nounType) || 0, totalLoaded);
+                    }
+                }
+                catch (error) {
+                    prodLog.error(`Failed to restore HNSW data for ${nounData.id}:`, error);
                 }
-                prodLog.info(`✅ Rebuilt ${type} index: ${index.size().toLocaleString()} entities`);
             }
-            catch (error) {
-                prodLog.error(`Failed to rebuild ${type} index:`, error);
-                errors.push({ type, error: error });
-                // Continue with other types instead of failing completely
+            hasMore = result.hasMore;
+            cursor = result.nextCursor;
+            // Progress logging
+            if (totalLoaded % 1000 === 0) {
+                prodLog.info(`Progress: ${totalLoaded.toLocaleString()} entities loaded...`);
             }
         }
-        // Report errors at end
-        if (errors.length > 0) {
-            const failedTypes = errors.map((e) => e.type).join(', ');
-            prodLog.warn(`⚠️ Failed to rebuild ${errors.length} type indexes: ${failedTypes}`);
-            // Throw if ALL rebuilds failed
-            if (errors.length === typesToRebuild.length) {
-                throw new Error('All type-aware HNSW rebuilds failed');
+        // Restore entry points for each type
+        for (const type of typesToRebuild) {
+            const index = this.getIndexForType(type);
+            let maxLevel = 0;
+            let entryPointId = null;
+            for (const [id, noun] of index.nouns.entries()) {
+                if (noun.level > maxLevel) {
+                    maxLevel = noun.level;
+                    entryPointId = id;
+                }
             }
+            ;
+            index.entryPointId = entryPointId;
+            index.maxLevel = maxLevel;
+            const loaded = loadedByType.get(type) || 0;
+            const cacheInfo = shouldPreload ? ' (vectors preloaded)' : ' (adaptive caching)';
+            prodLog.info(`✅ Rebuilt ${type} index: ${loaded.toLocaleString()} entities, ` +
+                `${maxLevel + 1} levels, entry point: ${entryPointId || 'none'}${cacheInfo}`);
         }
-        prodLog.info(`✅ TypeAwareHNSW rebuild complete: ${this.size().toLocaleString()} total entities across ${this.indexes.size} types`);
+        prodLog.info(`✅ TypeAwareHNSW rebuild complete: ${this.size().toLocaleString()} total entities across ${this.indexes.size} types (loaded from persisted graph structure)`);
     }
     /**
      * Get comprehensive statistics

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.47.0",
+  "version": "3.47.1",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",