@soulcraft/brainy 6.2.1 → 6.2.3

package/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [6.2.2](https://github.com/soulcraftlabs/brainy/compare/v6.2.1...v6.2.2) (2025-11-25)
+
+- refactor: remove 3,700+ LOC of unused HNSW implementations (e3146ce)
+- fix(hnsw): entry point recovery prevents import failures and log spam (52eae67)
+
+
 ## [6.2.0](https://github.com/soulcraftlabs/brainy/compare/v6.1.0...v6.2.0) (2025-11-20)
 
 ### ⚡ Critical Performance Fix

package/dist/brainy.js

@@ -6,7 +6,6 @@
  */
 import { v4 as uuidv4 } from './universal/uuid.js';
 import { HNSWIndex } from './hnsw/hnswIndex.js';
-import { HNSWIndexOptimized } from './hnsw/hnswIndexOptimized.js';
 import { TypeAwareHNSWIndex } from './hnsw/typeAwareHNSWIndex.js';
 import { createStorage } from './storage/storageFactory.js';
 import { defaultEmbeddingFunction, cosineDistance } from './utils/index.js';
@@ -802,7 +801,7 @@ export class Brainy {
                     if (this.index instanceof TypeAwareHNSWIndex && metadata.noun) {
                         tx.addOperation(new RemoveFromTypeAwareHNSWOperation(this.index, id, noun.vector, metadata.noun));
                     }
-                    else if (this.index instanceof HNSWIndex || this.index instanceof HNSWIndexOptimized) {
+                    else if (this.index instanceof HNSWIndex) {
                         tx.addOperation(new RemoveFromHNSWOperation(this.index, id, noun.vector));
                     }
                 }
@@ -1887,7 +1886,7 @@ export class Brainy {
                                 if (this.index instanceof TypeAwareHNSWIndex && metadata.noun) {
                                     tx.addOperation(new RemoveFromTypeAwareHNSWOperation(this.index, id, noun.vector, metadata.noun));
                                 }
-                                else if (this.index instanceof HNSWIndex || this.index instanceof HNSWIndexOptimized) {
+                                else if (this.index instanceof HNSWIndex) {
                                     tx.addOperation(new RemoveFromHNSWOperation(this.index, id, noun.vector));
                                 }
                             }

package/dist/hnsw/hnswIndex.d.ts

@@ -83,6 +83,12 @@ export declare class HNSWIndex {
      * Add a vector to the index
      */
     addItem(item: VectorDocument): Promise<string>;
+    /**
+     * O(1) entry point recovery using highLevelNodes index (v6.2.3).
+     * At any reasonable scale (1000+ nodes), level 2+ nodes are guaranteed to exist.
+     * For tiny indexes with only level 0-1 nodes, any node works as entry point.
+     */
+    private recoverEntryPointO1;
     /**
      * Search for nearest neighbors
      */

package/dist/hnsw/hnswIndex.js

@@ -210,9 +210,8 @@ export class HNSWIndex {
         }
         // Find entry point
         if (!this.entryPointId) {
-            console.error('Entry point ID is null');
-            // If there's no entry point, this is the first noun, so we should have returned earlier
-            // This is a safety check
+            // No entry point but nouns exist - corrupted state, recover by using this item
+            // This shouldn't normally happen as first item sets entry point above
             this.entryPointId = id;
             this.maxLevel = nounLevel;
             this.nouns.set(id, noun);
@@ -220,7 +219,7 @@ export class HNSWIndex {
         }
         const entryPoint = this.nouns.get(this.entryPointId);
         if (!entryPoint) {
-            console.error(`Entry point with ID ${this.entryPointId} not found`);
+            // Entry point was deleted but ID not updated - recover by using new item
             // If the entry point doesn't exist, treat this as the first noun
             this.entryPointId = id;
             this.maxLevel = nounLevel;
@@ -383,6 +382,27 @@ export class HNSWIndex {
         }
         return id;
     }
+    /**
+     * O(1) entry point recovery using highLevelNodes index (v6.2.3).
+     * At any reasonable scale (1000+ nodes), level 2+ nodes are guaranteed to exist.
+     * For tiny indexes with only level 0-1 nodes, any node works as entry point.
+     */
+    recoverEntryPointO1() {
+        // O(1) recovery: check highLevelNodes from highest to lowest level
+        for (let level = this.MAX_TRACKED_LEVELS; level >= 2; level--) {
+            const nodesAtLevel = this.highLevelNodes.get(level);
+            if (nodesAtLevel && nodesAtLevel.size > 0) {
+                for (const nodeId of nodesAtLevel) {
+                    if (this.nouns.has(nodeId)) {
+                        return { id: nodeId, level };
+                    }
+                }
+            }
+        }
+        // No high-level nodes - use any available node (works fine for HNSW)
+        const firstNode = this.nouns.keys().next().value;
+        return { id: firstNode ?? null, level: 0 };
+    }
     /**
      * Search for nearest neighbors
      */
@@ -398,14 +418,33 @@ export class HNSWIndex {
             throw new Error(`Query vector dimension mismatch: expected ${this.dimension}, got ${queryVector.length}`);
         }
         // Start from the entry point
+        // If entry point is null but nouns exist, attempt O(1) recovery (v6.2.3)
+        if (!this.entryPointId && this.nouns.size > 0) {
+            const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1();
+            if (recoveredId) {
+                this.entryPointId = recoveredId;
+                this.maxLevel = recoveredLevel;
+            }
+        }
         if (!this.entryPointId) {
-            console.error('Entry point ID is null');
+            // Truly empty index - return empty results silently
             return [];
         }
-        const entryPoint = this.nouns.get(this.entryPointId);
+        let entryPoint = this.nouns.get(this.entryPointId);
         if (!entryPoint) {
-            console.error(`Entry point with ID ${this.entryPointId} not found`);
-            return [];
+            // Entry point ID exists but noun was deleted - O(1) recovery (v6.2.3)
+            if (this.nouns.size > 0) {
+                const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1();
+                if (recoveredId) {
+                    this.entryPointId = recoveredId;
+                    this.maxLevel = recoveredLevel;
+                    entryPoint = this.nouns.get(recoveredId);
+                }
+            }
+            // If still no entry point, return empty
+            if (!entryPoint) {
+                return [];
+            }
         }
         let currObj = entryPoint;
         // OPTIMIZATION: Preload entry point vector
@@ -915,6 +954,40 @@ export class HNSWIndex {
                     offset += batchSize; // v5.7.11: Increment offset for next page
                 }
             }
+            // Step 5: CRITICAL - Recover entry point if missing (v6.2.3 - O(1))
+            // This ensures consistency even if getHNSWSystem() returned null
+            if (this.nouns.size > 0 && this.entryPointId === null) {
+                prodLog.warn('HNSW rebuild: Entry point was null after loading nouns - recovering with O(1) lookup');
+                const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1();
+                this.entryPointId = recoveredId;
+                this.maxLevel = recoveredLevel;
+                prodLog.info(`HNSW entry point recovered: ${recoveredId} at level ${recoveredLevel}`);
+                // Persist recovered state to prevent future recovery
+                if (this.storage && recoveredId) {
+                    await this.storage.saveHNSWSystem({
+                        entryPointId: this.entryPointId,
+                        maxLevel: this.maxLevel
+                    }).catch((error) => {
+                        prodLog.error('Failed to persist recovered HNSW system data:', error);
+                    });
+                }
+            }
+            // Step 6: Validate entry point exists if set (handles stale/deleted entry point)
+            if (this.entryPointId && !this.nouns.has(this.entryPointId)) {
+                prodLog.warn(`HNSW: Entry point ${this.entryPointId} not found in loaded nouns - recovering with O(1) lookup`);
+                const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1();
+                this.entryPointId = recoveredId;
+                this.maxLevel = recoveredLevel;
+                // Persist corrected state
+                if (this.storage && recoveredId) {
+                    await this.storage.saveHNSWSystem({
+                        entryPointId: this.entryPointId,
+                        maxLevel: this.maxLevel
+                    }).catch((error) => {
+                        prodLog.error('Failed to persist corrected HNSW system data:', error);
+                    });
+                }
+            }
             const cacheInfo = shouldPreload
                 ? ` (vectors preloaded)`
                 : ` (adaptive caching - vectors loaded on-demand)`;

package/dist/index.d.ts

@@ -51,9 +51,8 @@ export { StorageAugmentation, DynamicStorageAugmentation, MemoryStorageAugmentat
 export { WebSocketConduitAugmentation };
 import type { Vector, VectorDocument, SearchResult, DistanceFunction, EmbeddingFunction, EmbeddingModel, HNSWNoun, HNSWVerb, HNSWConfig, StorageAdapter } from './coreTypes.js';
 import { HNSWIndex } from './hnsw/hnswIndex.js';
-import { HNSWIndexOptimized, HNSWOptimizedConfig } from './hnsw/hnswIndexOptimized.js';
-export { HNSWIndex, HNSWIndexOptimized };
-export type { Vector, VectorDocument, SearchResult, DistanceFunction, EmbeddingFunction, EmbeddingModel, HNSWNoun, HNSWVerb, HNSWConfig, HNSWOptimizedConfig, StorageAdapter };
+export { HNSWIndex };
+export type { Vector, VectorDocument, SearchResult, DistanceFunction, EmbeddingFunction, EmbeddingModel, HNSWNoun, HNSWVerb, HNSWConfig, StorageAdapter };
 import type { AugmentationResponse, BrainyAugmentation, BaseAugmentation, AugmentationContext } from './types/augmentations.js';
 export { AugmentationManager, type AugmentationInfo } from './augmentationManager.js';
 export type { AugmentationResponse, BrainyAugmentation, BaseAugmentation, AugmentationContext };

package/dist/index.js

@@ -111,10 +111,9 @@ MemoryStorageAugmentation, FileSystemStorageAugmentation, OPFSStorageAugmentatio
 createAutoStorageAugmentation, createStorageAugmentationFromConfig };
 // Other augmentation exports
 export { WebSocketConduitAugmentation };
-// Export HNSW index and optimized version
+// Export HNSW index
 import { HNSWIndex } from './hnsw/hnswIndex.js';
-import { HNSWIndexOptimized } from './hnsw/hnswIndexOptimized.js';
-export { HNSWIndex, HNSWIndexOptimized };
+export { HNSWIndex };
 // Export augmentation manager for type-safe augmentation management
 export { AugmentationManager } from './augmentationManager.js';
 import { NounType, VerbType } from './types/graphTypes.js';

package/dist/storage/adapters/azureBlobStorage.js

@@ -1269,7 +1269,15 @@ export class AzureBlobStorage extends BaseStorage {
             return JSON.parse(downloaded.toString());
         }
         catch (error) {
-            if (error.statusCode === 404 || error.code === 'BlobNotFound') {
+            // Azure may return not found errors in different formats
+            const isNotFound = error.statusCode === 404 ||
+                error.code === 'BlobNotFound' ||
+                error.code === 404 ||
+                error.details?.code === 'BlobNotFound' ||
+                error.message?.includes('BlobNotFound') ||
+                error.message?.includes('not found') ||
+                error.message?.includes('404');
+            if (isNotFound) {
                 return null;
             }
             this.logger.error('Failed to get HNSW system data:', error);

package/dist/storage/adapters/gcsStorage.js

@@ -1260,7 +1260,14 @@ export class GcsStorage extends BaseStorage {
             return JSON.parse(contents.toString());
         }
         catch (error) {
-            if (error.code === 404) {
+            // GCS may return 404 in different formats depending on SDK version
+            const is404 = error.code === 404 ||
+                error.statusCode === 404 ||
+                error.status === 404 ||
+                error.message?.includes('No such object') ||
+                error.message?.includes('not found') ||
+                error.message?.includes('404');
+            if (is404) {
                 return null;
             }
             this.logger.error('Failed to get HNSW system data:', error);

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -2829,9 +2829,14 @@ export class S3CompatibleStorage extends BaseStorage {
             return JSON.parse(bodyContents);
         }
         catch (error) {
-            if (error.name === 'NoSuchKey' ||
+            // S3 may return not found errors in different formats
+            const isNotFound = error.name === 'NoSuchKey' ||
+                error.code === 'NoSuchKey' ||
+                error.$metadata?.httpStatusCode === 404 ||
                 error.message?.includes('NoSuchKey') ||
-                error.message?.includes('not found')) {
+                error.message?.includes('not found') ||
+                error.message?.includes('404');
+            if (isNotFound) {
                 return null;
             }
             this.logger.error('Failed to get HNSW system data:', error);

package/dist/storage/baseStorage.d.ts

@@ -618,6 +618,18 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * Periodically called when counts are updated
      */
     protected saveTypeStatistics(): Promise<void>;
+    /**
+     * Get noun counts by type (O(1) access to type statistics)
+     * v6.2.2: Exposed for MetadataIndexManager to use as single source of truth
+     * @returns Uint32Array indexed by NounType enum value (42 types)
+     */
+    getNounCountsByType(): Uint32Array;
+    /**
+     * Get verb counts by type (O(1) access to type statistics)
+     * v6.2.2: Exposed for MetadataIndexManager to use as single source of truth
+     * @returns Uint32Array indexed by VerbType enum value (127 types)
+     */
+    getVerbCountsByType(): Uint32Array;
     /**
      * Rebuild type counts from actual storage (v5.5.0)
      * Called when statistics are missing or inconsistent

package/dist/storage/baseStorage.js

@@ -1983,6 +1983,22 @@ export class BaseStorage extends BaseStorageAdapter {
         };
         await this.writeObjectToPath(`${SYSTEM_DIR}/type-statistics.json`, stats);
     }
+    /**
+     * Get noun counts by type (O(1) access to type statistics)
+     * v6.2.2: Exposed for MetadataIndexManager to use as single source of truth
+     * @returns Uint32Array indexed by NounType enum value (42 types)
+     */
+    getNounCountsByType() {
+        return this.nounCountsByType;
+    }
+    /**
+     * Get verb counts by type (O(1) access to type statistics)
+     * v6.2.2: Exposed for MetadataIndexManager to use as single source of truth
+     * @returns Uint32Array indexed by VerbType enum value (127 types)
+     */
+    getVerbCountsByType() {
+        return this.verbCountsByType;
+    }
     /**
      * Rebuild type counts from actual storage (v5.5.0)
      * Called when statistics are missing or inconsistent

package/dist/triple/TripleIntelligenceSystem.d.ts

@@ -13,7 +13,6 @@
  * - Fusion: O(k log k) where k = result count
  */
 import { HNSWIndex } from '../hnsw/hnswIndex.js';
-import { HNSWIndexOptimized } from '../hnsw/hnswIndexOptimized.js';
 import { TypeAwareHNSWIndex } from '../hnsw/typeAwareHNSWIndex.js';
 import { MetadataIndexManager } from '../utils/metadataIndex.js';
 import { Vector } from '../coreTypes.js';
@@ -68,7 +67,7 @@ export declare class TripleIntelligenceSystem {
     private planner;
     private embedder;
     private storage;
-    constructor(metadataIndex: MetadataIndexManager, hnswIndex: HNSWIndex | HNSWIndexOptimized | TypeAwareHNSWIndex, graphIndex: GraphAdjacencyIndex, embedder: (text: string) => Promise<Vector>, storage: any);
+    constructor(metadataIndex: MetadataIndexManager, hnswIndex: HNSWIndex | TypeAwareHNSWIndex, graphIndex: GraphAdjacencyIndex, embedder: (text: string) => Promise<Vector>, storage: any);
     /**
      * Main find method - executes Triple Intelligence queries
      * Phase 3: Now with automatic type inference for 40% latency reduction

package/dist/utils/metadataIndex.d.ts

@@ -112,8 +112,9 @@ export declare class MetadataIndexManager {
      */
     private releaseLock;
     /**
-     * Lazy load entity counts from storage statistics (O(1) operation)
-     * This avoids rebuilding the entire index on startup
+     * Lazy load entity counts from the 'noun' field sparse index (O(n) where n = number of types)
+     * v6.2.2 FIX: Previously read from stats.nounCount which was SERVICE-keyed, not TYPE-keyed
+     * Now computes counts from the sparse index which has the correct type information
      */
     private lazyLoadCounts;
     /**
@@ -434,6 +435,9 @@ export declare class MetadataIndexManager {
     getTotalEntityCount(): number;
     /**
      * Get all entity types and their counts - O(1) operation
+     * v6.2.2: Fixed - totalEntitiesByType is correctly populated by updateTypeFieldAffinity
+     * during add operations. lazyLoadCounts was reading wrong data but that doesn't
+     * affect freshly-added entities within the same session.
      */
     getAllEntityCounts(): Map<string, number>;
     /**

package/dist/utils/metadataIndex.js

@@ -84,8 +84,9 @@ export class MetadataIndexManager {
         this.chunkingStrategy = new AdaptiveChunkingStrategy();
         // Initialize Field Type Inference (v3.48.0)
         this.fieldTypeInference = new FieldTypeInference(storage);
-        // Lazy load counts from storage statistics on first access
-        this.lazyLoadCounts();
+        // v6.2.2: Removed lazyLoadCounts() call from constructor
+        // It was a race condition (not awaited) and read from wrong source.
+        // Now properly called in init() after warmCache() loads the sparse index.
     }
     /**
      * Initialize the metadata index manager
@@ -97,11 +98,15 @@ export class MetadataIndexManager {
         await this.loadFieldRegistry();
         // Initialize EntityIdMapper (loads UUID ↔ integer mappings from storage)
         await this.idMapper.init();
-        // Phase 1b: Sync loaded counts to fixed-size arrays
-        // This populates the Uint32Arrays from the Maps loaded by lazyLoadCounts()
-        this.syncTypeCountsToFixed();
         // Warm the cache with common fields (v3.44.1 - lazy loading optimization)
+        // This loads the 'noun' sparse index which is needed for type counts
         await this.warmCache();
+        // v6.2.2: Load type counts AFTER warmCache (sparse index is now cached)
+        // Previously called in constructor without await and read from wrong source
+        await this.lazyLoadCounts();
+        // Phase 1b: Sync loaded counts to fixed-size arrays
+        // Now correctly happens AFTER lazyLoadCounts() finishes
+        this.syncTypeCountsToFixed();
     }
     /**
      * Warm the cache by preloading common field sparse indices (v3.44.1)
@@ -226,25 +231,34 @@ export class MetadataIndexManager {
         this.activeLocks.delete(lockKey);
     }
     /**
-     * Lazy load entity counts from storage statistics (O(1) operation)
-     * This avoids rebuilding the entire index on startup
+     * Lazy load entity counts from the 'noun' field sparse index (O(n) where n = number of types)
+     * v6.2.2 FIX: Previously read from stats.nounCount which was SERVICE-keyed, not TYPE-keyed
+     * Now computes counts from the sparse index which has the correct type information
      */
     async lazyLoadCounts() {
         try {
-            // Get statistics from storage (should be O(1) with our FileSystemStorage improvements)
-            const stats = await this.storage.getStatistics();
-            if (stats && stats.nounCount) {
-                // Populate entity counts from storage statistics
-                for (const [type, count] of Object.entries(stats.nounCount)) {
-                    if (typeof count === 'number' && count > 0) {
-                        this.totalEntitiesByType.set(type, count);
+            // v6.2.2: Load counts from sparse index (correct source)
+            const nounSparseIndex = await this.loadSparseIndex('noun');
+            if (!nounSparseIndex) {
+                // No sparse index yet - counts will be populated as entities are added
+                return;
+            }
+            // Iterate through all chunks and sum up bitmap sizes by type
+            for (const chunkId of nounSparseIndex.getAllChunkIds()) {
+                const chunk = await this.chunkManager.loadChunk('noun', chunkId);
+                if (chunk) {
+                    for (const [type, bitmap] of chunk.entries) {
+                        const currentCount = this.totalEntitiesByType.get(type) || 0;
+                        this.totalEntitiesByType.set(type, currentCount + bitmap.size);
                     }
                 }
             }
+            prodLog.debug(`✅ Loaded type counts from sparse index: ${this.totalEntitiesByType.size} types`);
         }
         catch (error) {
             // Silently fail - counts will be populated as entities are added
             // This maintains zero-configuration principle
+            prodLog.debug('Could not load type counts from sparse index:', error);
         }
     }
     /**
@@ -1875,6 +1889,9 @@ export class MetadataIndexManager {
     }
     /**
      * Get all entity types and their counts - O(1) operation
+     * v6.2.2: Fixed - totalEntitiesByType is correctly populated by updateTypeFieldAffinity
+     * during add operations. lazyLoadCounts was reading wrong data but that doesn't
+     * affect freshly-added entities within the same session.
      */
     getAllEntityCounts() {
         return new Map(this.totalEntitiesByType);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "6.2.1",
+  "version": "6.2.3",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",