@soulcraft/brainy 3.32.2 → 3.35.0

package/dist/hnsw/hnswIndex.js

@@ -20,12 +20,14 @@ export class HNSWIndex {
         this.MAX_TRACKED_LEVELS = 10; // Only track top levels for memory efficiency
         this.dimension = null;
         this.useParallelization = true; // Whether to use parallelization for performance-critical operations
+        this.storage = null; // Storage adapter for HNSW persistence (v3.35.0+)
         this.config = { ...DEFAULT_CONFIG, ...config };
         this.distanceFunction = distanceFunction;
         this.useParallelization =
             options.useParallelization !== undefined
                 ? options.useParallelization
                 : true;
+        this.storage = options.storage || null;
     }
     /**
      * Set whether to use parallelization for performance-critical operations
@@ -182,6 +184,19 @@ export class HNSWIndex {
                 if (neighbor.connections.get(level).size > this.config.M) {
                     this.pruneConnections(neighbor, level);
                 }
+                // Persist updated neighbor HNSW data (v3.35.0+)
+                if (this.storage) {
+                    const neighborConnectionsObj = {};
+                    for (const [lvl, nounIds] of neighbor.connections.entries()) {
+                        neighborConnectionsObj[lvl.toString()] = Array.from(nounIds);
+                    }
+                    this.storage.saveHNSWData(neighborId, {
+                        level: neighbor.level,
+                        connections: neighborConnectionsObj
+                    }).catch((error) => {
+                        console.error(`Failed to persist neighbor HNSW data for ${neighborId}:`, error);
+                    });
+                }
             }
             // Update entry point for the next level
             if (nearestNouns.size > 0) {
@@ -213,6 +228,27 @@ export class HNSWIndex {
             }
             this.highLevelNodes.get(nounLevel).add(id);
         }
+        // Persist HNSW graph data to storage (v3.35.0+)
+        if (this.storage) {
+            // Convert connections Map to serializable format
+            const connectionsObj = {};
+            for (const [level, nounIds] of noun.connections.entries()) {
+                connectionsObj[level.toString()] = Array.from(nounIds);
+            }
+            await this.storage.saveHNSWData(id, {
+                level: nounLevel,
+                connections: connectionsObj
+            }).catch((error) => {
+                console.error(`Failed to persist HNSW data for ${id}:`, error);
+            });
+            // Persist system data (entry point and max level)
+            await this.storage.saveHNSWSystem({
+                entryPointId: this.entryPointId,
+                maxLevel: this.maxLevel
+            }).catch((error) => {
+                console.error('Failed to persist HNSW system data:', error);
+            });
+        }
         return id;
     }
     /**
@@ -451,6 +487,107 @@ export class HNSWIndex {
         }
         return nodesAtLevel;
     }
+    /**
+     * Rebuild HNSW index from persisted graph data (v3.35.0+)
+     *
+     * This is a production-grade O(N) rebuild that restores the pre-computed graph structure
+     * from storage. Much faster than re-building which is O(N log N).
+     *
+     * Designed for millions of entities with:
+     * - Cursor-based pagination (no memory overflow)
+     * - Batch processing (configurable batch size)
+     * - Progress reporting (optional callback)
+     * - Error recovery (continues on partial failures)
+     * - Lazy mode support (memory-efficient for constrained environments)
+     *
+     * @param options Rebuild options
+     * @returns Promise that resolves when rebuild is complete
+     */
+    async rebuild(options = {}) {
+        if (!this.storage) {
+            console.warn('HNSW rebuild skipped: no storage adapter configured');
+            return;
+        }
+        const batchSize = options.batchSize || 1000;
+        const lazy = options.lazy || false;
+        try {
+            // Step 1: Clear existing in-memory index
+            this.clear();
+            // Step 2: Load system data (entry point, max level)
+            const systemData = await this.storage.getHNSWSystem();
+            if (systemData) {
+                this.entryPointId = systemData.entryPointId;
+                this.maxLevel = systemData.maxLevel;
+            }
+            // Step 3: Paginate through all nouns and restore HNSW graph structure
+            let loadedCount = 0;
+            let totalCount = undefined;
+            let hasMore = true;
+            let cursor = undefined;
+            while (hasMore) {
+                // Fetch batch of nouns from storage (cast needed as method is not in base interface)
+                const result = await this.storage.getNounsWithPagination({
+                    limit: batchSize,
+                    cursor
+                });
+                // Set total count on first batch
+                if (totalCount === undefined && result.totalCount !== undefined) {
+                    totalCount = result.totalCount;
+                }
+                // Process each noun in the batch
+                for (const nounData of result.items) {
+                    try {
+                        // Load HNSW graph data for this entity
+                        const hnswData = await this.storage.getHNSWData(nounData.id);
+                        if (!hnswData) {
+                            // No HNSW data - skip (might be entity added before persistence)
+                            continue;
+                        }
+                        // Create noun object with restored connections
+                        const noun = {
+                            id: nounData.id,
+                            vector: lazy ? [] : nounData.vector, // Empty vector in lazy mode
+                            connections: new Map(),
+                            level: hnswData.level
+                        };
+                        // Restore connections from persisted data
+                        for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) {
+                            const level = parseInt(levelStr, 10);
+                            noun.connections.set(level, new Set(nounIds));
+                        }
+                        // Add to in-memory index
+                        this.nouns.set(nounData.id, noun);
+                        // Track high-level nodes for O(1) entry point selection
+                        if (noun.level >= 2 && noun.level <= this.MAX_TRACKED_LEVELS) {
+                            if (!this.highLevelNodes.has(noun.level)) {
+                                this.highLevelNodes.set(noun.level, new Set());
+                            }
+                            this.highLevelNodes.get(noun.level).add(nounData.id);
+                        }
+                        loadedCount++;
+                    }
+                    catch (error) {
+                        // Log error but continue (robust error recovery)
+                        console.error(`Failed to rebuild HNSW data for ${nounData.id}:`, error);
+                    }
+                }
+                // Report progress
+                if (options.onProgress && totalCount !== undefined) {
+                    options.onProgress(loadedCount, totalCount);
+                }
+                // Check for more data
+                hasMore = result.hasMore;
+                cursor = result.nextCursor;
+            }
+            console.log(`HNSW index rebuilt successfully: ${loadedCount} entities, ` +
+                `${this.maxLevel + 1} levels, entry point: ${this.entryPointId || 'none'}` +
+                (lazy ? ' (lazy mode - vectors loaded on-demand)' : ''));
+        }
+        catch (error) {
+            console.error('HNSW rebuild failed:', error);
+            throw new Error(`Failed to rebuild HNSW index: ${error}`);
+        }
+    }
     /**
      * Get level statistics for understanding the hierarchy
      */

package/dist/hnsw/hnswIndexOptimized.d.ts

@@ -5,7 +5,7 @@
  */
 import { DistanceFunction, HNSWConfig, Vector, VectorDocument } from '../coreTypes.js';
 import { HNSWIndex } from './hnswIndex.js';
-import { StorageAdapter } from '../coreTypes.js';
+import type { BaseStorage } from '../storage/baseStorage.js';
 export interface HNSWOptimizedConfig extends HNSWConfig {
     memoryThreshold?: number;
     productQuantization?: {
@@ -88,7 +88,6 @@ declare class ProductQuantizer {
 export declare class HNSWIndexOptimized extends HNSWIndex {
     private optimizedConfig;
     private productQuantizer;
-    private storage;
     private useDiskBasedIndex;
     private useProductQuantization;
     private quantizedVectors;
@@ -96,7 +95,7 @@ export declare class HNSWIndexOptimized extends HNSWIndex {
     private vectorCount;
     private memoryUpdateLock;
     private unifiedCache;
-    constructor(config: Partial<HNSWOptimizedConfig>, distanceFunction: DistanceFunction, storage?: StorageAdapter | null);
+    constructor(config: Partial<HNSWOptimizedConfig>, distanceFunction: DistanceFunction, storage?: BaseStorage | null);
     /**
      * Thread-safe method to update memory usage
      * @param memoryDelta Change in memory usage (can be negative)
@@ -145,16 +144,6 @@ export declare class HNSWIndexOptimized extends HNSWIndex {
      * @returns Estimated memory usage in bytes
      */
     getMemoryUsage(): number;
-    /**
-     * Set the storage adapter
-     * @param storage Storage adapter
-     */
-    setStorage(storage: StorageAdapter): void;
-    /**
-     * Get the storage adapter
-     * @returns Storage adapter or null if not set
-     */
-    getStorage(): StorageAdapter | null;
     /**
      * Set whether to use disk-based index
      * @param useDiskBasedIndex Whether to use disk-based index

package/dist/hnsw/hnswIndexOptimized.js

@@ -211,10 +211,9 @@ class ProductQuantizer {
  */
 export class HNSWIndexOptimized extends HNSWIndex {
     constructor(config = {}, distanceFunction, storage = null) {
-        // Initialize base HNSW index with standard config
-        super(config, distanceFunction);
+        // Initialize base HNSW index with standard config and storage
+        super(config, distanceFunction, { storage: storage || undefined });
         this.productQuantizer = null;
-        this.storage = null;
         this.useDiskBasedIndex = false;
         this.useProductQuantization = false;
         this.quantizedVectors = new Map();
@@ -224,8 +223,6 @@ export class HNSWIndexOptimized extends HNSWIndex {
         this.memoryUpdateLock = Promise.resolve();
         // Set optimized config
         this.optimizedConfig = { ...DEFAULT_OPTIMIZED_CONFIG, ...config };
-        // Set storage adapter
-        this.storage = storage;
         // Initialize product quantizer if enabled
         if (this.optimizedConfig.productQuantization?.enabled) {
             this.useProductQuantization = true;
@@ -302,18 +299,9 @@ export class HNSWIndexOptimized extends HNSWIndex {
             return await super.addItem({ id, vector: reconstructedVector });
         }
         // If disk-based index is active and storage is available, store the vector
-        if (this.useDiskBasedIndex && this.storage) {
-            // Create a noun object
-            const noun = {
-                id,
-                vector,
-                connections: new Map(),
-                level: 0
-            };
-            // Store the noun
-            this.storage.saveNoun(noun).catch((error) => {
-                console.error(`Failed to save noun ${id} to storage:`, error);
-            });
+        if (this.useDiskBasedIndex) {
+            // Storage is handled by the base class now via HNSW persistence
+            // No additional storage needed here
         }
         // Add the vector to the in-memory index
         return await super.addItem(item);
@@ -349,12 +337,8 @@ export class HNSWIndexOptimized extends HNSWIndex {
         if (this.useProductQuantization) {
             this.quantizedVectors.delete(id);
         }
-        // If disk-based index is active and storage is available, remove the vector from storage
-        if (this.useDiskBasedIndex && this.storage) {
-            this.storage.deleteNoun(id).catch((error) => {
-                console.error(`Failed to delete noun ${id} from storage:`, error);
-            });
-        }
+        // If disk-based index is active, removal is handled by base class
+        // No additional removal needed here
         // Update memory usage estimate (async operation, but don't block removal)
         this.getMemoryUsageAsync().then((currentMemoryUsage) => {
             if (currentMemoryUsage.vectorCount > 0) {
@@ -428,20 +412,7 @@ export class HNSWIndexOptimized extends HNSWIndex {
     getMemoryUsage() {
         return this.memoryUsage;
     }
-    /**
-     * Set the storage adapter
-     * @param storage Storage adapter
-     */
-    setStorage(storage) {
-        this.storage = storage;
-    }
-    /**
-     * Get the storage adapter
-     * @returns Storage adapter or null if not set
-     */
-    getStorage() {
-        return this.storage;
-    }
+    // Storage methods removed - now handled by base class
     /**
      * Set whether to use disk-based index
      * @param useDiskBasedIndex Whether to use disk-based index

package/dist/importers/SmartExcelImporter.js

@@ -37,6 +37,18 @@ export class SmartExcelImporter {
         const opts = {
             enableNeuralExtraction: true,
             enableRelationshipInference: true,
+            // CONCEPT EXTRACTION PRODUCTION-READY (v3.33.0+):
+            // Type embeddings are now pre-computed at build time - zero runtime cost!
+            // All 31 noun types + 40 verb types instantly available
+            //
+            // Performance profile:
+            // - Type embeddings: INSTANT (pre-computed at build time, ~100KB in-memory)
+            // - Model loading: ~2-5 seconds (one-time, cached after first use)
+            // - Per-row extraction: ~50-200ms depending on definition length
+            // - 100 rows: ~5-20 seconds total (production ready)
+            // - 1000 rows: ~50-200 seconds (disable if needed via enableConceptExtraction: false)
+            //
+            // Enabled by default for production use.
             enableConceptExtraction: true,
             confidenceThreshold: 0.6,
             termColumn: 'term|name|title|concept',

package/dist/interfaces/IIndex.d.ts

@@ -0,0 +1,186 @@
+/**
+ * Unified Index Interface (v3.35.0+)
+ *
+ * Standardizes index lifecycle across all index types in Brainy.
+ * All indexes (HNSW Vector, Graph Adjacency, Metadata Field) implement this interface
+ * for consistent rebuild, clear, and stats operations.
+ *
+ * This enables:
+ * - Parallel index rebuilds during initialization
+ * - Consistent index management across the system
+ * - Easy addition of new index types
+ * - Unified monitoring and health checks
+ */
+/**
+ * Index statistics returned by getStats()
+ */
+export interface IndexStats {
+    /**
+     * Total number of items in the index
+     */
+    totalItems: number;
+    /**
+     * Estimated memory usage in bytes (optional)
+     */
+    memoryUsage?: number;
+    /**
+     * Timestamp of last rebuild (optional)
+     */
+    lastRebuilt?: number;
+    /**
+     * Index-specific statistics (optional)
+     * - HNSW: { maxLevel, entryPointId, levels, avgDegree }
+     * - Graph: { totalRelationships, verbTypes }
+     * - Metadata: { totalFields, totalEntries }
+     */
+    specifics?: Record<string, any>;
+}
+/**
+ * Progress callback for rebuild operations
+ * Reports current progress and total count
+ */
+export type RebuildProgressCallback = (loaded: number, total: number) => void;
+/**
+ * Rebuild options for index rebuilding
+ */
+export interface RebuildOptions {
+    /**
+     * Lazy mode: Load structure only, data on-demand
+     * Saves memory at cost of first-access latency
+     * (HNSW: vectors loaded on-demand, Graph: relationships cached, Metadata: lazy field indexing)
+     */
+    lazy?: boolean;
+    /**
+     * Batch size for pagination during rebuild
+     * Default: 1000 (tune based on available memory)
+     */
+    batchSize?: number;
+    /**
+     * Progress callback for monitoring rebuild progress
+     * Called periodically with (loaded, total) counts
+     */
+    onProgress?: RebuildProgressCallback;
+    /**
+     * Force rebuild even if index appears populated
+     * Useful for repairing corrupted indexes
+     */
+    force?: boolean;
+}
+/**
+ * Unified Index Interface
+ *
+ * All indexes in Brainy implement this interface for consistent lifecycle management.
+ * This enables parallel rebuilds, unified monitoring, and standardized operations.
+ */
+export interface IIndex {
+    /**
+     * Rebuild index from persisted storage
+     *
+     * Called during Brainy initialization when:
+     * - Container restarts and in-memory indexes are empty
+     * - Storage has persisted data but indexes need rebuilding
+     * - Force rebuild is requested
+     *
+     * Implementation must:
+     * - Clear existing in-memory state
+     * - Load data from storage using pagination
+     * - Restore index structure efficiently (O(N) preferred over O(N log N))
+     * - Handle millions of entities via batching
+     * - Support lazy loading for memory-constrained environments
+     * - Provide progress reporting for large datasets
+     * - Recover gracefully from partial failures
+     *
+     * @param options Rebuild options (lazy mode, batch size, progress callback, force)
+     * @returns Promise that resolves when rebuild is complete
+     * @throws Error if rebuild fails critically (should log warnings for partial failures)
+     */
+    rebuild(options?: RebuildOptions): Promise<void>;
+    /**
+     * Clear all in-memory index data
+     *
+     * Called when:
+     * - User explicitly calls brain.clear()
+     * - System needs to reset without rebuilding
+     * - Tests need clean state
+     *
+     * Implementation must:
+     * - Clear all in-memory data structures
+     * - Reset counters and statistics
+     * - NOT delete persisted storage data
+     * - Be idempotent (safe to call multiple times)
+     *
+     * Note: This is a memory-only operation. To delete persisted data,
+     * use storage.clear() instead.
+     */
+    clear(): void;
+    /**
+     * Get current index statistics
+     *
+     * Returns real-time statistics about the index state:
+     * - Total items indexed
+     * - Memory usage (if available)
+     * - Last rebuild timestamp
+     * - Index-specific metrics
+     *
+     * Used for:
+     * - Health monitoring
+     * - Determining if rebuild is needed
+     * - Performance analysis
+     * - Debugging
+     *
+     * @returns Promise that resolves to index statistics
+     */
+    getStats(): Promise<IndexStats>;
+    /**
+     * Get the current size of the index
+     *
+     * Fast O(1) operation returning the number of items in the index.
+     * Used for quick health checks and deciding rebuild strategy.
+     *
+     * @returns Number of items in the index
+     */
+    size(): number;
+}
+/**
+ * Extended index interface with cache support (optional)
+ *
+ * Indexes can optionally implement cache integration for:
+ * - Hot/warm/cold tier management
+ * - Memory-efficient lazy loading
+ * - Adaptive caching based on access patterns
+ */
+export interface ICachedIndex extends IIndex {
+    /**
+     * Set cache for resource management
+     *
+     * Enables the index to use UnifiedCache for:
+     * - Lazy loading of vectors/data
+     * - Hot/warm/cold tier management
+     * - Memory pressure handling
+     *
+     * @param cache UnifiedCache instance
+     */
+    setCache?(cache: any): void;
+}
+/**
+ * Extended index interface with persistence support (optional)
+ *
+ * Indexes can optionally implement explicit persistence:
+ * - Manual triggering of data saves
+ * - Batch write optimization
+ * - Checkpoint creation
+ */
+export interface IPersistentIndex extends IIndex {
+    /**
+     * Manually persist current index state to storage
+     *
+     * Most indexes auto-persist during operations (e.g., HNSW persists on addItem).
+     * This method allows explicit persistence for:
+     * - Checkpointing before risky operations
+     * - Forced flush before shutdown
+     * - Manual backup creation
+     *
+     * @returns Promise that resolves when persistence is complete
+     */
+    persist?(): Promise<void>;
+}

package/dist/interfaces/IIndex.js

@@ -0,0 +1,15 @@
+/**
+ * Unified Index Interface (v3.35.0+)
+ *
+ * Standardizes index lifecycle across all index types in Brainy.
+ * All indexes (HNSW Vector, Graph Adjacency, Metadata Field) implement this interface
+ * for consistent rebuild, clear, and stats operations.
+ *
+ * This enables:
+ * - Parallel index rebuilds during initialization
+ * - Consistent index management across the system
+ * - Easy addition of new index types
+ * - Unified monitoring and health checks
+ */
+export {};
+//# sourceMappingURL=IIndex.js.map

package/dist/neural/embeddedTypeEmbeddings.d.ts

@@ -0,0 +1,34 @@
+/**
+ * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS
+ *
+ * AUTO-GENERATED - DO NOT EDIT
+ * Generated: 2025-10-10T01:27:22.642Z
+ * Noun Types: 31
+ * Verb Types: 40
+ *
+ * This file contains pre-computed embeddings for all NounTypes and VerbTypes.
+ * No runtime computation needed, instant availability!
+ */
+import { NounType, VerbType } from '../types/graphTypes.js';
+import { Vector } from '../coreTypes.js';
+export declare const TYPE_METADATA: {
+    nounTypes: number;
+    verbTypes: number;
+    totalTypes: number;
+    embeddingDimensions: number;
+    generatedAt: string;
+    sizeBytes: {
+        embeddings: number;
+        base64: number;
+    };
+};
+/**
+ * Get noun type embeddings as a Map for fast lookup
+ * This is called once and cached
+ */
+export declare function getNounTypeEmbeddings(): Map<NounType, Vector>;
+/**
+ * Get verb type embeddings as a Map for fast lookup
+ * This is called once and cached
+ */
+export declare function getVerbTypeEmbeddings(): Map<VerbType, Vector>;