@soulcraft/brainy 3.34.0 → 3.36.0

package/dist/hnsw/hnswIndexOptimized.js

@@ -4,7 +4,6 @@
  * Uses product quantization for dimensionality reduction and disk-based storage when needed
  */
 import { HNSWIndex } from './hnswIndex.js';
-import { getGlobalCache } from '../utils/unifiedCache.js';
 // Default configuration for the optimized HNSW index
 const DEFAULT_OPTIMIZED_CONFIG = {
     M: 16,
@@ -211,10 +210,9 @@ class ProductQuantizer {
  */
 export class HNSWIndexOptimized extends HNSWIndex {
     constructor(config = {}, distanceFunction, storage = null) {
-        // Initialize base HNSW index with standard config
-        super(config, distanceFunction);
+        // Initialize base HNSW index with standard config and storage
+        super(config, distanceFunction, { storage: storage || undefined });
         this.productQuantizer = null;
-        this.storage = null;
         this.useDiskBasedIndex = false;
         this.useProductQuantization = false;
         this.quantizedVectors = new Map();
@@ -224,8 +222,6 @@ export class HNSWIndexOptimized extends HNSWIndex {
         this.memoryUpdateLock = Promise.resolve();
         // Set optimized config
         this.optimizedConfig = { ...DEFAULT_OPTIMIZED_CONFIG, ...config };
-        // Set storage adapter
-        this.storage = storage;
         // Initialize product quantizer if enabled
         if (this.optimizedConfig.productQuantization?.enabled) {
             this.useProductQuantization = true;
@@ -233,8 +229,7 @@ export class HNSWIndexOptimized extends HNSWIndex {
         }
         // Set disk-based index flag
         this.useDiskBasedIndex = this.optimizedConfig.useDiskBasedIndex || false;
-        // Get global unified cache for coordinated memory management
-        this.unifiedCache = getGlobalCache();
+        // Note: UnifiedCache is inherited from base HNSWIndex class
     }
     /**
      * Thread-safe method to update memory usage
@@ -302,18 +297,9 @@ export class HNSWIndexOptimized extends HNSWIndex {
             return await super.addItem({ id, vector: reconstructedVector });
         }
         // If disk-based index is active and storage is available, store the vector
-        if (this.useDiskBasedIndex && this.storage) {
-            // Create a noun object
-            const noun = {
-                id,
-                vector,
-                connections: new Map(),
-                level: 0
-            };
-            // Store the noun
-            this.storage.saveNoun(noun).catch((error) => {
-                console.error(`Failed to save noun ${id} to storage:`, error);
-            });
+        if (this.useDiskBasedIndex) {
+            // Storage is handled by the base class now via HNSW persistence
+            // No additional storage needed here
         }
         // Add the vector to the in-memory index
         return await super.addItem(item);
@@ -344,17 +330,13 @@ export class HNSWIndexOptimized extends HNSWIndex {
     /**
      * Remove an item from the index
      */
-    removeItem(id) {
+    async removeItem(id) {
         // If product quantization is active, remove the quantized vector
         if (this.useProductQuantization) {
             this.quantizedVectors.delete(id);
         }
-        // If disk-based index is active and storage is available, remove the vector from storage
-        if (this.useDiskBasedIndex && this.storage) {
-            this.storage.deleteNoun(id).catch((error) => {
-                console.error(`Failed to delete noun ${id} from storage:`, error);
-            });
-        }
+        // If disk-based index is active, removal is handled by base class
+        // No additional removal needed here
         // Update memory usage estimate (async operation, but don't block removal)
         this.getMemoryUsageAsync().then((currentMemoryUsage) => {
             if (currentMemoryUsage.vectorCount > 0) {
@@ -365,7 +347,7 @@ export class HNSWIndexOptimized extends HNSWIndex {
             console.error('Failed to update memory usage after removal:', error);
         });
         // Remove the item from the in-memory index
-        return super.removeItem(id);
+        return await super.removeItem(id);
     }
     /**
      * Clear the index
@@ -428,20 +410,7 @@ export class HNSWIndexOptimized extends HNSWIndex {
     getMemoryUsage() {
         return this.memoryUsage;
     }
-    /**
-     * Set the storage adapter
-     * @param storage Storage adapter
-     */
-    setStorage(storage) {
-        this.storage = storage;
-    }
-    /**
-     * Get the storage adapter
-     * @returns Storage adapter or null if not set
-     */
-    getStorage() {
-        return this.storage;
-    }
+    // Storage methods removed - now handled by base class
     /**
      * Set whether to use disk-based index
      * @param useDiskBasedIndex Whether to use disk-based index

package/dist/hnsw/partitionedHNSWIndex.js

@@ -274,7 +274,7 @@ export class PartitionedHNSWIndex {
     async removeItem(id) {
         // Find which partition contains this item
         for (const [partitionId, partition] of this.partitions.entries()) {
-            if (partition.removeItem(id)) {
+            if (await partition.removeItem(id)) {
                 // Update metadata
                 const metadata = this.partitionMetadata.get(partitionId);
                 metadata.nodeCount = partition.size();

package/dist/interfaces/IIndex.d.ts

@@ -0,0 +1,195 @@
+/**
+ * Unified Index Interface (v3.35.0+)
+ *
+ * Standardizes index lifecycle across all index types in Brainy.
+ * All indexes (HNSW Vector, Graph Adjacency, Metadata Field) implement this interface
+ * for consistent rebuild, clear, and stats operations.
+ *
+ * This enables:
+ * - Parallel index rebuilds during initialization
+ * - Consistent index management across the system
+ * - Easy addition of new index types
+ * - Unified monitoring and health checks
+ */
+/**
+ * Index statistics returned by getStats()
+ */
+export interface IndexStats {
+    /**
+     * Total number of items in the index
+     */
+    totalItems: number;
+    /**
+     * Estimated memory usage in bytes (optional)
+     */
+    memoryUsage?: number;
+    /**
+     * Timestamp of last rebuild (optional)
+     */
+    lastRebuilt?: number;
+    /**
+     * Index-specific statistics (optional)
+     * - HNSW: { maxLevel, entryPointId, levels, avgDegree }
+     * - Graph: { totalRelationships, verbTypes }
+     * - Metadata: { totalFields, totalEntries }
+     */
+    specifics?: Record<string, any>;
+}
+/**
+ * Progress callback for rebuild operations
+ * Reports current progress and total count
+ */
+export type RebuildProgressCallback = (loaded: number, total: number) => void;
+/**
+ * Rebuild options for index rebuilding
+ */
+export interface RebuildOptions {
+    /**
+     * @deprecated Lazy mode is now auto-detected based on available memory.
+     * System automatically chooses between:
+     * - Preloading: Small datasets that fit comfortably in cache (< 80% threshold)
+     * - On-demand: Large datasets loaded adaptively via UnifiedCache
+     *
+     * This option is kept for backwards compatibility but is ignored.
+     * The system always uses adaptive caching (v3.36.0+).
+     */
+    lazy?: boolean;
+    /**
+     * Batch size for pagination during rebuild
+     * Default: 1000 (tune based on available memory)
+     */
+    batchSize?: number;
+    /**
+     * Progress callback for monitoring rebuild progress
+     * Called periodically with (loaded, total) counts
+     */
+    onProgress?: RebuildProgressCallback;
+    /**
+     * Force rebuild even if index appears populated
+     * Useful for repairing corrupted indexes
+     */
+    force?: boolean;
+}
+/**
+ * Unified Index Interface
+ *
+ * All indexes in Brainy implement this interface for consistent lifecycle management.
+ * This enables parallel rebuilds, unified monitoring, and standardized operations.
+ */
+export interface IIndex {
+    /**
+     * Rebuild index from persisted storage
+     *
+     * Called during Brainy initialization when:
+     * - Container restarts and in-memory indexes are empty
+     * - Storage has persisted data but indexes need rebuilding
+     * - Force rebuild is requested
+     *
+     * Implementation must:
+     * - Clear existing in-memory state
+     * - Load data from storage using pagination
+     * - Restore index structure efficiently (O(N) preferred over O(N log N))
+     * - Handle millions of entities via batching
+     * - Auto-detect caching strategy based on dataset size vs available memory
+     * - Provide progress reporting for large datasets
+     * - Recover gracefully from partial failures
+     *
+     * Adaptive Caching (v3.36.0+):
+     * System automatically chooses optimal strategy:
+     * - Small datasets: Preload all data at init for zero-latency access
+     * - Large datasets: Load on-demand via UnifiedCache for memory efficiency
+     *
+     * @param options Rebuild options (batch size, progress callback, force)
+     * @returns Promise that resolves when rebuild is complete
+     * @throws Error if rebuild fails critically (should log warnings for partial failures)
+     */
+    rebuild(options?: RebuildOptions): Promise<void>;
+    /**
+     * Clear all in-memory index data
+     *
+     * Called when:
+     * - User explicitly calls brain.clear()
+     * - System needs to reset without rebuilding
+     * - Tests need clean state
+     *
+     * Implementation must:
+     * - Clear all in-memory data structures
+     * - Reset counters and statistics
+     * - NOT delete persisted storage data
+     * - Be idempotent (safe to call multiple times)
+     *
+     * Note: This is a memory-only operation. To delete persisted data,
+     * use storage.clear() instead.
+     */
+    clear(): void;
+    /**
+     * Get current index statistics
+     *
+     * Returns real-time statistics about the index state:
+     * - Total items indexed
+     * - Memory usage (if available)
+     * - Last rebuild timestamp
+     * - Index-specific metrics
+     *
+     * Used for:
+     * - Health monitoring
+     * - Determining if rebuild is needed
+     * - Performance analysis
+     * - Debugging
+     *
+     * @returns Promise that resolves to index statistics
+     */
+    getStats(): Promise<IndexStats>;
+    /**
+     * Get the current size of the index
+     *
+     * Fast O(1) operation returning the number of items in the index.
+     * Used for quick health checks and deciding rebuild strategy.
+     *
+     * @returns Number of items in the index
+     */
+    size(): number;
+}
+/**
+ * Extended index interface with cache support (optional)
+ *
+ * Indexes can optionally implement cache integration for:
+ * - Hot/warm/cold tier management
+ * - Memory-efficient lazy loading
+ * - Adaptive caching based on access patterns
+ */
+export interface ICachedIndex extends IIndex {
+    /**
+     * Set cache for resource management
+     *
+     * Enables the index to use UnifiedCache for:
+     * - Lazy loading of vectors/data
+     * - Hot/warm/cold tier management
+     * - Memory pressure handling
+     *
+     * @param cache UnifiedCache instance
+     */
+    setCache?(cache: any): void;
+}
+/**
+ * Extended index interface with persistence support (optional)
+ *
+ * Indexes can optionally implement explicit persistence:
+ * - Manual triggering of data saves
+ * - Batch write optimization
+ * - Checkpoint creation
+ */
+export interface IPersistentIndex extends IIndex {
+    /**
+     * Manually persist current index state to storage
+     *
+     * Most indexes auto-persist during operations (e.g., HNSW persists on addItem).
+     * This method allows explicit persistence for:
+     * - Checkpointing before risky operations
+     * - Forced flush before shutdown
+     * - Manual backup creation
+     *
+     * @returns Promise that resolves when persistence is complete
+     */
+    persist?(): Promise<void>;
+}

package/dist/interfaces/IIndex.js

@@ -0,0 +1,15 @@
+/**
+ * Unified Index Interface (v3.35.0+)
+ *
+ * Standardizes index lifecycle across all index types in Brainy.
+ * All indexes (HNSW Vector, Graph Adjacency, Metadata Field) implement this interface
+ * for consistent rebuild, clear, and stats operations.
+ *
+ * This enables:
+ * - Parallel index rebuilds during initialization
+ * - Consistent index management across the system
+ * - Easy addition of new index types
+ * - Unified monitoring and health checks
+ */
+export {};
+//# sourceMappingURL=IIndex.js.map

package/dist/storage/adapters/baseStorageAdapter.d.ts

@@ -23,6 +23,23 @@ export declare abstract class BaseStorageAdapter implements StorageAdapter {
     abstract getNounMetadata(id: string): Promise<any | null>;
     abstract saveVerbMetadata(id: string, metadata: any): Promise<void>;
     abstract getVerbMetadata(id: string): Promise<any | null>;
+    abstract getNounVector(id: string): Promise<number[] | null>;
+    abstract saveHNSWData(nounId: string, hnswData: {
+        level: number;
+        connections: Record<string, string[]>;
+    }): Promise<void>;
+    abstract getHNSWData(nounId: string): Promise<{
+        level: number;
+        connections: Record<string, string[]>;
+    } | null>;
+    abstract saveHNSWSystem(systemData: {
+        entryPointId: string | null;
+        maxLevel: number;
+    }): Promise<void>;
+    abstract getHNSWSystem(): Promise<{
+        entryPointId: string | null;
+        maxLevel: number;
+    } | null>;
     abstract clear(): Promise<void>;
     abstract getStorageStatus(): Promise<{
         type: string;

package/dist/storage/adapters/fileSystemStorage.d.ts

@@ -361,5 +361,37 @@ export declare class FileSystemStorage extends BaseStorage {
      * Check if a file exists (handles both sharded and non-sharded)
      */
     private fileExists;
+    /**
+     * Get vector for a noun
+     */
+    getNounVector(id: string): Promise<number[] | null>;
+    /**
+     * Save HNSW graph data for a noun
+     */
+    saveHNSWData(nounId: string, hnswData: {
+        level: number;
+        connections: Record<string, string[]>;
+    }): Promise<void>;
+    /**
+     * Get HNSW graph data for a noun
+     */
+    getHNSWData(nounId: string): Promise<{
+        level: number;
+        connections: Record<string, string[]>;
+    } | null>;
+    /**
+     * Save HNSW system data (entry point, max level)
+     */
+    saveHNSWSystem(systemData: {
+        entryPointId: string | null;
+        maxLevel: number;
+    }): Promise<void>;
+    /**
+     * Get HNSW system data
+     */
+    getHNSWSystem(): Promise<{
+        entryPointId: string | null;
+        maxLevel: number;
+    } | null>;
 }
 export {};

package/dist/storage/adapters/fileSystemStorage.js

@@ -2108,5 +2108,71 @@ export class FileSystemStorage extends BaseStorage {
             return false;
         }
     }
+    // =============================================
+    // HNSW Index Persistence (v3.35.0+)
+    // =============================================
+    /**
+     * Get vector for a noun
+     */
+    async getNounVector(id) {
+        await this.ensureInitialized();
+        const noun = await this.getNode(id);
+        return noun ? noun.vector : null;
+    }
+    /**
+     * Save HNSW graph data for a noun
+     */
+    async saveHNSWData(nounId, hnswData) {
+        await this.ensureInitialized();
+        // Use sharded path for HNSW data
+        const shard = nounId.substring(0, 2).toLowerCase();
+        const hnswDir = path.join(this.rootDir, 'entities', 'nouns', 'hnsw', shard);
+        await this.ensureDirectoryExists(hnswDir);
+        const filePath = path.join(hnswDir, `${nounId}.json`);
+        await fs.promises.writeFile(filePath, JSON.stringify(hnswData, null, 2));
+    }
+    /**
+     * Get HNSW graph data for a noun
+     */
+    async getHNSWData(nounId) {
+        await this.ensureInitialized();
+        const shard = nounId.substring(0, 2).toLowerCase();
+        const filePath = path.join(this.rootDir, 'entities', 'nouns', 'hnsw', shard, `${nounId}.json`);
+        try {
+            const data = await fs.promises.readFile(filePath, 'utf-8');
+            return JSON.parse(data);
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT') {
+                console.error(`Error reading HNSW data for ${nounId}:`, error);
+            }
+            return null;
+        }
+    }
+    /**
+     * Save HNSW system data (entry point, max level)
+     */
+    async saveHNSWSystem(systemData) {
+        await this.ensureInitialized();
+        const filePath = path.join(this.systemDir, 'hnsw-system.json');
+        await fs.promises.writeFile(filePath, JSON.stringify(systemData, null, 2));
+    }
+    /**
+     * Get HNSW system data
+     */
+    async getHNSWSystem() {
+        await this.ensureInitialized();
+        const filePath = path.join(this.systemDir, 'hnsw-system.json');
+        try {
+            const data = await fs.promises.readFile(filePath, 'utf-8');
+            return JSON.parse(data);
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT') {
+                console.error('Error reading HNSW system data:', error);
+            }
+            return null;
+        }
+    }
 }
 //# sourceMappingURL=fileSystemStorage.js.map

package/dist/storage/adapters/gcsStorage.d.ts

@@ -339,5 +339,41 @@ export declare class GcsStorage extends BaseStorage {
      * Persist counts to storage
      */
     protected persistCounts(): Promise<void>;
+    /**
+     * Get a noun's vector for HNSW rebuild
+     */
+    getNounVector(id: string): Promise<number[] | null>;
+    /**
+     * Save HNSW graph data for a noun
+     * Storage path: entities/nouns/hnsw/{shard}/{id}.json
+     */
+    saveHNSWData(nounId: string, hnswData: {
+        level: number;
+        connections: Record<string, string[]>;
+    }): Promise<void>;
+    /**
+     * Get HNSW graph data for a noun
+     * Storage path: entities/nouns/hnsw/{shard}/{id}.json
+     */
+    getHNSWData(nounId: string): Promise<{
+        level: number;
+        connections: Record<string, string[]>;
+    } | null>;
+    /**
+     * Save HNSW system data (entry point, max level)
+     * Storage path: system/hnsw-system.json
+     */
+    saveHNSWSystem(systemData: {
+        entryPointId: string | null;
+        maxLevel: number;
+    }): Promise<void>;
+    /**
+     * Get HNSW system data (entry point, max level)
+     * Storage path: system/hnsw-system.json
+     */
+    getHNSWSystem(): Promise<{
+        entryPointId: string | null;
+        maxLevel: number;
+    } | null>;
 }
 export {};

package/dist/storage/adapters/gcsStorage.js

@@ -1195,5 +1195,95 @@ export class GcsStorage extends BaseStorage {
             this.logger.error('Error persisting counts:', error);
         }
     }
+    // HNSW Index Persistence (v3.35.0+)
+    /**
+     * Get a noun's vector for HNSW rebuild
+     */
+    async getNounVector(id) {
+        await this.ensureInitialized();
+        const noun = await this.getNode(id);
+        return noun ? noun.vector : null;
+    }
+    /**
+     * Save HNSW graph data for a noun
+     * Storage path: entities/nouns/hnsw/{shard}/{id}.json
+     */
+    async saveHNSWData(nounId, hnswData) {
+        await this.ensureInitialized();
+        try {
+            // Use sharded path for HNSW data
+            const shard = getShardIdFromUuid(nounId);
+            const key = `entities/nouns/hnsw/${shard}/${nounId}.json`;
+            const file = this.bucket.file(key);
+            await file.save(JSON.stringify(hnswData, null, 2), {
+                contentType: 'application/json',
+                resumable: false
+            });
+        }
+        catch (error) {
+            this.logger.error(`Failed to save HNSW data for ${nounId}:`, error);
+            throw new Error(`Failed to save HNSW data for ${nounId}: ${error}`);
+        }
+    }
+    /**
+     * Get HNSW graph data for a noun
+     * Storage path: entities/nouns/hnsw/{shard}/{id}.json
+     */
+    async getHNSWData(nounId) {
+        await this.ensureInitialized();
+        try {
+            const shard = getShardIdFromUuid(nounId);
+            const key = `entities/nouns/hnsw/${shard}/${nounId}.json`;
+            const file = this.bucket.file(key);
+            const [contents] = await file.download();
+            return JSON.parse(contents.toString());
+        }
+        catch (error) {
+            if (error.code === 404) {
+                return null;
+            }
+            this.logger.error(`Failed to get HNSW data for ${nounId}:`, error);
+            throw new Error(`Failed to get HNSW data for ${nounId}: ${error}`);
+        }
+    }
+    /**
+     * Save HNSW system data (entry point, max level)
+     * Storage path: system/hnsw-system.json
+     */
+    async saveHNSWSystem(systemData) {
+        await this.ensureInitialized();
+        try {
+            const key = `${this.systemPrefix}hnsw-system.json`;
+            const file = this.bucket.file(key);
+            await file.save(JSON.stringify(systemData, null, 2), {
+                contentType: 'application/json',
+                resumable: false
+            });
+        }
+        catch (error) {
+            this.logger.error('Failed to save HNSW system data:', error);
+            throw new Error(`Failed to save HNSW system data: ${error}`);
+        }
+    }
+    /**
+     * Get HNSW system data (entry point, max level)
+     * Storage path: system/hnsw-system.json
+     */
+    async getHNSWSystem() {
+        await this.ensureInitialized();
+        try {
+            const key = `${this.systemPrefix}hnsw-system.json`;
+            const file = this.bucket.file(key);
+            const [contents] = await file.download();
+            return JSON.parse(contents.toString());
+        }
+        catch (error) {
+            if (error.code === 404) {
+                return null;
+            }
+            this.logger.error('Failed to get HNSW system data:', error);
+            throw new Error(`Failed to get HNSW system data: ${error}`);
+        }
+    }
 }
 //# sourceMappingURL=gcsStorage.js.map

package/dist/storage/adapters/memoryStorage.d.ts

@@ -174,4 +174,36 @@ export declare class MemoryStorage extends BaseStorage {
      * Persist counts to storage - no-op for memory storage
      */
     protected persistCounts(): Promise<void>;
+    /**
+     * Get vector for a noun
+     */
+    getNounVector(id: string): Promise<number[] | null>;
+    /**
+     * Save HNSW graph data for a noun
+     */
+    saveHNSWData(nounId: string, hnswData: {
+        level: number;
+        connections: Record<string, string[]>;
+    }): Promise<void>;
+    /**
+     * Get HNSW graph data for a noun
+     */
+    getHNSWData(nounId: string): Promise<{
+        level: number;
+        connections: Record<string, string[]>;
+    } | null>;
+    /**
+     * Save HNSW system data (entry point, max level)
+     */
+    saveHNSWSystem(systemData: {
+        entryPointId: string | null;
+        maxLevel: number;
+    }): Promise<void>;
+    /**
+     * Get HNSW system data
+     */
+    getHNSWSystem(): Promise<{
+        entryPointId: string | null;
+        maxLevel: number;
+    } | null>;
 }

package/dist/storage/adapters/memoryStorage.js

@@ -595,5 +595,48 @@ export class MemoryStorage extends BaseStorage {
         // No persistence needed for in-memory storage
         // Counts are always accurate from the live data structures
     }
+    // =============================================
+    // HNSW Index Persistence (v3.35.0+)
+    // =============================================
+    /**
+     * Get vector for a noun
+     */
+    async getNounVector(id) {
+        const noun = this.nouns.get(id);
+        return noun ? [...noun.vector] : null;
+    }
+    /**
+     * Save HNSW graph data for a noun
+     */
+    async saveHNSWData(nounId, hnswData) {
+        // For memory storage, HNSW data is already in the noun object
+        // This method is a no-op since saveNoun already stores the full graph
+        // But we store it separately for consistency with other adapters
+        const path = `hnsw/${nounId}.json`;
+        await this.writeObjectToPath(path, hnswData);
+    }
+    /**
+     * Get HNSW graph data for a noun
+     */
+    async getHNSWData(nounId) {
+        const path = `hnsw/${nounId}.json`;
+        const data = await this.readObjectFromPath(path);
+        return data || null;
+    }
+    /**
+     * Save HNSW system data (entry point, max level)
+     */
+    async saveHNSWSystem(systemData) {
+        const path = 'system/hnsw-system.json';
+        await this.writeObjectToPath(path, systemData);
+    }
+    /**
+     * Get HNSW system data
+     */
+    async getHNSWSystem() {
+        const path = 'system/hnsw-system.json';
+        const data = await this.readObjectFromPath(path);
+        return data || null;
+    }
 }
 //# sourceMappingURL=memoryStorage.js.map