@soulcraft/brainy 3.34.0 → 3.36.0

package/dist/storage/adapters/opfsStorage.d.ts

@@ -262,5 +262,41 @@ export declare class OPFSStorage extends BaseStorage {
      * Persist counts to OPFS storage
      */
     protected persistCounts(): Promise<void>;
+    /**
+     * Get a noun's vector for HNSW rebuild
+     */
+    getNounVector(id: string): Promise<number[] | null>;
+    /**
+     * Save HNSW graph data for a noun
+     * Storage path: nouns/hnsw/{shard}/{id}.json
+     */
+    saveHNSWData(nounId: string, hnswData: {
+        level: number;
+        connections: Record<string, string[]>;
+    }): Promise<void>;
+    /**
+     * Get HNSW graph data for a noun
+     * Storage path: nouns/hnsw/{shard}/{id}.json
+     */
+    getHNSWData(nounId: string): Promise<{
+        level: number;
+        connections: Record<string, string[]>;
+    } | null>;
+    /**
+     * Save HNSW system data (entry point, max level)
+     * Storage path: index/hnsw-system.json
+     */
+    saveHNSWSystem(systemData: {
+        entryPointId: string | null;
+        maxLevel: number;
+    }): Promise<void>;
+    /**
+     * Get HNSW system data (entry point, max level)
+     * Storage path: index/hnsw-system.json
+     */
+    getHNSWSystem(): Promise<{
+        entryPointId: string | null;
+        maxLevel: number;
+    } | null>;
 }
 export {};

package/dist/storage/adapters/opfsStorage.js

@@ -1471,5 +1471,106 @@ export class OPFSStorage extends BaseStorage {
             console.error('Error persisting counts to OPFS:', error);
         }
     }
+    // HNSW Index Persistence (v3.35.0+)
+    /**
+     * Get a noun's vector for HNSW rebuild
+     */
+    async getNounVector(id) {
+        await this.ensureInitialized();
+        const noun = await this.getNoun_internal(id);
+        return noun ? noun.vector : null;
+    }
+    /**
+     * Save HNSW graph data for a noun
+     * Storage path: nouns/hnsw/{shard}/{id}.json
+     */
+    async saveHNSWData(nounId, hnswData) {
+        await this.ensureInitialized();
+        try {
+            // Get or create the hnsw directory under nouns
+            const hnswDir = await this.nounsDir.getDirectoryHandle('hnsw', { create: true });
+            // Use sharded path for HNSW data
+            const shard = getShardIdFromUuid(nounId);
+            const shardDir = await hnswDir.getDirectoryHandle(shard, { create: true });
+            // Create or get the file in the shard directory
+            const fileHandle = await shardDir.getFileHandle(`${nounId}.json`, { create: true });
+            // Write the HNSW data to the file
+            const writable = await fileHandle.createWritable();
+            await writable.write(JSON.stringify(hnswData, null, 2));
+            await writable.close();
+        }
+        catch (error) {
+            console.error(`Failed to save HNSW data for ${nounId}:`, error);
+            throw new Error(`Failed to save HNSW data for ${nounId}: ${error}`);
+        }
+    }
+    /**
+     * Get HNSW graph data for a noun
+     * Storage path: nouns/hnsw/{shard}/{id}.json
+     */
+    async getHNSWData(nounId) {
+        await this.ensureInitialized();
+        try {
+            // Get the hnsw directory under nouns
+            const hnswDir = await this.nounsDir.getDirectoryHandle('hnsw');
+            // Use sharded path for HNSW data
+            const shard = getShardIdFromUuid(nounId);
+            const shardDir = await hnswDir.getDirectoryHandle(shard);
+            // Get the file handle from the shard directory
+            const fileHandle = await shardDir.getFileHandle(`${nounId}.json`);
+            // Read the HNSW data from the file
+            const file = await fileHandle.getFile();
+            const text = await file.text();
+            return JSON.parse(text);
+        }
+        catch (error) {
+            if (error.name === 'NotFoundError') {
+                return null;
+            }
+            console.error(`Failed to get HNSW data for ${nounId}:`, error);
+            throw new Error(`Failed to get HNSW data for ${nounId}: ${error}`);
+        }
+    }
+    /**
+     * Save HNSW system data (entry point, max level)
+     * Storage path: index/hnsw-system.json
+     */
+    async saveHNSWSystem(systemData) {
+        await this.ensureInitialized();
+        try {
+            // Create or get the file in the index directory
+            const fileHandle = await this.indexDir.getFileHandle('hnsw-system.json', { create: true });
+            // Write the system data to the file
+            const writable = await fileHandle.createWritable();
+            await writable.write(JSON.stringify(systemData, null, 2));
+            await writable.close();
+        }
+        catch (error) {
+            console.error('Failed to save HNSW system data:', error);
+            throw new Error(`Failed to save HNSW system data: ${error}`);
+        }
+    }
+    /**
+     * Get HNSW system data (entry point, max level)
+     * Storage path: index/hnsw-system.json
+     */
+    async getHNSWSystem() {
+        await this.ensureInitialized();
+        try {
+            // Get the file handle from the index directory
+            const fileHandle = await this.indexDir.getFileHandle('hnsw-system.json');
+            // Read the system data from the file
+            const file = await fileHandle.getFile();
+            const text = await file.text();
+            return JSON.parse(text);
+        }
+        catch (error) {
+            if (error.name === 'NotFoundError') {
+                return null;
+            }
+            console.error('Failed to get HNSW system data:', error);
+            throw new Error(`Failed to get HNSW system data: ${error}`);
+        }
+    }
 }
 //# sourceMappingURL=opfsStorage.js.map

package/dist/storage/adapters/s3CompatibleStorage.d.ts

@@ -572,4 +572,40 @@ export declare class S3CompatibleStorage extends BaseStorage {
      * @returns true (S3 is cloud storage)
      */
     protected isCloudStorage(): boolean;
+    /**
+     * Get a noun's vector for HNSW rebuild
+     */
+    getNounVector(id: string): Promise<number[] | null>;
+    /**
+     * Save HNSW graph data for a noun
+     * Storage path: entities/nouns/hnsw/{shard}/{id}.json
+     */
+    saveHNSWData(nounId: string, hnswData: {
+        level: number;
+        connections: Record<string, string[]>;
+    }): Promise<void>;
+    /**
+     * Get HNSW graph data for a noun
+     * Storage path: entities/nouns/hnsw/{shard}/{id}.json
+     */
+    getHNSWData(nounId: string): Promise<{
+        level: number;
+        connections: Record<string, string[]>;
+    } | null>;
+    /**
+     * Save HNSW system data (entry point, max level)
+     * Storage path: system/hnsw-system.json
+     */
+    saveHNSWSystem(systemData: {
+        entryPointId: string | null;
+        maxLevel: number;
+    }): Promise<void>;
+    /**
+     * Get HNSW system data (entry point, max level)
+     * Storage path: system/hnsw-system.json
+     */
+    getHNSWSystem(): Promise<{
+        entryPointId: string | null;
+        maxLevel: number;
+    } | null>;
 }

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -2765,5 +2765,117 @@ export class S3CompatibleStorage extends BaseStorage {
     isCloudStorage() {
         return true; // S3 benefits from batching
     }
+    // HNSW Index Persistence (v3.35.0+)
+    /**
+     * Get a noun's vector for HNSW rebuild
+     */
+    async getNounVector(id) {
+        await this.ensureInitialized();
+        const noun = await this.getNode(id);
+        return noun ? noun.vector : null;
+    }
+    /**
+     * Save HNSW graph data for a noun
+     * Storage path: entities/nouns/hnsw/{shard}/{id}.json
+     */
+    async saveHNSWData(nounId, hnswData) {
+        await this.ensureInitialized();
+        try {
+            const { PutObjectCommand } = await import('@aws-sdk/client-s3');
+            // Use sharded path for HNSW data
+            const shard = getShardIdFromUuid(nounId);
+            const key = `entities/nouns/hnsw/${shard}/${nounId}.json`;
+            await this.s3Client.send(new PutObjectCommand({
+                Bucket: this.bucketName,
+                Key: key,
+                Body: JSON.stringify(hnswData, null, 2),
+                ContentType: 'application/json'
+            }));
+        }
+        catch (error) {
+            this.logger.error(`Failed to save HNSW data for ${nounId}:`, error);
+            throw new Error(`Failed to save HNSW data for ${nounId}: ${error}`);
+        }
+    }
+    /**
+     * Get HNSW graph data for a noun
+     * Storage path: entities/nouns/hnsw/{shard}/{id}.json
+     */
+    async getHNSWData(nounId) {
+        await this.ensureInitialized();
+        try {
+            const { GetObjectCommand } = await import('@aws-sdk/client-s3');
+            const shard = getShardIdFromUuid(nounId);
+            const key = `entities/nouns/hnsw/${shard}/${nounId}.json`;
+            const response = await this.s3Client.send(new GetObjectCommand({
+                Bucket: this.bucketName,
+                Key: key
+            }));
+            if (!response || !response.Body) {
+                return null;
+            }
+            const bodyContents = await response.Body.transformToString();
+            return JSON.parse(bodyContents);
+        }
+        catch (error) {
+            if (error.name === 'NoSuchKey' ||
+                error.message?.includes('NoSuchKey') ||
+                error.message?.includes('not found')) {
+                return null;
+            }
+            this.logger.error(`Failed to get HNSW data for ${nounId}:`, error);
+            throw new Error(`Failed to get HNSW data for ${nounId}: ${error}`);
+        }
+    }
+    /**
+     * Save HNSW system data (entry point, max level)
+     * Storage path: system/hnsw-system.json
+     */
+    async saveHNSWSystem(systemData) {
+        await this.ensureInitialized();
+        try {
+            const { PutObjectCommand } = await import('@aws-sdk/client-s3');
+            const key = `${this.systemPrefix}hnsw-system.json`;
+            await this.s3Client.send(new PutObjectCommand({
+                Bucket: this.bucketName,
+                Key: key,
+                Body: JSON.stringify(systemData, null, 2),
+                ContentType: 'application/json'
+            }));
+        }
+        catch (error) {
+            this.logger.error('Failed to save HNSW system data:', error);
+            throw new Error(`Failed to save HNSW system data: ${error}`);
+        }
+    }
+    /**
+     * Get HNSW system data (entry point, max level)
+     * Storage path: system/hnsw-system.json
+     */
+    async getHNSWSystem() {
+        await this.ensureInitialized();
+        try {
+            const { GetObjectCommand } = await import('@aws-sdk/client-s3');
+            const key = `${this.systemPrefix}hnsw-system.json`;
+            const response = await this.s3Client.send(new GetObjectCommand({
+                Bucket: this.bucketName,
+                Key: key
+            }));
+            if (!response || !response.Body) {
+                return null;
+            }
+            const bodyContents = await response.Body.transformToString();
+            return JSON.parse(bodyContents);
+        }
+        catch (error) {
+            if (error.name === 'NoSuchKey' ||
+                error.message?.includes('NoSuchKey') ||
+                error.message?.includes('not found')) {
+                return null;
+            }
+            this.logger.error('Failed to get HNSW system data:', error);
+            throw new Error(`Failed to get HNSW system data: ${error}`);
+        }
+    }
 }
 //# sourceMappingURL=s3CompatibleStorage.js.map

package/dist/utils/memoryDetection.d.ts

@@ -0,0 +1,119 @@
+/**
+ * Memory Detection Utilities
+ * Detects available system memory across different environments:
+ * - Docker/Kubernetes (cgroups v1 and v2)
+ * - Bare metal servers
+ * - Cloud instances
+ * - Development environments
+ *
+ * Scales from 2GB to 128GB+ with intelligent allocation
+ */
+export interface MemoryInfo {
+    /** Total memory available to this process (bytes) */
+    available: number;
+    /** Source of memory information */
+    source: 'cgroup-v2' | 'cgroup-v1' | 'system' | 'fallback';
+    /** Whether running in a container */
+    isContainer: boolean;
+    /** System total memory (may differ from available in containers) */
+    systemTotal: number;
+    /** Currently free memory (best-effort estimate) */
+    free: number;
+    /** Detection warnings (if any) */
+    warnings: string[];
+}
+export interface CacheAllocationStrategy {
+    /** Recommended cache size (bytes) */
+    cacheSize: number;
+    /** Allocation ratio used (0-1) */
+    ratio: number;
+    /** Minimum guaranteed size (bytes) */
+    minSize: number;
+    /** Maximum allowed size (bytes) */
+    maxSize: number | null;
+    /** Environment type detected */
+    environment: 'production' | 'development' | 'container' | 'unknown';
+    /** Model memory reserved (bytes) - v3.36.0+ */
+    modelMemory: number;
+    /** Model precision (q8 or fp32) */
+    modelPrecision: 'q8' | 'fp32';
+    /** Available memory after model reservation (bytes) */
+    availableForCache: number;
+    /** Reasoning for allocation */
+    reasoning: string;
+}
+/**
+ * Detect available memory across all environments
+ */
+export declare function detectAvailableMemory(): MemoryInfo;
+/**
+ * Calculate optimal cache size based on available memory
+ * Scales intelligently from 2GB to 128GB+
+ *
+ * v3.36.0+: Accounts for embedding model memory (150MB Q8, 250MB FP32)
+ */
+export declare function calculateOptimalCacheSize(memoryInfo: MemoryInfo, options?: {
+    /** Manual override (bytes) - takes precedence */
+    manualSize?: number;
+    /** Minimum cache size (bytes) - default 256MB */
+    minSize?: number;
+    /** Maximum cache size (bytes) - default unlimited */
+    maxSize?: number;
+    /** Force development mode allocation (more conservative) */
+    developmentMode?: boolean;
+    /** Model precision for memory calculation - default 'q8' */
+    modelPrecision?: 'q8' | 'fp32';
+}): CacheAllocationStrategy;
+/**
+ * Get recommended cache configuration for current environment
+ */
+export declare function getRecommendedCacheConfig(options?: {
+    /** Manual cache size override (bytes) */
+    manualSize?: number;
+    /** Minimum cache size (bytes) */
+    minSize?: number;
+    /** Maximum cache size (bytes) */
+    maxSize?: number;
+    /** Force development mode */
+    developmentMode?: boolean;
+}): {
+    memoryInfo: MemoryInfo;
+    allocation: CacheAllocationStrategy;
+    warnings: string[];
+};
+/**
+ * Detect embedding model memory usage
+ *
+ * Returns estimated runtime memory for the embedding model:
+ * - Q8 (quantized, default): ~150MB runtime (22MB on disk)
+ * - FP32 (full precision): ~250MB runtime (86MB on disk)
+ *
+ * Breakdown for Q8:
+ * - Model weights: 22MB
+ * - ONNX Runtime: 15-30MB
+ * - Session workspace: 50-100MB (peak during inference)
+ * - Total: ~100-150MB (we use 150MB conservative)
+ */
+export declare function detectModelMemory(options?: {
+    /** Model precision (default: 'q8') */
+    precision?: 'q8' | 'fp32';
+}): {
+    bytes: number;
+    precision: 'q8' | 'fp32';
+    breakdown: {
+        modelWeights: number;
+        onnxRuntime: number;
+        sessionWorkspace: number;
+    };
+};
+/**
+ * Format bytes to human-readable string
+ */
+export declare function formatBytes(bytes: number): string;
+/**
+ * Monitor memory usage and warn if approaching limits
+ */
+export declare function checkMemoryPressure(cacheSize: number, memoryInfo: MemoryInfo): {
+    pressure: 'none' | 'moderate' | 'high' | 'critical';
+    warnings: string[];
+};