@soulcraft/brainy 3.8.2 → 3.9.0

package/dist/storage/adapters/opfsStorage.js

@@ -1303,5 +1303,73 @@ export class OPFSStorage extends BaseStorage {
             nextCursor
         };
     }
+    /**
+     * Initialize counts from OPFS storage
+     */
+    async initializeCounts() {
+        try {
+            // Try to load existing counts from counts.json
+            const systemDir = await this.rootDir.getDirectoryHandle('system', { create: true });
+            const countsFile = await systemDir.getFileHandle('counts.json');
+            const file = await countsFile.getFile();
+            const data = await file.text();
+            const counts = JSON.parse(data);
+            // Restore counts from OPFS
+            this.entityCounts = new Map(Object.entries(counts.entityCounts || {}));
+            this.verbCounts = new Map(Object.entries(counts.verbCounts || {}));
+            this.totalNounCount = counts.totalNounCount || 0;
+            this.totalVerbCount = counts.totalVerbCount || 0;
+        }
+        catch (error) {
+            // If counts don't exist, initialize by scanning (one-time operation)
+            await this.initializeCountsFromScan();
+        }
+    }
+    /**
+     * Initialize counts by scanning OPFS (fallback for missing counts file)
+     */
+    async initializeCountsFromScan() {
+        try {
+            // Count nouns
+            let nounCount = 0;
+            for await (const [,] of this.nounsDir.entries()) {
+                nounCount++;
+            }
+            this.totalNounCount = nounCount;
+            // Count verbs
+            let verbCount = 0;
+            for await (const [,] of this.verbsDir.entries()) {
+                verbCount++;
+            }
+            this.totalVerbCount = verbCount;
+            // Save initial counts
+            await this.persistCounts();
+        }
+        catch (error) {
+            console.error('Error initializing counts from OPFS scan:', error);
+        }
+    }
+    /**
+     * Persist counts to OPFS storage
+     */
+    async persistCounts() {
+        try {
+            const systemDir = await this.rootDir.getDirectoryHandle('system', { create: true });
+            const countsFile = await systemDir.getFileHandle('counts.json', { create: true });
+            const writable = await countsFile.createWritable();
+            const counts = {
+                entityCounts: Object.fromEntries(this.entityCounts),
+                verbCounts: Object.fromEntries(this.verbCounts),
+                totalNounCount: this.totalNounCount,
+                totalVerbCount: this.totalVerbCount,
+                lastUpdated: new Date().toISOString()
+            };
+            await writable.write(JSON.stringify(counts));
+            await writable.close();
+        }
+        catch (error) {
+            console.error('Error persisting counts to OPFS:', error);
+        }
+    }
 }
 //# sourceMappingURL=opfsStorage.js.map

package/dist/storage/adapters/s3CompatibleStorage.d.ts

@@ -72,6 +72,10 @@ export declare class S3CompatibleStorage extends BaseStorage {
     private backpressure;
     private nounWriteBuffer;
     private verbWriteBuffer;
+    private coordinator?;
+    private shardManager?;
+    private cacheSync?;
+    private readWriteSeparation?;
     private requestCoalescer;
     private highVolumeMode;
     private lastVolumeCheck;
@@ -106,6 +110,24 @@ export declare class S3CompatibleStorage extends BaseStorage {
      * Initialize the storage adapter
      */
     init(): Promise<void>;
+    /**
+     * Set distributed components for multi-node coordination
+     * Zero-config: Automatically optimizes based on components provided
+     */
+    setDistributedComponents(components: {
+        coordinator?: any;
+        shardManager?: any;
+        cacheSync?: any;
+        readWriteSeparation?: any;
+    }): void;
+    /**
+     * Get the S3 key for a noun, using sharding if available
+     */
+    private getNounKey;
+    /**
+     * Get the S3 key for a verb, using sharding if available
+     */
+    private getVerbKey;
     /**
      * Override base class method to detect S3-specific throttling errors
      */
@@ -495,4 +517,16 @@ export declare class S3CompatibleStorage extends BaseStorage {
         hasMore: boolean;
         nextCursor?: string;
     }>;
+    /**
+     * Initialize counts from S3 storage
+     */
+    protected initializeCounts(): Promise<void>;
+    /**
+     * Initialize counts by scanning S3 (fallback for missing counts file)
+     */
+    private initializeCountsFromScan;
+    /**
+     * Persist counts to S3 storage
+     */
+    protected persistCounts(): Promise<void>;
 }

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -240,6 +240,49 @@ export class S3CompatibleStorage extends BaseStorage {
             throw new Error(`Failed to initialize ${this.serviceType} storage: ${error}`);
         }
     }
+    /**
+     * Set distributed components for multi-node coordination
+     * Zero-config: Automatically optimizes based on components provided
+     */
+    setDistributedComponents(components) {
+        this.coordinator = components.coordinator;
+        this.shardManager = components.shardManager;
+        this.cacheSync = components.cacheSync;
+        this.readWriteSeparation = components.readWriteSeparation;
+        // Auto-configure based on what's available
+        if (this.shardManager) {
+            console.log(`🎯 S3 Storage: Sharding enabled with ${this.shardManager.config?.shardCount || 64} shards`);
+        }
+        if (this.coordinator) {
+            console.log(`🤝 S3 Storage: Distributed coordination active (node: ${this.coordinator.nodeId})`);
+        }
+        if (this.cacheSync) {
+            console.log('🔄 S3 Storage: Cache synchronization enabled');
+        }
+        if (this.readWriteSeparation) {
+            console.log(`📖 S3 Storage: Read/write separation with ${this.readWriteSeparation.config?.replicationFactor || 3}x replication`);
+        }
+    }
+    /**
+     * Get the S3 key for a noun, using sharding if available
+     */
+    getNounKey(id) {
+        if (this.shardManager) {
+            const shardId = this.shardManager.getShardForKey(id);
+            return `shards/${shardId}/${this.nounPrefix}${id}.json`;
+        }
+        return `${this.nounPrefix}${id}.json`;
+    }
+    /**
+     * Get the S3 key for a verb, using sharding if available
+     */
+    getVerbKey(id) {
+        if (this.shardManager) {
+            const shardId = this.shardManager.getShardForKey(id);
+            return `shards/${shardId}/${this.verbPrefix}${id}.json`;
+        }
+        return `${this.verbPrefix}${id}.json`;
+    }
     /**
      * Override base class method to detect S3-specific throttling errors
      */
@@ -668,7 +711,8 @@ export class S3CompatibleStorage extends BaseStorage {
             };
             // Import the PutObjectCommand only when needed
             const { PutObjectCommand } = await import('@aws-sdk/client-s3');
-            const key = `${this.nounPrefix}${node.id}.json`;
+            // Use sharding if available
+            const key = this.getNounKey(node.id);
             const body = JSON.stringify(serializableNode, null, 2);
             this.logger.trace(`Saving to key: ${key}`);
             // Save the node to S3-compatible storage
@@ -1013,10 +1057,10 @@ export class S3CompatibleStorage extends BaseStorage {
             };
             // Import the PutObjectCommand only when needed
             const { PutObjectCommand } = await import('@aws-sdk/client-s3');
-            // Save the edge to S3-compatible storage
+            // Save the edge to S3-compatible storage using sharding if available
             await this.s3Client.send(new PutObjectCommand({
                 Bucket: this.bucketName,
-                Key: `${this.verbPrefix}${edge.id}.json`,
+                Key: this.getVerbKey(edge.id),
                 Body: JSON.stringify(serializableEdge, null, 2),
                 ContentType: 'application/json'
             }));
@@ -2660,5 +2704,87 @@ export class S3CompatibleStorage extends BaseStorage {
             nextCursor: result.nextCursor
         };
     }
+    /**
+     * Initialize counts from S3 storage
+     */
+    async initializeCounts() {
+        const countsKey = `${this.systemPrefix}counts.json`;
+        try {
+            const { GetObjectCommand } = await import('@aws-sdk/client-s3');
+            // Try to load existing counts
+            const response = await this.s3Client.send(new GetObjectCommand({
+                Bucket: this.bucketName,
+                Key: countsKey
+            }));
+            if (response.Body) {
+                const data = await response.Body.transformToString();
+                const counts = JSON.parse(data);
+                // Restore counts from S3
+                this.entityCounts = new Map(Object.entries(counts.entityCounts || {}));
+                this.verbCounts = new Map(Object.entries(counts.verbCounts || {}));
+                this.totalNounCount = counts.totalNounCount || 0;
+                this.totalVerbCount = counts.totalVerbCount || 0;
+            }
+        }
+        catch (error) {
+            if (error.name !== 'NoSuchKey') {
+                console.error('Error loading counts from S3:', error);
+            }
+            // If counts don't exist, initialize by scanning (one-time operation)
+            await this.initializeCountsFromScan();
+        }
+    }
+    /**
+     * Initialize counts by scanning S3 (fallback for missing counts file)
+     */
+    async initializeCountsFromScan() {
+        // This is expensive but only happens once for legacy data
+        // In production, counts are maintained incrementally
+        try {
+            const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
+            // Count nouns
+            const nounResponse = await this.s3Client.send(new ListObjectsV2Command({
+                Bucket: this.bucketName,
+                Prefix: this.nounPrefix
+            }));
+            this.totalNounCount = nounResponse.Contents?.filter((obj) => obj.Key?.endsWith('.json')).length || 0;
+            // Count verbs
+            const verbResponse = await this.s3Client.send(new ListObjectsV2Command({
+                Bucket: this.bucketName,
+                Prefix: this.verbPrefix
+            }));
+            this.totalVerbCount = verbResponse.Contents?.filter((obj) => obj.Key?.endsWith('.json')).length || 0;
+            // Save initial counts
+            await this.persistCounts();
+        }
+        catch (error) {
+            console.error('Error initializing counts from S3 scan:', error);
+        }
+    }
+    /**
+     * Persist counts to S3 storage
+     */
+    async persistCounts() {
+        const countsKey = `${this.systemPrefix}counts.json`;
+        try {
+            const { PutObjectCommand } = await import('@aws-sdk/client-s3');
+            const counts = {
+                entityCounts: Object.fromEntries(this.entityCounts),
+                verbCounts: Object.fromEntries(this.verbCounts),
+                totalNounCount: this.totalNounCount,
+                totalVerbCount: this.totalVerbCount,
+                lastUpdated: new Date().toISOString()
+            };
+            await this.s3Client.send(new PutObjectCommand({
+                Bucket: this.bucketName,
+                Key: countsKey,
+                Body: JSON.stringify(counts),
+                ContentType: 'application/json'
+            }));
+        }
+        catch (error) {
+            console.error('Error persisting counts to S3:', error);
+        }
+    }
 }
 //# sourceMappingURL=s3CompatibleStorage.js.map

package/dist/storage/baseStorage.js

@@ -337,14 +337,15 @@ export class BaseStorage extends BaseStorageAdapter {
             }
             // Check if the adapter has a paginated method for getting nouns
             if (typeof this.getNounsWithPagination === 'function') {
-                // Use the adapter's paginated method
+                // Use the adapter's paginated method - pass offset directly to adapter
                 const result = await this.getNounsWithPagination({
                     limit,
+                    offset, // Let the adapter handle offset for O(1) operation
                     cursor,
                     filter: options?.filter
                 });
-                // Apply offset if needed (some adapters might not support offset)
-                const items = result.items.slice(offset);
+                // Don't slice here - the adapter should handle offset efficiently
+                const items = result.items;
                 // CRITICAL SAFETY CHECK: Prevent infinite loops
                 // If we have no items but hasMore is true, force hasMore to false
                 // This prevents pagination bugs from causing infinite loops

package/dist/storage/readOnlyOptimizations.d.ts

@@ -6,7 +6,6 @@ import { HNSWNoun, Vector } from '../coreTypes.js';
 declare enum CompressionType {
     NONE = "none",
     GZIP = "gzip",
-    BROTLI = "brotli",
     QUANTIZATION = "quantization",
     HYBRID = "hybrid"
 }
@@ -74,14 +73,6 @@ export declare class ReadOnlyOptimizations {
      * GZIP decompression
      */
     private gzipDecompress;
-    /**
-     * Brotli compression (placeholder - similar to GZIP)
-     */
-    private brotliCompress;
-    /**
-     * Brotli decompression (placeholder)
-     */
-    private brotliDecompress;
     /**
      * Create prebuilt index segments for faster loading
      */

package/dist/storage/readOnlyOptimizations.js

@@ -7,9 +7,9 @@ var CompressionType;
 (function (CompressionType) {
     CompressionType["NONE"] = "none";
     CompressionType["GZIP"] = "gzip";
-    CompressionType["BROTLI"] = "brotli";
     CompressionType["QUANTIZATION"] = "quantization";
     CompressionType["HYBRID"] = "hybrid";
+    // BROTLI removed - was not actually implemented
 })(CompressionType || (CompressionType = {}));
 // Vector quantization methods
 var QuantizationType;
@@ -67,10 +67,7 @@ export class ReadOnlyOptimizations {
                 const gzipBuffer = new Float32Array(vector).buffer;
                 compressedData = await this.gzipCompress(gzipBuffer.slice(0));
                 break;
-            case CompressionType.BROTLI:
-                const brotliBuffer = new Float32Array(vector).buffer;
-                compressedData = await this.brotliCompress(brotliBuffer.slice(0));
-                break;
+            // Brotli removed - was not implemented
             case CompressionType.HYBRID:
                 // First quantize, then compress
                 const quantized = await this.quantizeVector(vector, segmentId);
@@ -99,9 +96,7 @@ export class ReadOnlyOptimizations {
             case CompressionType.GZIP:
                 const gzipDecompressed = await this.gzipDecompress(compressedData);
                 return Array.from(new Float32Array(gzipDecompressed));
-            case CompressionType.BROTLI:
-                const brotliDecompressed = await this.brotliDecompress(compressedData);
-                return Array.from(new Float32Array(brotliDecompressed));
+            // Brotli removed - was not implemented
             case CompressionType.HYBRID:
                 const gzipStage = await this.gzipDecompress(compressedData);
                 return this.dequantizeVector(gzipStage, segmentId, originalDimension);
@@ -219,21 +214,7 @@ export class ReadOnlyOptimizations {
             return compressedData;
         }
     }
-    /**
-     * Brotli compression (placeholder - similar to GZIP)
-     */
-    async brotliCompress(data) {
-        // Would implement Brotli compression here
-        console.warn('Brotli compression not implemented, falling back to GZIP');
-        return this.gzipCompress(data);
-    }
-    /**
-     * Brotli decompression (placeholder)
-     */
-    async brotliDecompress(compressedData) {
-        console.warn('Brotli decompression not implemented, falling back to GZIP');
-        return this.gzipDecompress(compressedData);
-    }
+    // Brotli methods removed - were not implemented
     /**
      * Create prebuilt index segments for faster loading
      */
@@ -277,8 +258,7 @@ export class ReadOnlyOptimizations {
         switch (this.config.compression.metadataCompression) {
             case CompressionType.GZIP:
                 return this.gzipCompress(data.buffer.slice(0));
-            case CompressionType.BROTLI:
-                return this.brotliCompress(data.buffer.slice(0));
+            // Brotli removed - was not implemented
             default:
                 return data.buffer.slice(0);
         }
@@ -329,9 +309,7 @@ export class ReadOnlyOptimizations {
             case CompressionType.GZIP:
                 decompressed = await this.gzipDecompress(compressedData);
                 break;
-            case CompressionType.BROTLI:
-                decompressed = await this.brotliDecompress(compressedData);
-                break;
+            // Brotli removed - was not implemented
             default:
                 decompressed = compressedData;
                 break;

package/dist/types/brainy.types.d.ts

@@ -269,10 +269,25 @@ export interface BrainyConfig {
         ttl?: number;
     };
     augmentations?: Record<string, any>;
+    distributed?: {
+        enabled: boolean;
+        nodeId?: string;
+        nodes?: string[];
+        coordinatorUrl?: string;
+        shardCount?: number;
+        replicationFactor?: number;
+        consensus?: 'raft' | 'none';
+        transport?: 'tcp' | 'http' | 'udp';
+    };
     warmup?: boolean;
     realtime?: boolean;
     multiTenancy?: boolean;
     telemetry?: boolean;
+    disableAutoRebuild?: boolean;
+    disableMetrics?: boolean;
+    disableAutoOptimize?: boolean;
+    batchWrites?: boolean;
+    maxConcurrentOperations?: number;
     verbose?: boolean;
     silent?: boolean;
 }

package/dist/utils/embedding.js

@@ -3,8 +3,6 @@
  * Complete rewrite to eliminate TensorFlow.js and use ONNX-based models
  */
 import { isBrowser } from './environment.js';
-import { join } from 'node:path';
-import { existsSync } from 'node:fs';
 // @ts-ignore - Transformers.js is now the primary embedding library
 import { pipeline, env } from '@huggingface/transformers';
 // CRITICAL: Disable ONNX memory arena to prevent 4-8GB allocation
@@ -281,15 +279,23 @@ export class TransformerEmbedding {
             }
             try {
                 // For Q8 models, we need to explicitly specify the model file
-                if (actualType === 'q8') {
-                    // Check if quantized model exists
-                    const modelPath = join(cacheDir, this.options.model, 'onnx', 'model_quantized.onnx');
-                    if (existsSync(modelPath)) {
-                        this.logger('log', '✅ Q8 model found locally');
+                if (actualType === 'q8' && !isBrowser()) {
+                    try {
+                        // Check if quantized model exists (Node.js only)
+                        const { join } = await import('node:path');
+                        const { existsSync } = await import('node:fs');
+                        const modelPath = join(cacheDir, this.options.model, 'onnx', 'model_quantized.onnx');
+                        if (existsSync(modelPath)) {
+                            this.logger('log', '✅ Q8 model found locally');
+                        }
+                        else {
+                            this.logger('warn', '⚠️ Q8 model not found');
+                            actualType = 'q8'; // Always Q8
+                        }
                     }
-                    else {
-                        this.logger('warn', '⚠️ Q8 model not found');
-                        actualType = 'q8'; // Always Q8
+                    catch (error) {
+                        // Skip model path check in browser or if imports fail
+                        this.logger('log', '🌐 Skipping local model check in browser environment');
                     }
                 }
                 this.extractor = await pipeline('feature-extraction', this.options.model, pipelineOptions);

package/dist/utils/metadataIndex.d.ts

@@ -68,6 +68,11 @@ export declare class MetadataIndexManager {
     private totalEntitiesByType;
     private unifiedCache;
     constructor(storage: StorageAdapter, config?: MetadataIndexConfig);
+    /**
+     * Lazy load entity counts from storage statistics (O(1) operation)
+     * This avoids rebuilding the entire index on startup
+     */
+    private lazyLoadCounts;
     /**
      * Get index key for field and value
      */

package/dist/utils/metadataIndex.js

@@ -45,6 +45,30 @@ export class MetadataIndexManager {
         });
         // Get global unified cache for coordinated memory management
         this.unifiedCache = getGlobalCache();
+        // Lazy load counts from storage statistics on first access
+        this.lazyLoadCounts();
+    }
+    /**
+     * Lazy load entity counts from storage statistics (O(1) operation)
+     * This avoids rebuilding the entire index on startup
+     */
+    async lazyLoadCounts() {
+        try {
+            // Get statistics from storage (should be O(1) with our FileSystemStorage improvements)
+            const stats = await this.storage.getStatistics();
+            if (stats && stats.nounCount) {
+                // Populate entity counts from storage statistics
+                for (const [type, count] of Object.entries(stats.nounCount)) {
+                    if (typeof count === 'number' && count > 0) {
+                        this.totalEntitiesByType.set(type, count);
+                    }
+                }
+            }
+        }
+        catch (error) {
+            // Silently fail - counts will be populated as entities are added
+            // This maintains zero-configuration principle
+        }
     }
     /**
      * Get index key for field and value

package/dist/utils/mutex.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Universal Mutex Implementation for Thread-Safe Operations
+ * Provides consistent locking across all storage adapters
+ * Critical for preventing race conditions in count operations
+ */
+export interface MutexInterface {
+    acquire(key: string, timeout?: number): Promise<() => void>;
+    runExclusive<T>(key: string, fn: () => Promise<T>, timeout?: number): Promise<T>;
+    isLocked(key: string): boolean;
+}
+/**
+ * In-memory mutex for single-process scenarios
+ * Used by MemoryStorage and as fallback for other adapters
+ */
+export declare class InMemoryMutex implements MutexInterface {
+    private locks;
+    acquire(key: string, timeout?: number): Promise<() => void>;
+    private release;
+    runExclusive<T>(key: string, fn: () => Promise<T>, timeout?: number): Promise<T>;
+    isLocked(key: string): boolean;
+}
+/**
+ * File-based mutex for multi-process scenarios (Node.js)
+ * Uses atomic file operations to prevent TOCTOU races
+ */
+export declare class FileMutex implements MutexInterface {
+    private fs;
+    private path;
+    private lockDir;
+    private processLocks;
+    private lockTimers;
+    constructor(lockDir: string);
+    acquire(key: string, timeout?: number): Promise<() => void>;
+    private release;
+    runExclusive<T>(key: string, fn: () => Promise<T>, timeout?: number): Promise<T>;
+    isLocked(key: string): boolean;
+    /**
+     * Clean up all locks held by this process
+     */
+    cleanup(): Promise<void>;
+}
+/**
+ * Factory to create appropriate mutex for the environment
+ */
+export declare function createMutex(options?: {
+    type?: 'memory' | 'file';
+    lockDir?: string;
+}): MutexInterface;
+export declare function getGlobalMutex(): MutexInterface;
+/**
+ * Cleanup function for graceful shutdown
+ */
+export declare function cleanupMutexes(): Promise<void>;