@soulcraft/brainy 3.8.3 → 3.9.1

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -240,6 +240,49 @@ export class S3CompatibleStorage extends BaseStorage {
             throw new Error(`Failed to initialize ${this.serviceType} storage: ${error}`);
         }
     }
+    /**
+     * Set distributed components for multi-node coordination
+     * Zero-config: Automatically optimizes based on components provided
+     */
+    setDistributedComponents(components) {
+        this.coordinator = components.coordinator;
+        this.shardManager = components.shardManager;
+        this.cacheSync = components.cacheSync;
+        this.readWriteSeparation = components.readWriteSeparation;
+        // Auto-configure based on what's available
+        if (this.shardManager) {
+            console.log(`🎯 S3 Storage: Sharding enabled with ${this.shardManager.config?.shardCount || 64} shards`);
+        }
+        if (this.coordinator) {
+            console.log(`🤝 S3 Storage: Distributed coordination active (node: ${this.coordinator.nodeId})`);
+        }
+        if (this.cacheSync) {
+            console.log('🔄 S3 Storage: Cache synchronization enabled');
+        }
+        if (this.readWriteSeparation) {
+            console.log(`📖 S3 Storage: Read/write separation with ${this.readWriteSeparation.config?.replicationFactor || 3}x replication`);
+        }
+    }
+    /**
+     * Get the S3 key for a noun, using sharding if available
+     */
+    getNounKey(id) {
+        if (this.shardManager) {
+            const shardId = this.shardManager.getShardForKey(id);
+            return `shards/${shardId}/${this.nounPrefix}${id}.json`;
+        }
+        return `${this.nounPrefix}${id}.json`;
+    }
+    /**
+     * Get the S3 key for a verb, using sharding if available
+     */
+    getVerbKey(id) {
+        if (this.shardManager) {
+            const shardId = this.shardManager.getShardForKey(id);
+            return `shards/${shardId}/${this.verbPrefix}${id}.json`;
+        }
+        return `${this.verbPrefix}${id}.json`;
+    }
     /**
      * Override base class method to detect S3-specific throttling errors
      */
@@ -668,7 +711,8 @@ export class S3CompatibleStorage extends BaseStorage {
             };
             // Import the PutObjectCommand only when needed
             const { PutObjectCommand } = await import('@aws-sdk/client-s3');
-            const key = `${this.nounPrefix}${node.id}.json`;
+            // Use sharding if available
+            const key = this.getNounKey(node.id);
             const body = JSON.stringify(serializableNode, null, 2);
             this.logger.trace(`Saving to key: ${key}`);
             // Save the node to S3-compatible storage
@@ -1013,10 +1057,10 @@ export class S3CompatibleStorage extends BaseStorage {
             };
             // Import the PutObjectCommand only when needed
             const { PutObjectCommand } = await import('@aws-sdk/client-s3');
-            // Save the edge to S3-compatible storage
+            // Save the edge to S3-compatible storage using sharding if available
             await this.s3Client.send(new PutObjectCommand({
                 Bucket: this.bucketName,
-                Key: `${this.verbPrefix}${edge.id}.json`,
+                Key: this.getVerbKey(edge.id),
                 Body: JSON.stringify(serializableEdge, null, 2),
                 ContentType: 'application/json'
             }));
@@ -2660,5 +2704,87 @@ export class S3CompatibleStorage extends BaseStorage {
             nextCursor: result.nextCursor
         };
     }
+    /**
+     * Initialize counts from S3 storage
+     */
+    async initializeCounts() {
+        const countsKey = `${this.systemPrefix}counts.json`;
+        try {
+            const { GetObjectCommand } = await import('@aws-sdk/client-s3');
+            // Try to load existing counts
+            const response = await this.s3Client.send(new GetObjectCommand({
+                Bucket: this.bucketName,
+                Key: countsKey
+            }));
+            if (response.Body) {
+                const data = await response.Body.transformToString();
+                const counts = JSON.parse(data);
+                // Restore counts from S3
+                this.entityCounts = new Map(Object.entries(counts.entityCounts || {}));
+                this.verbCounts = new Map(Object.entries(counts.verbCounts || {}));
+                this.totalNounCount = counts.totalNounCount || 0;
+                this.totalVerbCount = counts.totalVerbCount || 0;
+            }
+        }
+        catch (error) {
+            if (error.name !== 'NoSuchKey') {
+                console.error('Error loading counts from S3:', error);
+            }
+            // If counts don't exist, initialize by scanning (one-time operation)
+            await this.initializeCountsFromScan();
+        }
+    }
+    /**
+     * Initialize counts by scanning S3 (fallback for missing counts file)
+     */
+    async initializeCountsFromScan() {
+        // This is expensive but only happens once for legacy data
+        // In production, counts are maintained incrementally
+        try {
+            const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
+            // Count nouns
+            const nounResponse = await this.s3Client.send(new ListObjectsV2Command({
+                Bucket: this.bucketName,
+                Prefix: this.nounPrefix
+            }));
+            this.totalNounCount = nounResponse.Contents?.filter((obj) => obj.Key?.endsWith('.json')).length || 0;
+            // Count verbs
+            const verbResponse = await this.s3Client.send(new ListObjectsV2Command({
+                Bucket: this.bucketName,
+                Prefix: this.verbPrefix
+            }));
+            this.totalVerbCount = verbResponse.Contents?.filter((obj) => obj.Key?.endsWith('.json')).length || 0;
+            // Save initial counts
+            await this.persistCounts();
+        }
+        catch (error) {
+            console.error('Error initializing counts from S3 scan:', error);
+        }
+    }
+    /**
+     * Persist counts to S3 storage
+     */
+    async persistCounts() {
+        const countsKey = `${this.systemPrefix}counts.json`;
+        try {
+            const { PutObjectCommand } = await import('@aws-sdk/client-s3');
+            const counts = {
+                entityCounts: Object.fromEntries(this.entityCounts),
+                verbCounts: Object.fromEntries(this.verbCounts),
+                totalNounCount: this.totalNounCount,
+                totalVerbCount: this.totalVerbCount,
+                lastUpdated: new Date().toISOString()
+            };
+            await this.s3Client.send(new PutObjectCommand({
+                Bucket: this.bucketName,
+                Key: countsKey,
+                Body: JSON.stringify(counts),
+                ContentType: 'application/json'
+            }));
+        }
+        catch (error) {
+            console.error('Error persisting counts to S3:', error);
+        }
+    }
 }
 //# sourceMappingURL=s3CompatibleStorage.js.map

package/dist/storage/baseStorage.js

@@ -337,14 +337,15 @@ export class BaseStorage extends BaseStorageAdapter {
             }
             // Check if the adapter has a paginated method for getting nouns
             if (typeof this.getNounsWithPagination === 'function') {
-                // Use the adapter's paginated method
+                // Use the adapter's paginated method - pass offset directly to adapter
                 const result = await this.getNounsWithPagination({
                     limit,
+                    offset, // Let the adapter handle offset for O(1) operation
                     cursor,
                     filter: options?.filter
                 });
-                // Apply offset if needed (some adapters might not support offset)
-                const items = result.items.slice(offset);
+                // Don't slice here - the adapter should handle offset efficiently
+                const items = result.items;
                 // CRITICAL SAFETY CHECK: Prevent infinite loops
                 // If we have no items but hasMore is true, force hasMore to false
                 // This prevents pagination bugs from causing infinite loops

package/dist/storage/readOnlyOptimizations.d.ts

@@ -6,7 +6,6 @@ import { HNSWNoun, Vector } from '../coreTypes.js';
 declare enum CompressionType {
     NONE = "none",
     GZIP = "gzip",
-    BROTLI = "brotli",
     QUANTIZATION = "quantization",
     HYBRID = "hybrid"
 }
@@ -74,14 +73,6 @@ export declare class ReadOnlyOptimizations {
      * GZIP decompression
      */
     private gzipDecompress;
-    /**
-     * Brotli compression (placeholder - similar to GZIP)
-     */
-    private brotliCompress;
-    /**
-     * Brotli decompression (placeholder)
-     */
-    private brotliDecompress;
     /**
      * Create prebuilt index segments for faster loading
      */

package/dist/storage/readOnlyOptimizations.js

@@ -7,9 +7,9 @@ var CompressionType;
 (function (CompressionType) {
     CompressionType["NONE"] = "none";
     CompressionType["GZIP"] = "gzip";
-    CompressionType["BROTLI"] = "brotli";
     CompressionType["QUANTIZATION"] = "quantization";
     CompressionType["HYBRID"] = "hybrid";
+    // BROTLI removed - was not actually implemented
 })(CompressionType || (CompressionType = {}));
 // Vector quantization methods
 var QuantizationType;
@@ -67,10 +67,7 @@ export class ReadOnlyOptimizations {
                 const gzipBuffer = new Float32Array(vector).buffer;
                 compressedData = await this.gzipCompress(gzipBuffer.slice(0));
                 break;
-            case CompressionType.BROTLI:
-                const brotliBuffer = new Float32Array(vector).buffer;
-                compressedData = await this.brotliCompress(brotliBuffer.slice(0));
-                break;
+            // Brotli removed - was not implemented
             case CompressionType.HYBRID:
                 // First quantize, then compress
                 const quantized = await this.quantizeVector(vector, segmentId);
@@ -99,9 +96,7 @@ export class ReadOnlyOptimizations {
             case CompressionType.GZIP:
                 const gzipDecompressed = await this.gzipDecompress(compressedData);
                 return Array.from(new Float32Array(gzipDecompressed));
-            case CompressionType.BROTLI:
-                const brotliDecompressed = await this.brotliDecompress(compressedData);
-                return Array.from(new Float32Array(brotliDecompressed));
+            // Brotli removed - was not implemented
             case CompressionType.HYBRID:
                 const gzipStage = await this.gzipDecompress(compressedData);
                 return this.dequantizeVector(gzipStage, segmentId, originalDimension);
@@ -219,21 +214,7 @@ export class ReadOnlyOptimizations {
             return compressedData;
         }
     }
-    /**
-     * Brotli compression (placeholder - similar to GZIP)
-     */
-    async brotliCompress(data) {
-        // Would implement Brotli compression here
-        console.warn('Brotli compression not implemented, falling back to GZIP');
-        return this.gzipCompress(data);
-    }
-    /**
-     * Brotli decompression (placeholder)
-     */
-    async brotliDecompress(compressedData) {
-        console.warn('Brotli decompression not implemented, falling back to GZIP');
-        return this.gzipDecompress(compressedData);
-    }
+    // Brotli methods removed - were not implemented
     /**
      * Create prebuilt index segments for faster loading
      */
@@ -277,8 +258,7 @@ export class ReadOnlyOptimizations {
         switch (this.config.compression.metadataCompression) {
             case CompressionType.GZIP:
                 return this.gzipCompress(data.buffer.slice(0));
-            case CompressionType.BROTLI:
-                return this.brotliCompress(data.buffer.slice(0));
+            // Brotli removed - was not implemented
             default:
                 return data.buffer.slice(0);
         }
@@ -329,9 +309,7 @@ export class ReadOnlyOptimizations {
             case CompressionType.GZIP:
                 decompressed = await this.gzipDecompress(compressedData);
                 break;
-            case CompressionType.BROTLI:
-                decompressed = await this.brotliDecompress(compressedData);
-                break;
+            // Brotli removed - was not implemented
             default:
                 decompressed = compressedData;
                 break;

package/dist/types/brainy.types.d.ts

@@ -269,10 +269,25 @@ export interface BrainyConfig {
         ttl?: number;
     };
     augmentations?: Record<string, any>;
+    distributed?: {
+        enabled: boolean;
+        nodeId?: string;
+        nodes?: string[];
+        coordinatorUrl?: string;
+        shardCount?: number;
+        replicationFactor?: number;
+        consensus?: 'raft' | 'none';
+        transport?: 'tcp' | 'http' | 'udp';
+    };
     warmup?: boolean;
     realtime?: boolean;
     multiTenancy?: boolean;
     telemetry?: boolean;
+    disableAutoRebuild?: boolean;
+    disableMetrics?: boolean;
+    disableAutoOptimize?: boolean;
+    batchWrites?: boolean;
+    maxConcurrentOperations?: number;
     verbose?: boolean;
     silent?: boolean;
 }

package/dist/utils/metadataIndex.d.ts

@@ -68,6 +68,11 @@ export declare class MetadataIndexManager {
     private totalEntitiesByType;
     private unifiedCache;
     constructor(storage: StorageAdapter, config?: MetadataIndexConfig);
+    /**
+     * Lazy load entity counts from storage statistics (O(1) operation)
+     * This avoids rebuilding the entire index on startup
+     */
+    private lazyLoadCounts;
     /**
      * Get index key for field and value
      */

package/dist/utils/metadataIndex.js

@@ -45,6 +45,30 @@ export class MetadataIndexManager {
         });
         // Get global unified cache for coordinated memory management
         this.unifiedCache = getGlobalCache();
+        // Lazy load counts from storage statistics on first access
+        this.lazyLoadCounts();
+    }
+    /**
+     * Lazy load entity counts from storage statistics (O(1) operation)
+     * This avoids rebuilding the entire index on startup
+     */
+    async lazyLoadCounts() {
+        try {
+            // Get statistics from storage (should be O(1) with our FileSystemStorage improvements)
+            const stats = await this.storage.getStatistics();
+            if (stats && stats.nounCount) {
+                // Populate entity counts from storage statistics
+                for (const [type, count] of Object.entries(stats.nounCount)) {
+                    if (typeof count === 'number' && count > 0) {
+                        this.totalEntitiesByType.set(type, count);
+                    }
+                }
+            }
+        }
+        catch (error) {
+            // Silently fail - counts will be populated as entities are added
+            // This maintains zero-configuration principle
+        }
     }
     /**
      * Get index key for field and value

package/dist/utils/mutex.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Universal Mutex Implementation for Thread-Safe Operations
+ * Provides consistent locking across all storage adapters
+ * Critical for preventing race conditions in count operations
+ */
+export interface MutexInterface {
+    acquire(key: string, timeout?: number): Promise<() => void>;
+    runExclusive<T>(key: string, fn: () => Promise<T>, timeout?: number): Promise<T>;
+    isLocked(key: string): boolean;
+}
+/**
+ * In-memory mutex for single-process scenarios
+ * Used by MemoryStorage and as fallback for other adapters
+ */
+export declare class InMemoryMutex implements MutexInterface {
+    private locks;
+    acquire(key: string, timeout?: number): Promise<() => void>;
+    private release;
+    runExclusive<T>(key: string, fn: () => Promise<T>, timeout?: number): Promise<T>;
+    isLocked(key: string): boolean;
+}
+/**
+ * File-based mutex for multi-process scenarios (Node.js)
+ * Uses atomic file operations to prevent TOCTOU races
+ */
+export declare class FileMutex implements MutexInterface {
+    private fs;
+    private path;
+    private lockDir;
+    private processLocks;
+    private lockTimers;
+    constructor(lockDir: string);
+    acquire(key: string, timeout?: number): Promise<() => void>;
+    private release;
+    runExclusive<T>(key: string, fn: () => Promise<T>, timeout?: number): Promise<T>;
+    isLocked(key: string): boolean;
+    /**
+     * Clean up all locks held by this process
+     */
+    cleanup(): Promise<void>;
+}
+/**
+ * Factory to create appropriate mutex for the environment
+ */
+export declare function createMutex(options?: {
+    type?: 'memory' | 'file';
+    lockDir?: string;
+}): MutexInterface;
+export declare function getGlobalMutex(): MutexInterface;
+/**
+ * Cleanup function for graceful shutdown
+ */
+export declare function cleanupMutexes(): Promise<void>;

package/dist/utils/mutex.js

@@ -0,0 +1,221 @@
+/**
+ * Universal Mutex Implementation for Thread-Safe Operations
+ * Provides consistent locking across all storage adapters
+ * Critical for preventing race conditions in count operations
+ */
+/**
+ * In-memory mutex for single-process scenarios
+ * Used by MemoryStorage and as fallback for other adapters
+ */
+export class InMemoryMutex {
+    constructor() {
+        this.locks = new Map();
+    }
+    async acquire(key, timeout = 30000) {
+        if (!this.locks.has(key)) {
+            this.locks.set(key, { queue: [], locked: false });
+        }
+        const lock = this.locks.get(key);
+        if (!lock.locked) {
+            lock.locked = true;
+            return () => this.release(key);
+        }
+        // Wait in queue
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                const index = lock.queue.indexOf(resolver);
+                if (index !== -1) {
+                    lock.queue.splice(index, 1);
+                }
+                reject(new Error(`Mutex timeout for key: ${key}`));
+            }, timeout);
+            const resolver = () => {
+                clearTimeout(timer);
+                lock.locked = true;
+                resolve(() => this.release(key));
+            };
+            lock.queue.push(resolver);
+        });
+    }
+    release(key) {
+        const lock = this.locks.get(key);
+        if (!lock)
+            return;
+        if (lock.queue.length > 0) {
+            const next = lock.queue.shift();
+            next();
+        }
+        else {
+            lock.locked = false;
+            // Clean up if no waiters
+            if (lock.queue.length === 0) {
+                this.locks.delete(key);
+            }
+        }
+    }
+    async runExclusive(key, fn, timeout) {
+        const release = await this.acquire(key, timeout);
+        try {
+            return await fn();
+        }
+        finally {
+            release();
+        }
+    }
+    isLocked(key) {
+        return this.locks.get(key)?.locked || false;
+    }
+}
+/**
+ * File-based mutex for multi-process scenarios (Node.js)
+ * Uses atomic file operations to prevent TOCTOU races
+ */
+export class FileMutex {
+    constructor(lockDir) {
+        this.processLocks = new Map();
+        this.lockTimers = new Map();
+        this.lockDir = lockDir;
+        // Lazy load Node.js modules
+        if (typeof window === 'undefined') {
+            this.fs = require('fs');
+            this.path = require('path');
+        }
+    }
+    async acquire(key, timeout = 30000) {
+        if (!this.fs || !this.path) {
+            throw new Error('FileMutex is only available in Node.js environments');
+        }
+        const lockFile = this.path.join(this.lockDir, `${key}.lock`);
+        const lockId = `${Date.now()}_${Math.random()}_${process.pid}`;
+        const startTime = Date.now();
+        // Ensure lock directory exists
+        await this.fs.promises.mkdir(this.lockDir, { recursive: true });
+        while (Date.now() - startTime < timeout) {
+            try {
+                // Atomic lock creation using 'wx' flag
+                await this.fs.promises.writeFile(lockFile, JSON.stringify({
+                    lockId,
+                    pid: process.pid,
+                    timestamp: Date.now(),
+                    expiresAt: Date.now() + timeout
+                }), { flag: 'wx' } // Write exclusive - fails if exists
+                );
+                // Successfully acquired lock
+                const release = () => this.release(key, lockFile, lockId);
+                this.processLocks.set(key, release);
+                // Auto-release on timeout
+                const timer = setTimeout(() => {
+                    release();
+                }, timeout);
+                this.lockTimers.set(key, timer);
+                return release;
+            }
+            catch (error) {
+                if (error.code === 'EEXIST') {
+                    // Lock exists - check if expired
+                    try {
+                        const data = await this.fs.promises.readFile(lockFile, 'utf-8');
+                        const lock = JSON.parse(data);
+                        if (lock.expiresAt < Date.now()) {
+                            // Expired - try to remove
+                            try {
+                                await this.fs.promises.unlink(lockFile);
+                                continue; // Retry acquisition
+                            }
+                            catch (unlinkError) {
+                                if (unlinkError.code !== 'ENOENT') {
+                                    // Someone else removed it, continue
+                                    continue;
+                                }
+                            }
+                        }
+                    }
+                    catch {
+                        // Can't read lock file, assume it's valid
+                    }
+                    // Wait before retry
+                    await new Promise(resolve => setTimeout(resolve, 50));
+                }
+                else {
+                    throw error;
+                }
+            }
+        }
+        throw new Error(`Failed to acquire mutex for key: ${key} after ${timeout}ms`);
+    }
+    async release(key, lockFile, lockId) {
+        // Clear timer
+        const timer = this.lockTimers.get(key);
+        if (timer) {
+            clearTimeout(timer);
+            this.lockTimers.delete(key);
+        }
+        // Remove from process locks
+        this.processLocks.delete(key);
+        try {
+            // Verify we own the lock before releasing
+            const data = await this.fs.promises.readFile(lockFile, 'utf-8');
+            const lock = JSON.parse(data);
+            if (lock.lockId === lockId) {
+                await this.fs.promises.unlink(lockFile);
+            }
+        }
+        catch {
+            // Lock already released or doesn't exist
+        }
+    }
+    async runExclusive(key, fn, timeout) {
+        const release = await this.acquire(key, timeout);
+        try {
+            return await fn();
+        }
+        finally {
+            release();
+        }
+    }
+    isLocked(key) {
+        return this.processLocks.has(key);
+    }
+    /**
+     * Clean up all locks held by this process
+     */
+    async cleanup() {
+        // Clear all timers
+        for (const timer of this.lockTimers.values()) {
+            clearTimeout(timer);
+        }
+        this.lockTimers.clear();
+        // Release all locks
+        const releases = Array.from(this.processLocks.values());
+        await Promise.all(releases.map(release => release()));
+        this.processLocks.clear();
+    }
+}
+/**
+ * Factory to create appropriate mutex for the environment
+ */
+export function createMutex(options) {
+    const type = options?.type || (typeof window === 'undefined' ? 'file' : 'memory');
+    if (type === 'file' && typeof window === 'undefined') {
+        const lockDir = options?.lockDir || '.brainy/locks';
+        return new FileMutex(lockDir);
+    }
+    return new InMemoryMutex();
+}
+// Global mutex instance for count operations
+let globalMutex = null;
+export function getGlobalMutex() {
+    if (!globalMutex) {
+        globalMutex = createMutex();
+    }
+    return globalMutex;
+}
+/**
+ * Cleanup function for graceful shutdown
+ */
+export async function cleanupMutexes() {
+    if (globalMutex && 'cleanup' in globalMutex) {
+        await globalMutex.cleanup();
+    }
+}
+//# sourceMappingURL=mutex.js.map

package/dist/utils/paramValidation.js

@@ -130,11 +130,24 @@ export function validateFindParams(params) {
 export function validateAddParams(params) {
     // Universal truth: must have data or vector
     if (!params.data && !params.vector) {
-        throw new Error('must provide either data or vector');
+        throw new Error(`Invalid add() parameters: Missing required field 'data'\n` +
+            `\nReceived: ${JSON.stringify({
+                type: params.type,
+                hasMetadata: !!params.metadata,
+                hasId: !!params.id
+            }, null, 2)}\n` +
+            `\nExpected one of:\n` +
+            `  { data: 'text to store', type?: 'note', metadata?: {...} }\n` +
+            `  { vector: [0.1, 0.2, ...], type?: 'embedding', metadata?: {...} }\n` +
+            `\nExamples:\n` +
+            `  await brain.add({ data: 'Machine learning is AI', type: 'concept' })\n` +
+            `  await brain.add({ data: { title: 'Doc', content: '...' }, type: 'document' })`);
     }
     // Validate noun type
     if (!Object.values(NounType).includes(params.type)) {
-        throw new Error(`invalid NounType: ${params.type}`);
+        throw new Error(`Invalid NounType: '${params.type}'\n` +
+            `\nValid types: ${Object.values(NounType).join(', ')}\n` +
+            `\nExample: await brain.add({ data: 'text', type: NounType.Note })`);
     }
     // Validate vector dimensions if provided
     if (params.vector) {
@@ -182,8 +195,11 @@ export function validateRelateParams(params) {
     if (params.from === params.to) {
         throw new Error('cannot create self-referential relationship');
     }
-    // Validate verb type
-    if (!Object.values(VerbType).includes(params.type)) {
+    // Validate verb type - default to RelatedTo if not specified
+    if (params.type === undefined) {
+        params.type = VerbType.RelatedTo;
+    }
+    else if (!Object.values(VerbType).includes(params.type)) {
         throw new Error(`invalid VerbType: ${params.type}`);
     }
     // Universal truth: weight must be 0-1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.8.3",
+  "version": "3.9.1",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",