@soulcraft/brainy 4.10.0 → 4.10.2

package/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [4.10.1](https://github.com/soulcraftlabs/brainy/compare/v4.10.0...v4.10.1) (2025-10-29)
+
+- fix: add mutex locks to FileSystemStorage for HNSW concurrency (CRITICAL) (ff86e88)
+
+
 ### [4.10.0](https://github.com/soulcraftlabs/brainy/compare/v4.9.2...v4.10.0) (2025-10-29)
 
 - perf: 48-64× faster HNSW bulk imports via concurrent neighbor updates (4038afd)

package/dist/importers/VFSStructureGenerator.js

@@ -29,16 +29,10 @@ export class VFSStructureGenerator {
     async init() {
         // Get brain's cached VFS instance (creates if doesn't exist)
         this.vfs = this.brain.vfs();
-        // Initialize if not already initialized
-        // VFS.init() is idempotent (safe to call multiple times)
-        try {
-            // Check if already initialized
-            await this.vfs.stat('/');
-        }
-        catch (error) {
-            // Not initialized, initialize now
-            await this.vfs.init();
-        }
+        // CRITICAL FIX (v4.10.2): Always call vfs.init() explicitly
+        // The previous code tried to check if initialized via stat('/') but this was unreliable
+        // vfs.init() is idempotent, so calling it multiple times is safe
+        await this.vfs.init();
     }
     /**
      * Generate VFS structure from import result

package/dist/storage/adapters/fileSystemStorage.d.ts

@@ -27,6 +27,7 @@ export declare class FileSystemStorage extends BaseStorage {
     private activeLocks;
     private lockTimers;
     private allTimers;
+    private hnswLocks;
     private compressionEnabled;
     private compressionLevel;
     /**
@@ -392,7 +393,7 @@ export declare class FileSystemStorage extends BaseStorage {
     /**
      * Save HNSW system data (entry point, max level)
      *
-     * CRITICAL FIX (v4.10.1): Atomic write to prevent race conditions during concurrent updates
+     * CRITICAL FIX (v4.10.1): Mutex lock + atomic write to prevent race conditions
      */
     saveHNSWSystem(systemData: {
         entryPointId: string | null;

package/dist/storage/adapters/fileSystemStorage.js

@@ -53,6 +53,10 @@ export class FileSystemStorage extends BaseStorage {
         this.activeLocks = new Set();
         this.lockTimers = new Map(); // Track timers for cleanup
         this.allTimers = new Set(); // Track all timers for cleanup
+        // CRITICAL FIX (v4.10.1): Mutex locks for HNSW concurrency control
+        // Prevents read-modify-write races during concurrent neighbor updates at scale (1000+ ops)
+        // Matches MemoryStorage and OPFSStorage behavior (tested in production)
+        this.hnswLocks = new Map();
         // Compression configuration (v4.0.0)
         this.compressionEnabled = true; // Enable gzip compression by default for 60-80% disk savings
         this.compressionLevel = 6; // zlib compression level (1-9, default: 6 = balanced)
@@ -2174,51 +2178,74 @@ export class FileSystemStorage extends BaseStorage {
      */
     async saveHNSWData(nounId, hnswData) {
         await this.ensureInitialized();
-        // CRITICAL FIX (v4.7.3): Must preserve existing node data (id, vector) when updating HNSW metadata
-        // Previous implementation overwrote the entire file, destroying vector data
-        // Now we READ the existing node, UPDATE only connections/level, then WRITE back the complete node
-        // CRITICAL FIX (v4.10.1): Atomic write to prevent race conditions during concurrent HNSW updates
-        // Uses temp file + atomic rename strategy (POSIX guarantees rename() atomicity)
-        // Prevents data corruption when multiple entities connect to same neighbor simultaneously
         const filePath = this.getNodePath(nounId);
-        const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`;
+        const lockKey = `hnsw/${nounId}`;
+        // CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races
+        // Problem: Without mutex, concurrent operations can:
+        //   1. Thread A reads file (connections: [1,2,3])
+        //   2. Thread B reads file (connections: [1,2,3])
+        //   3. Thread A adds connection 4, writes [1,2,3,4]
+        //   4. Thread B adds connection 5, writes [1,2,3,5] ← Connection 4 LOST!
+        // Solution: Mutex serializes operations per entity (like Memory/OPFS adapters)
+        // Production scale: Prevents corruption at 1000+ concurrent operations
+        // Wait for any pending operations on this entity
+        while (this.hnswLocks.has(lockKey)) {
+            await this.hnswLocks.get(lockKey);
+        }
+        // Acquire lock
+        let releaseLock;
+        const lockPromise = new Promise(resolve => { releaseLock = resolve; });
+        this.hnswLocks.set(lockKey, lockPromise);
         try {
-            // Read existing node data (if exists)
-            let existingNode = {};
+            // CRITICAL FIX (v4.7.3): Must preserve existing node data (id, vector) when updating HNSW metadata
+            // Previous implementation overwrote the entire file, destroying vector data
+            // Now we READ the existing node, UPDATE only connections/level, then WRITE back the complete node
+            // CRITICAL FIX (v4.9.2): Atomic write to prevent torn writes during crashes
+            // Uses temp file + atomic rename strategy (POSIX guarantees rename() atomicity)
+            // Note: Atomic rename alone does NOT prevent concurrent read-modify-write races (needs mutex above)
+            const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`;
             try {
-                const existingData = await fs.promises.readFile(filePath, 'utf-8');
-                existingNode = JSON.parse(existingData);
+                // Read existing node data (if exists)
+                let existingNode = {};
+                try {
+                    const existingData = await fs.promises.readFile(filePath, 'utf-8');
+                    existingNode = JSON.parse(existingData);
+                }
+                catch (error) {
+                    // File doesn't exist yet - will create new
+                    if (error.code !== 'ENOENT') {
+                        throw error;
+                    }
+                }
+                // Preserve id and vector, update only HNSW graph metadata
+                const updatedNode = {
+                    ...existingNode, // Preserve all existing fields (id, vector, etc.)
+                    level: hnswData.level,
+                    connections: hnswData.connections
+                };
+                // ATOMIC WRITE SEQUENCE:
+                // 1. Write to temp file
+                await this.ensureDirectoryExists(path.dirname(tempPath));
+                await fs.promises.writeFile(tempPath, JSON.stringify(updatedNode, null, 2));
+                // 2. Atomic rename temp → final (POSIX atomicity guarantee)
+                // This operation is guaranteed atomic by POSIX - either succeeds completely or fails
+                await fs.promises.rename(tempPath, filePath);
             }
             catch (error) {
-                // File doesn't exist yet - will create new
-                if (error.code !== 'ENOENT') {
-                    throw error;
+                // Clean up temp file on any error
+                try {
+                    await fs.promises.unlink(tempPath);
                 }
+                catch (cleanupError) {
+                    // Ignore cleanup errors - temp file may not exist
+                }
+                throw error;
             }
-            // Preserve id and vector, update only HNSW graph metadata
-            const updatedNode = {
-                ...existingNode, // Preserve all existing fields (id, vector, etc.)
-                level: hnswData.level,
-                connections: hnswData.connections
-            };
-            // ATOMIC WRITE SEQUENCE:
-            // 1. Write to temp file
-            await this.ensureDirectoryExists(path.dirname(tempPath));
-            await fs.promises.writeFile(tempPath, JSON.stringify(updatedNode, null, 2));
-            // 2. Atomic rename temp → final (POSIX atomicity guarantee)
-            // This operation is guaranteed atomic by POSIX - either succeeds completely or fails
-            // Multiple concurrent renames will serialize at the kernel level
-            await fs.promises.rename(tempPath, filePath);
         }
-        catch (error) {
-            // Clean up temp file on any error
-            try {
-                await fs.promises.unlink(tempPath);
-            }
-            catch (cleanupError) {
-                // Ignore cleanup errors - temp file may not exist
-            }
-            throw error;
+        finally {
+            // Release lock (ALWAYS runs, even if error thrown)
+            this.hnswLocks.delete(lockKey);
+            releaseLock();
         }
     }
     /**
@@ -2242,28 +2269,47 @@ export class FileSystemStorage extends BaseStorage {
     /**
      * Save HNSW system data (entry point, max level)
      *
-     * CRITICAL FIX (v4.10.1): Atomic write to prevent race conditions during concurrent updates
+     * CRITICAL FIX (v4.10.1): Mutex lock + atomic write to prevent race conditions
      */
     async saveHNSWSystem(systemData) {
         await this.ensureInitialized();
-        const filePath = path.join(this.systemDir, 'hnsw-system.json');
-        const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`;
+        const lockKey = 'hnsw/system';
+        // CRITICAL FIX (v4.10.1): Mutex lock to serialize system updates
+        // System data (entry point, max level) updated frequently during HNSW construction
+        // Without mutex, concurrent updates can lose data (same as entity-level problem)
+        // Wait for any pending system updates
+        while (this.hnswLocks.has(lockKey)) {
+            await this.hnswLocks.get(lockKey);
+        }
+        // Acquire lock
+        let releaseLock;
+        const lockPromise = new Promise(resolve => { releaseLock = resolve; });
+        this.hnswLocks.set(lockKey, lockPromise);
         try {
-            // Write to temp file
-            await this.ensureDirectoryExists(path.dirname(tempPath));
-            await fs.promises.writeFile(tempPath, JSON.stringify(systemData, null, 2));
-            // Atomic rename temp → final (POSIX atomicity guarantee)
-            await fs.promises.rename(tempPath, filePath);
-        }
-        catch (error) {
-            // Clean up temp file on any error
+            const filePath = path.join(this.systemDir, 'hnsw-system.json');
+            const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`;
             try {
-                await fs.promises.unlink(tempPath);
+                // Write to temp file
+                await this.ensureDirectoryExists(path.dirname(tempPath));
+                await fs.promises.writeFile(tempPath, JSON.stringify(systemData, null, 2));
+                // Atomic rename temp → final (POSIX atomicity guarantee)
+                await fs.promises.rename(tempPath, filePath);
             }
-            catch (cleanupError) {
-                // Ignore cleanup errors
+            catch (error) {
+                // Clean up temp file on any error
+                try {
+                    await fs.promises.unlink(tempPath);
+                }
+                catch (cleanupError) {
+                    // Ignore cleanup errors
+                }
+                throw error;
             }
-            throw error;
+        }
+        finally {
+            // Release lock
+            this.hnswLocks.delete(lockKey);
+            releaseLock();
         }
     }
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "4.10.0",
+  "version": "4.10.2",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",