@soulcraft/brainy 5.7.0 → 5.7.2

package/CHANGELOG.md

@@ -2,8 +2,85 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [5.7.2](https://github.com/soulcraftlabs/brainy/compare/v5.7.1...v5.7.2) (2025-11-12)
+
+
+### 🐛 Bug Fixes
+
+* resolve v5.7.x race condition with write-through cache (v5.7.2) ([732d23b](https://github.com/soulcraftlabs/brainy/commit/732d23bd2afb4ac9559a9beb7835e0f623065ff2))
+
+### [5.7.1](https://github.com/soulcraftlabs/brainy/compare/v5.7.0...v5.7.1) (2025-11-11)
+
+- fix: resolve v5.7.0 deadlock by restoring storage layer separation (v5.7.1) (eb9af45)
+
+
+## [5.7.1](https://github.com/soulcraftlabs/brainy/compare/v5.7.0...v5.7.1) (2025-11-11)
+
+### 🚨 CRITICAL BUG FIX
+
+**v5.7.0 caused complete production failure - ALL imports hung indefinitely. This hotfix restores functionality.**
+
+### Bug Description
+v5.7.0 introduced a circular dependency deadlock during GraphAdjacencyIndex initialization:
+- `GraphAdjacencyIndex.rebuild()` → `storage.getVerbs()`
+- `storage.getVerbsBySource_internal()` → `getGraphIndex()` (NEW in v5.7.0)
+- `getGraphIndex()` waiting for rebuild to complete
+- **DEADLOCK**: Each component waiting for the other
+
+### Symptoms
+- ❌ ALL imports hung at "Reading Data Structure" stage for 760+ seconds
+- ❌ `brain.add()` operations took 12+ seconds per entity (50x slower than expected)
+- ❌ No errors thrown - infinite wait
+- ❌ Zero entities imported successfully
+- ❌ 100% of users unable to import files
+
+### Root Cause
+v5.7.0 modified storage internal methods (`getVerbsBySource_internal`, `getVerbsByTarget_internal`) to use GraphAdjacencyIndex, creating tight coupling where:
+- Storage layer depends on index
+- Index depends on storage layer
+- Circular dependency = deadlock during initialization
+
+### Fix (Architectural)
+Reverted storage internals to v5.6.3 implementation:
+- ✅ Storage layer is now simple and has no index dependencies
+- ✅ GraphAdjacencyIndex can safely call storage.getVerbs() to rebuild
+- ✅ No circular dependency possible
+- ✅ Proper separation of concerns restored
+
+**Files changed**:
+- `src/storage/baseStorage.ts`: Reverted lines 2320-2444 to v5.6.3 implementation
+- `tests/regression/v5.7.0-deadlock.test.ts`: Added comprehensive regression tests
+
+### Performance Impact
+- Slightly slower GraphAdjacencyIndex initialization (one-time cost during rebuild)
+- High-level query operations still use optimized index
+- Import performance unaffected (writes don't trigger index initialization)
+- **NO breaking changes to public API**
+
+### Testing
+- ✅ 4 new regression tests verify no deadlock
+- ✅ All 1146 existing tests pass
+- ✅ Import + relationships complete in <1 second (not 760+ seconds)
+- ✅ No 12+ second delays per entity
+
+### Verification
+Workshop team (production users) should upgrade immediately:
+```bash
+npm install @soulcraft/brainy@5.7.1
+```
+
+Expected behavior after upgrade:
+- ✅ Imports work again
+- ✅ Fast entity creation (<100ms per entity)
+- ✅ No hangs or infinite waits
+- ✅ File operations responsive
+
+---
+
 ### [5.7.0](https://github.com/soulcraftlabs/brainy/compare/v5.6.3...v5.7.0) (2025-11-11)
 
+**⚠️ WARNING: This version has a critical deadlock bug. Use v5.7.1 instead.**
+
 - test: skip flaky concurrent relationship test (race condition in duplicate detection) (a71785b)
 - perf: optimize imports with background deduplication (12-24x speedup) (02c80a0)
 

package/dist/storage/baseStorage.d.ts

@@ -53,6 +53,7 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
     protected graphIndex?: GraphAdjacencyIndex;
     protected graphIndexPromise?: Promise<GraphAdjacencyIndex>;
     protected readOnly: boolean;
+    private writeCache;
     refManager?: RefManager;
     blobStorage?: BlobStorage;
     commitLog?: CommitLog;
@@ -488,7 +489,8 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
     protected getVerbsBySource_internal(sourceId: string): Promise<HNSWVerbWithMetadata[]>;
     /**
      * Get verbs by target (COW-aware implementation)
-     * v5.7.0: BILLION-SCALE OPTIMIZATION - Use GraphAdjacencyIndex for O(log n) lookup
+     * v5.7.1: Reverted to v5.6.3 implementation to fix circular dependency deadlock
+     * v5.4.0: Fixed to directly list verb files instead of directories
      */
     protected getVerbsByTarget_internal(targetId: string): Promise<HNSWVerbWithMetadata[]>;
     /**

package/dist/storage/baseStorage.js

@@ -74,6 +74,12 @@ export class BaseStorage extends BaseStorageAdapter {
         super(...arguments);
         this.isInitialized = false;
         this.readOnly = false;
+        // v5.7.2: Write-through cache for read-after-write consistency
+        // Guarantees that immediately after writeObjectToBranch(), readWithInheritance() returns the data
+        // Cache key: resolved branchPath (includes branch scope for COW isolation)
+        // Cache lifetime: write start → write completion (microseconds to milliseconds)
+        // Memory footprint: Typically <10 items (only in-flight writes), <1KB total
+        this.writeCache = new Map();
         this.currentBranch = 'main';
         this.cowEnabled = false;
         // Type-first indexing support (v5.4.0)
@@ -353,7 +359,17 @@ export class BaseStorage extends BaseStorageAdapter {
      */
     async writeObjectToBranch(path, data, branch) {
         const branchPath = this.resolveBranchPath(path, branch);
-        return this.writeObjectToPath(branchPath, data);
+        // v5.7.2: Add to write cache BEFORE async write (guarantees read-after-write consistency)
+        // This ensures readWithInheritance() returns data immediately, fixing "Source entity not found" bug
+        this.writeCache.set(branchPath, data);
+        try {
+            await this.writeObjectToPath(branchPath, data);
+        }
+        finally {
+            // v5.7.2: Remove from cache after write completes (success or failure)
+            // Small memory footprint: cache only holds in-flight writes (typically <10 items)
+            this.writeCache.delete(branchPath);
+        }
     }
     /**
      * Read object with inheritance from parent branches (COW layer)
@@ -362,12 +378,24 @@ export class BaseStorage extends BaseStorageAdapter {
      */
     async readWithInheritance(path, branch) {
         if (!this.cowEnabled) {
-            // COW disabled, direct read
+            // COW disabled: check write cache, then direct read
+            // v5.7.2: Check cache first for read-after-write consistency
+            const cachedData = this.writeCache.get(path);
+            if (cachedData !== undefined) {
+                return cachedData;
+            }
             return this.readObjectFromPath(path);
         }
         const targetBranch = branch || this.currentBranch || 'main';
-        // Try current branch first
         const branchPath = this.resolveBranchPath(path, targetBranch);
+        // v5.7.2: Check write cache FIRST (synchronous, instant)
+        // This guarantees read-after-write consistency within the same process
+        // Fixes bug: brain.add() → brain.relate() → "Source entity not found"
+        const cachedData = this.writeCache.get(branchPath);
+        if (cachedData !== undefined) {
+            return cachedData;
+        }
+        // Try current branch first
         let data = await this.readObjectFromPath(branchPath);
         if (data !== null) {
             return data; // Found in current branch
@@ -411,6 +439,9 @@ export class BaseStorage extends BaseStorageAdapter {
      */
     async deleteObjectFromBranch(path, branch) {
         const branchPath = this.resolveBranchPath(path, branch);
+        // v5.7.2: Remove from write cache immediately (before async delete)
+        // Ensures subsequent reads don't return stale cached data
+        this.writeCache.delete(branchPath);
         return this.deleteObjectFromPath(branchPath);
     }
     /**
@@ -1868,71 +1899,115 @@ export class BaseStorage extends BaseStorageAdapter {
      * v5.4.0: Fixed to directly list verb files instead of directories
      */
     async getVerbsBySource_internal(sourceId) {
-        // v5.7.0: BILLION-SCALE OPTIMIZATION - Use GraphAdjacencyIndex for O(log n) lookup
-        // Previous: O(total_verbs) - scanned all 127 verb types
-        // Now: O(log n) LSM-tree lookup + O(verbs_for_source) load
+        // v5.7.1: Reverted to v5.6.3 implementation to fix circular dependency deadlock
+        // v5.7.0 called getGraphIndex() here, creating deadlock during initialization:
+        //   GraphAdjacencyIndex.rebuild() → storage.getVerbs() → getVerbsBySource_internal() → getGraphIndex() → [deadlock]
+        // v5.4.0: Type-first implementation - scan across all verb types
+        // COW-aware: uses readWithInheritance for each verb
         await this.ensureInitialized();
-        const startTime = performance.now();
-        // Get GraphAdjacencyIndex (lazy-initialized)
-        const graphIndex = await this.getGraphIndex();
-        // O(log n) lookup with bloom filter optimization
-        const verbIds = await graphIndex.getVerbIdsBySource(sourceId);
-        // Load each verb by ID (uses existing optimized getVerb())
         const results = [];
-        for (const verbId of verbIds) {
+        // Iterate through all verb types
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const type = TypeUtils.getVerbFromIndex(i);
+            const typeDir = `entities/verbs/${type}/vectors`;
             try {
-                const verb = await this.getVerb(verbId);
-                if (verb) {
-                    results.push(verb);
+                // v5.4.0 FIX: List all verb files directly (not shard directories)
+                // listObjectsInBranch returns full paths to .json files, not directories
+                const verbFiles = await this.listObjectsInBranch(typeDir);
+                for (const verbPath of verbFiles) {
+                    // Skip if not a .json file
+                    if (!verbPath.endsWith('.json'))
+                        continue;
+                    try {
+                        const verb = await this.readWithInheritance(verbPath);
+                        if (verb && verb.sourceId === sourceId) {
+                            // v5.4.0: Use proper path helper instead of string replacement
+                            const metadataPath = getVerbMetadataPath(type, verb.id);
+                            const metadata = await this.readWithInheritance(metadataPath);
+                            // v5.4.0: Extract standard fields from metadata to top-level (like nouns)
+                            results.push({
+                                ...verb,
+                                weight: metadata?.weight,
+                                confidence: metadata?.confidence,
+                                createdAt: metadata?.createdAt
+                                    ? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
+                                    : Date.now(),
+                                updatedAt: metadata?.updatedAt
+                                    ? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
+                                    : Date.now(),
+                                service: metadata?.service,
+                                createdBy: metadata?.createdBy,
+                                metadata: metadata || {}
+                            });
+                        }
+                    }
+                    catch (error) {
+                        // Skip verbs that fail to load
+                    }
                 }
             }
             catch (error) {
-                // Skip verbs that fail to load (handles deleted/corrupted verbs gracefully)
+                // Skip types that have no data
             }
         }
-        const elapsed = performance.now() - startTime;
-        // Performance monitoring - should be 100-10,000x faster than old O(n) scan
-        if (elapsed > 50.0) {
-            prodLog.warn(`getVerbsBySource_internal: Slow query for ${sourceId} ` +
-                `(${verbIds.length} verbs, ${elapsed.toFixed(2)}ms). ` +
-                `Expected <50ms with index optimization.`);
-        }
         return results;
     }
     /**
      * Get verbs by target (COW-aware implementation)
-     * v5.7.0: BILLION-SCALE OPTIMIZATION - Use GraphAdjacencyIndex for O(log n) lookup
+     * v5.7.1: Reverted to v5.6.3 implementation to fix circular dependency deadlock
+     * v5.4.0: Fixed to directly list verb files instead of directories
      */
     async getVerbsByTarget_internal(targetId) {
-        // v5.7.0: BILLION-SCALE OPTIMIZATION - Use GraphAdjacencyIndex for O(log n) lookup
-        // Previous: O(total_verbs) - scanned all 127 verb types
-        // Now: O(log n) LSM-tree lookup + O(verbs_for_target) load
+        // v5.7.1: Reverted to v5.6.3 implementation to fix circular dependency deadlock
+        // v5.7.0 called getGraphIndex() here, creating deadlock during initialization
+        // v5.4.0: Type-first implementation - scan across all verb types
+        // COW-aware: uses readWithInheritance for each verb
         await this.ensureInitialized();
-        const startTime = performance.now();
-        // Get GraphAdjacencyIndex (lazy-initialized)
-        const graphIndex = await this.getGraphIndex();
-        // O(log n) lookup with bloom filter optimization
-        const verbIds = await graphIndex.getVerbIdsByTarget(targetId);
-        // Load each verb by ID (uses existing optimized getVerb())
         const results = [];
-        for (const verbId of verbIds) {
+        // Iterate through all verb types
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const type = TypeUtils.getVerbFromIndex(i);
+            const typeDir = `entities/verbs/${type}/vectors`;
             try {
-                const verb = await this.getVerb(verbId);
-                if (verb) {
-                    results.push(verb);
+                // v5.4.0 FIX: List all verb files directly (not shard directories)
+                // listObjectsInBranch returns full paths to .json files, not directories
+                const verbFiles = await this.listObjectsInBranch(typeDir);
+                for (const verbPath of verbFiles) {
+                    // Skip if not a .json file
+                    if (!verbPath.endsWith('.json'))
+                        continue;
+                    try {
+                        const verb = await this.readWithInheritance(verbPath);
+                        if (verb && verb.targetId === targetId) {
+                            // v5.4.0: Use proper path helper instead of string replacement
+                            const metadataPath = getVerbMetadataPath(type, verb.id);
+                            const metadata = await this.readWithInheritance(metadataPath);
+                            // v5.4.0: Extract standard fields from metadata to top-level (like nouns)
+                            results.push({
+                                ...verb,
+                                weight: metadata?.weight,
+                                confidence: metadata?.confidence,
+                                createdAt: metadata?.createdAt
+                                    ? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
+                                    : Date.now(),
+                                updatedAt: metadata?.updatedAt
+                                    ? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
+                                    : Date.now(),
+                                service: metadata?.service,
+                                createdBy: metadata?.createdBy,
+                                metadata: metadata || {}
+                            });
+                        }
+                    }
+                    catch (error) {
+                        // Skip verbs that fail to load
+                    }
                 }
             }
             catch (error) {
-                // Skip verbs that fail to load (handles deleted/corrupted verbs gracefully)
+                // Skip types that have no data
             }
         }
-        const elapsed = performance.now() - startTime;
-        // Performance monitoring - should be 100-10,000x faster than old O(n) scan
-        if (elapsed > 50.0) {
-            prodLog.warn(`getVerbsByTarget_internal: Slow query for ${targetId} ` +
-                `(${verbIds.length} verbs, ${elapsed.toFixed(2)}ms). ` +
-                `Expected <50ms with index optimization.`);
-        }
         return results;
     }
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "5.7.0",
+  "version": "5.7.2",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",