@soulcraft/brainy 4.1.1 → 4.1.2

package/CHANGELOG.md

@@ -2,6 +2,20 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [4.1.2](https://github.com/soulcraftlabs/brainy/compare/v4.1.1...v4.1.2) (2025-10-21)
+
+
+### 🐛 Bug Fixes
+
+* **storage**: resolve count synchronization race condition across all storage adapters ([798a694](https://github.com/soulcraftlabs/brainy/commit/798a694))
+  - Fixed critical bug where entity and relationship counts were not tracked correctly during add(), relate(), and import()
+  - Root cause: Race condition where count increment tried to read metadata before it was saved
+  - Fixed in baseStorage for all storage adapters (FileSystem, GCS, R2, Azure, Memory, OPFS, S3, TypeAware)
+  - Added verb type to VerbMetadata for proper count tracking
+  - Refactored verb count methods to prevent mutex deadlocks
+  - Added rebuildCounts utility to repair corrupted counts from actual storage data
+  - Added comprehensive integration tests (11 tests covering all operations)
+
 ### [4.1.1](https://github.com/soulcraftlabs/brainy/compare/v4.1.0...v4.1.1) (2025-10-20)
 
 

package/dist/brainy.js

@@ -648,7 +648,9 @@ export class Brainy {
         const relationVector = fromEntity.vector.map((v, i) => (v + toEntity.vector[i]) / 2);
         return this.augmentationRegistry.execute('relate', params, async () => {
             // v4.0.0: Prepare verb metadata
+            // CRITICAL (v4.1.2): Include verb type in metadata for count tracking
             const verbMetadata = {
+                verb: params.type, // Store verb type for count synchronization
                 weight: params.weight ?? 1.0,
                 ...(params.metadata || {}),
                 createdAt: Date.now()

package/dist/storage/adapters/azureBlobStorage.js

@@ -791,11 +791,8 @@ export class AzureBlobStorage extends BaseStorage {
             });
             // Update cache
             this.verbCacheManager.set(edge.id, edge);
-            // Increment verb count
-            const metadata = await this.getVerbMetadata(edge.id);
-            if (metadata && metadata.type) {
-                await this.incrementVerbCount(metadata.type);
-            }
+            // Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
+            // This fixes the race condition where metadata didn't exist yet
             this.logger.trace(`Edge ${edge.id} saved successfully`);
             this.releaseBackpressure(true, requestId);
         }

package/dist/storage/adapters/baseStorageAdapter.d.ts

@@ -320,15 +320,27 @@ export declare abstract class BaseStorageAdapter implements StorageAdapter {
      */
     protected decrementEntityCountSafe(type: string): Promise<void>;
     /**
-     * Increment verb count - O(1) operation with mutex protection
+     * Increment verb count - O(1) operation (v4.1.2: now synchronous)
+     * Protected by storage-specific mechanisms (mutex, distributed consensus, etc.)
+     * @param type The verb type
+     */
+    protected incrementVerbCount(type: string): void;
+    /**
+     * Thread-safe increment for verb counts (v4.1.2)
+     * Uses mutex for single-node, distributed consensus for multi-node
+     * @param type The verb type
+     */
+    protected incrementVerbCountSafe(type: string): Promise<void>;
+    /**
+     * Decrement verb count - O(1) operation (v4.1.2: now synchronous)
      * @param type The verb type
      */
-    protected incrementVerbCount(type: string): Promise<void>;
+    protected decrementVerbCount(type: string): void;
     /**
-     * Decrement verb count - O(1) operation with mutex protection
+     * Thread-safe decrement for verb counts (v4.1.2)
      * @param type The verb type
      */
-    protected decrementVerbCount(type: string): Promise<void>;
+    protected decrementVerbCountSafe(type: string): Promise<void>;
     /**
      * Detect if this storage adapter uses cloud storage (network I/O)
      * Cloud storage benefits from batching; local storage does not.

package/dist/storage/adapters/baseStorageAdapter.js

@@ -709,45 +709,61 @@ export class BaseStorageAdapter {
         });
     }
     /**
-     * Increment verb count - O(1) operation with mutex protection
+     * Increment verb count - O(1) operation (v4.1.2: now synchronous)
+     * Protected by storage-specific mechanisms (mutex, distributed consensus, etc.)
+     * @param type The verb type
+     */
+    incrementVerbCount(type) {
+        this.verbCounts.set(type, (this.verbCounts.get(type) || 0) + 1);
+        this.totalVerbCount++;
+        // Update cache
+        this.countCache.set('verbs_count', {
+            count: this.totalVerbCount,
+            timestamp: Date.now()
+        });
+    }
+    /**
+     * Thread-safe increment for verb counts (v4.1.2)
+     * Uses mutex for single-node, distributed consensus for multi-node
      * @param type The verb type
      */
-    async incrementVerbCount(type) {
+    async incrementVerbCountSafe(type) {
         const mutex = getGlobalMutex();
         await mutex.runExclusive(`count-verb-${type}`, async () => {
-            this.verbCounts.set(type, (this.verbCounts.get(type) || 0) + 1);
-            this.totalVerbCount++;
-            // Update cache
-            this.countCache.set('verbs_count', {
-                count: this.totalVerbCount,
-                timestamp: Date.now()
-            });
+            this.incrementVerbCount(type);
             // Smart batching (v3.32.3+): Adapts to storage type
             await this.scheduleCountPersist();
         });
     }
     /**
-     * Decrement verb count - O(1) operation with mutex protection
+     * Decrement verb count - O(1) operation (v4.1.2: now synchronous)
+     * @param type The verb type
+     */
+    decrementVerbCount(type) {
+        const current = this.verbCounts.get(type) || 0;
+        if (current > 1) {
+            this.verbCounts.set(type, current - 1);
+        }
+        else {
+            this.verbCounts.delete(type);
+        }
+        if (this.totalVerbCount > 0) {
+            this.totalVerbCount--;
+        }
+        // Update cache
+        this.countCache.set('verbs_count', {
+            count: this.totalVerbCount,
+            timestamp: Date.now()
+        });
+    }
+    /**
+     * Thread-safe decrement for verb counts (v4.1.2)
      * @param type The verb type
      */
-    async decrementVerbCount(type) {
+    async decrementVerbCountSafe(type) {
         const mutex = getGlobalMutex();
         await mutex.runExclusive(`count-verb-${type}`, async () => {
-            const current = this.verbCounts.get(type) || 0;
-            if (current > 1) {
-                this.verbCounts.set(type, current - 1);
-            }
-            else {
-                this.verbCounts.delete(type);
-            }
-            if (this.totalVerbCount > 0) {
-                this.totalVerbCount--;
-            }
-            // Update cache
-            this.countCache.set('verbs_count', {
-                count: this.totalVerbCount,
-                timestamp: Date.now()
-            });
+            this.decrementVerbCount(type);
             // Smart batching (v3.32.3+): Adapts to storage type
             await this.scheduleCountPersist();
         });

package/dist/storage/adapters/fileSystemStorage.js

@@ -179,8 +179,6 @@ export class FileSystemStorage extends BaseStorage {
      */
     async saveNode(node) {
         await this.ensureInitialized();
-        // Check if this is a new node to update counts
-        const isNew = !(await this.fileExists(this.getNodePath(node.id)));
         // Convert connections Map to a serializable format
         // CRITICAL: Only save lightweight vector data (no metadata)
         // Metadata is saved separately via saveNounMetadata() (2-file system)
@@ -194,17 +192,8 @@ export class FileSystemStorage extends BaseStorage {
         const filePath = this.getNodePath(node.id);
         await this.ensureDirectoryExists(path.dirname(filePath));
         await fs.promises.writeFile(filePath, JSON.stringify(serializableNode, null, 2));
-        // Update counts for new nodes (v4.0.0: load metadata separately)
-        if (isNew) {
-            // v4.0.0: Get type from separate metadata storage
-            const metadata = await this.getNounMetadata(node.id);
-            const type = metadata?.noun || 'default';
-            this.incrementEntityCount(type);
-            // Persist counts periodically (every 10 operations for efficiency)
-            if (this.totalNounCount % 10 === 0) {
-                await this.persistCounts();
-            }
-        }
+        // Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2)
+        // This fixes the race condition where metadata didn't exist yet
     }
     /**
      * Get a node from storage
@@ -354,8 +343,6 @@ export class FileSystemStorage extends BaseStorage {
      */
     async saveEdge(edge) {
         await this.ensureInitialized();
-        // Check if this is a new edge to update counts
-        const isNew = !(await this.fileExists(this.getVerbPath(edge.id)));
         // Convert connections Map to a serializable format
         // ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
         // These fields are essential for 90% of operations - no metadata lookup needed
@@ -373,14 +360,8 @@ export class FileSystemStorage extends BaseStorage {
         const filePath = this.getVerbPath(edge.id);
         await this.ensureDirectoryExists(path.dirname(filePath));
         await fs.promises.writeFile(filePath, JSON.stringify(serializableEdge, null, 2));
-        // Update verb count for new edges (production-scale optimizations)
-        if (isNew) {
-            this.totalVerbCount++;
-            // Persist counts periodically (every 10 operations for efficiency)
-            if (this.totalVerbCount % 10 === 0) {
-                this.persistCounts(); // Async persist, don't await
-            }
-        }
+        // Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
+        // This fixes the race condition where metadata didn't exist yet
     }
     /**
      * Get an edge from storage

package/dist/storage/adapters/gcsStorage.js

@@ -353,11 +353,8 @@ export class GcsStorage extends BaseStorage {
                 this.nounCacheManager.set(node.id, node);
             }
             // Note: Empty vectors are intentional during HNSW lazy mode - not logged
-            // Increment noun count
-            const metadata = await this.getNounMetadata(node.id);
-            if (metadata && metadata.type) {
-                await this.incrementEntityCountSafe(metadata.type);
-            }
+            // Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2)
+            // This fixes the race condition where metadata didn't exist yet
             this.logger.trace(`Node ${node.id} saved successfully`);
             this.releaseBackpressure(true, requestId);
         }
@@ -663,11 +660,8 @@ export class GcsStorage extends BaseStorage {
             });
             // Update cache
             this.verbCacheManager.set(edge.id, edge);
-            // Increment verb count
-            const metadata = await this.getVerbMetadata(edge.id);
-            if (metadata && metadata.type) {
-                await this.incrementVerbCount(metadata.type);
-            }
+            // Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
+            // This fixes the race condition where metadata didn't exist yet
             this.logger.trace(`Edge ${edge.id} saved successfully`);
             this.releaseBackpressure(true, requestId);
         }

package/dist/storage/adapters/memoryStorage.js

@@ -56,7 +56,8 @@ export class MemoryStorage extends BaseStorage {
         }
         // Save the noun directly in the nouns map
         this.nouns.set(noun.id, nounCopy);
-        // Note: Count tracking happens in saveNounMetadata since type info is in metadata now
+        // Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2)
+        // This fixes the race condition where metadata didn't exist yet
     }
     /**
      * Get a noun from storage (v4.0.0: returns pure vector only)

package/dist/storage/adapters/r2Storage.js

@@ -574,10 +574,8 @@ export class R2Storage extends BaseStorage {
                 ContentType: 'application/json'
             }));
             this.verbCacheManager.set(edge.id, edge);
-            const metadata = await this.getVerbMetadata(edge.id);
-            if (metadata && metadata.type) {
-                await this.incrementVerbCount(metadata.type);
-            }
+            // Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
+            // This fixes the race condition where metadata didn't exist yet
             this.releaseBackpressure(true, requestId);
         }
         catch (error) {

package/dist/storage/baseStorage.d.ts

@@ -231,6 +231,11 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
     /**
      * Internal method for saving noun metadata (v4.0.0: now typed)
      * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
+     *
+     * CRITICAL (v4.1.2): Count synchronization happens here
+     * This ensures counts are updated AFTER metadata exists, fixing the race condition
+     * where storage adapters tried to read metadata before it was saved.
+     *
      * @protected
      */
     protected saveNounMetadata_internal(id: string, metadata: NounMetadata): Promise<void>;
@@ -252,6 +257,13 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
     /**
      * Internal method for saving verb metadata (v4.0.0: now typed)
      * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
+     *
+     * CRITICAL (v4.1.2): Count synchronization happens here
+     * This ensures verb counts are updated AFTER metadata exists, fixing the race condition
+     * where storage adapters tried to read metadata before it was saved.
+     *
+     * Note: Verb type is now stored in both HNSWVerb (vector file) and VerbMetadata for count tracking
+     *
      * @protected
      */
     protected saveVerbMetadata_internal(id: string, metadata: VerbMetadata): Promise<void>;

package/dist/storage/baseStorage.js

@@ -706,12 +706,32 @@ export class BaseStorage extends BaseStorageAdapter {
     /**
      * Internal method for saving noun metadata (v4.0.0: now typed)
      * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
+     *
+     * CRITICAL (v4.1.2): Count synchronization happens here
+     * This ensures counts are updated AFTER metadata exists, fixing the race condition
+     * where storage adapters tried to read metadata before it was saved.
+     *
      * @protected
      */
     async saveNounMetadata_internal(id, metadata) {
         await this.ensureInitialized();
+        // Determine if this is a new entity by checking if metadata already exists
         const keyInfo = this.analyzeKey(id, 'noun-metadata');
-        return this.writeObjectToPath(keyInfo.fullPath, metadata);
+        const existingMetadata = await this.readObjectFromPath(keyInfo.fullPath);
+        const isNew = !existingMetadata;
+        // Save the metadata
+        await this.writeObjectToPath(keyInfo.fullPath, metadata);
+        // CRITICAL FIX (v4.1.2): Increment count for new entities
+        // This runs AFTER metadata is saved, guaranteeing type information is available
+        // Uses synchronous increment since storage operations are already serialized
+        // Fixes Bug #1: Count synchronization failure during add() and import()
+        if (isNew && metadata.noun) {
+            this.incrementEntityCount(metadata.noun);
+            // Persist counts asynchronously (fire and forget)
+            this.scheduleCountPersist().catch(() => {
+                // Ignore persist errors - will retry on next operation
+            });
+        }
     }
     /**
      * Get noun metadata from storage (v4.0.0: now typed)
@@ -742,12 +762,35 @@ export class BaseStorage extends BaseStorageAdapter {
     /**
      * Internal method for saving verb metadata (v4.0.0: now typed)
      * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
+     *
+     * CRITICAL (v4.1.2): Count synchronization happens here
+     * This ensures verb counts are updated AFTER metadata exists, fixing the race condition
+     * where storage adapters tried to read metadata before it was saved.
+     *
+     * Note: Verb type is now stored in both HNSWVerb (vector file) and VerbMetadata for count tracking
+     *
      * @protected
      */
     async saveVerbMetadata_internal(id, metadata) {
         await this.ensureInitialized();
+        // Determine if this is a new verb by checking if metadata already exists
         const keyInfo = this.analyzeKey(id, 'verb-metadata');
-        return this.writeObjectToPath(keyInfo.fullPath, metadata);
+        const existingMetadata = await this.readObjectFromPath(keyInfo.fullPath);
+        const isNew = !existingMetadata;
+        // Save the metadata
+        await this.writeObjectToPath(keyInfo.fullPath, metadata);
+        // CRITICAL FIX (v4.1.2): Increment verb count for new relationships
+        // This runs AFTER metadata is saved
+        // Verb type is now stored in metadata (as of v4.1.2) to avoid loading HNSWVerb
+        // Uses synchronous increment since storage operations are already serialized
+        // Fixes Bug #2: Count synchronization failure during relate() and import()
+        if (isNew && metadata.verb) {
+            this.incrementVerbCount(metadata.verb);
+            // Persist counts asynchronously (fire and forget)
+            this.scheduleCountPersist().catch(() => {
+                // Ignore persist errors - will retry on next operation
+            });
+        }
     }
     /**
      * Get verb metadata from storage (v4.0.0: now typed)

package/dist/utils/rebuildCounts.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Rebuild Counts Utility
+ *
+ * Scans storage and rebuilds counts.json from actual data
+ * Use this to fix databases affected by the v4.1.1 count synchronization bug
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+import type { BaseStorage } from '../storage/baseStorage.js';
+export interface RebuildCountsResult {
+    /** Total number of entities (nouns) found */
+    nounCount: number;
+    /** Total number of relationships (verbs) found */
+    verbCount: number;
+    /** Entity counts by type */
+    entityCounts: Map<string, number>;
+    /** Verb counts by type */
+    verbCounts: Map<string, number>;
+    /** Processing time in milliseconds */
+    duration: number;
+}
+/**
+ * Rebuild counts.json from actual storage data
+ *
+ * This scans all entities and relationships in storage and reconstructs
+ * the counts index from scratch. Use this to fix count desynchronization.
+ *
+ * @param storage - The storage adapter to rebuild counts for
+ * @returns Promise that resolves to rebuild statistics
+ *
+ * @example
+ * ```typescript
+ * const brain = new Brainy({ storage: { type: 'filesystem', path: './brainy-data' } })
+ * await brain.init()
+ *
+ * const result = await rebuildCounts(brain.storage)
+ * console.log(`Rebuilt counts: ${result.nounCount} nouns, ${result.verbCount} verbs`)
+ * ```
+ */
+export declare function rebuildCounts(storage: BaseStorage): Promise<RebuildCountsResult>;

package/dist/utils/rebuildCounts.js

@@ -0,0 +1,112 @@
+/**
+ * Rebuild Counts Utility
+ *
+ * Scans storage and rebuilds counts.json from actual data
+ * Use this to fix databases affected by the v4.1.1 count synchronization bug
+ *
+ * NO MOCKS - Production-ready implementation
+ */
+/**
+ * Rebuild counts.json from actual storage data
+ *
+ * This scans all entities and relationships in storage and reconstructs
+ * the counts index from scratch. Use this to fix count desynchronization.
+ *
+ * @param storage - The storage adapter to rebuild counts for
+ * @returns Promise that resolves to rebuild statistics
+ *
+ * @example
+ * ```typescript
+ * const brain = new Brainy({ storage: { type: 'filesystem', path: './brainy-data' } })
+ * await brain.init()
+ *
+ * const result = await rebuildCounts(brain.storage)
+ * console.log(`Rebuilt counts: ${result.nounCount} nouns, ${result.verbCount} verbs`)
+ * ```
+ */
+export async function rebuildCounts(storage) {
+    const startTime = Date.now();
+    console.log('🔧 Rebuilding counts from storage...');
+    const entityCounts = new Map();
+    const verbCounts = new Map();
+    let totalNouns = 0;
+    let totalVerbs = 0;
+    // Scan all nouns using pagination
+    console.log('📊 Scanning entities...');
+    // Check if pagination method exists
+    const storageWithPagination = storage;
+    if (typeof storageWithPagination.getNounsWithPagination !== 'function') {
+        throw new Error('Storage adapter does not support getNounsWithPagination');
+    }
+    let hasMore = true;
+    let cursor;
+    while (hasMore) {
+        const result = await storageWithPagination.getNounsWithPagination({ limit: 100, cursor });
+        for (const noun of result.items) {
+            const metadata = await storage.getNounMetadata(noun.id);
+            if (metadata?.noun) {
+                const entityType = metadata.noun;
+                entityCounts.set(entityType, (entityCounts.get(entityType) || 0) + 1);
+                totalNouns++;
+            }
+        }
+        hasMore = result.hasMore;
+        cursor = result.nextCursor;
+    }
+    console.log(`   Found ${totalNouns} entities across ${entityCounts.size} types`);
+    // Scan all verbs using pagination
+    console.log('🔗 Scanning relationships...');
+    if (typeof storageWithPagination.getVerbsWithPagination !== 'function') {
+        throw new Error('Storage adapter does not support getVerbsWithPagination');
+    }
+    hasMore = true;
+    cursor = undefined;
+    while (hasMore) {
+        const result = await storageWithPagination.getVerbsWithPagination({ limit: 100, cursor });
+        for (const verb of result.items) {
+            if (verb.verb) {
+                const verbType = verb.verb;
+                verbCounts.set(verbType, (verbCounts.get(verbType) || 0) + 1);
+                totalVerbs++;
+            }
+        }
+        hasMore = result.hasMore;
+        cursor = result.nextCursor;
+    }
+    console.log(`   Found ${totalVerbs} relationships across ${verbCounts.size} types`);
+    // Update storage adapter's in-memory counts FIRST
+    storageWithPagination.totalNounCount = totalNouns;
+    storageWithPagination.totalVerbCount = totalVerbs;
+    storageWithPagination.entityCounts = entityCounts;
+    storageWithPagination.verbCounts = verbCounts;
+    // Mark counts as pending persist (required for flushCounts to actually persist)
+    storageWithPagination.pendingCountPersist = true;
+    storageWithPagination.pendingCountOperations = 1;
+    // Persist counts using storage adapter's own persist method
+    // This ensures counts.json is written correctly (compressed or uncompressed)
+    await storageWithPagination.flushCounts();
+    const duration = Date.now() - startTime;
+    console.log(`✅ Counts rebuilt successfully in ${duration}ms`);
+    console.log(`   Entities: ${totalNouns}`);
+    console.log(`   Relationships: ${totalVerbs}`);
+    console.log('');
+    console.log('Entity breakdown:');
+    entityCounts.forEach((count, entityType) => {
+        console.log(`   ${entityType}: ${count}`);
+    });
+    if (verbCounts.size > 0) {
+        console.log('');
+        console.log('Relationship breakdown:');
+        verbCounts.forEach((count, verbType) => {
+            console.log(`   ${verbType}: ${count}`);
+        });
+    }
+    return {
+        nounCount: totalNouns,
+        verbCount: totalVerbs,
+        entityCounts,
+        verbCounts,
+        duration
+    };
+}
+//# sourceMappingURL=rebuildCounts.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "4.1.1",
+  "version": "4.1.2",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",