@soulcraft/brainy 6.2.8 → 6.3.0

package/dist/brainy.js

@@ -18,7 +18,6 @@ import { TripleIntelligenceSystem } from './triple/TripleIntelligenceSystem.js';
 import { VirtualFileSystem } from './vfs/VirtualFileSystem.js';
 import { VersioningAPI } from './versioning/VersioningAPI.js';
 import { MetadataIndexManager } from './utils/metadataIndex.js';
-import { GraphAdjacencyIndex } from './graph/graphAdjacencyIndex.js';
 import { CommitBuilder } from './storage/cow/CommitObject.js';
 import { NULL_HASH } from './storage/cow/constants.js';
 import { createPipeline } from './streaming/pipeline.js';
@@ -127,8 +126,11 @@ export class Brainy {
             // Initialize core metadata index
             this.metadataIndex = new MetadataIndexManager(this.storage);
             await this.metadataIndex.init();
-            // Initialize core graph index
-            this.graphIndex = new GraphAdjacencyIndex(this.storage);
+            // v6.3.0: Get GraphAdjacencyIndex from storage (SINGLETON pattern)
+            // Storage owns the single instance, Brainy accesses it via getGraphIndex()
+            // This fixes the dual-ownership bug where Brainy and Storage had separate instances
+            // causing verbIdSet to be out of sync and VFS tree queries to fail
+            this.graphIndex = await this.storage.getGraphIndex();
             // Rebuild indexes if needed for existing data
             await this.rebuildIndexesIfNeeded();
             // Initialize augmentations
@@ -2198,8 +2200,10 @@ export class Brainy {
             // Fast rebuild for small indexes from COW storage (Metadata/Graph are fast)
             clone.metadataIndex = new MetadataIndexManager(clone.storage);
             await clone.metadataIndex.init();
-            clone.graphIndex = new GraphAdjacencyIndex(clone.storage);
-            await clone.graphIndex.rebuild();
+            clone.storage.graphIndex = undefined;
+            clone.storage.graphIndexPromise = undefined;
+            clone.graphIndex = await clone.storage.getGraphIndex();
+            // getGraphIndex() will rebuild automatically if data exists (via _initializeGraphIndex)
             // Setup augmentations
             clone.augmentationRegistry = this.setupAugmentations();
             await clone.augmentationRegistry.initializeAll({
@@ -2289,14 +2293,21 @@ export class Brainy {
             this.index = this.setupIndex();
             this.metadataIndex = new MetadataIndexManager(this.storage);
             await this.metadataIndex.init();
-            this.graphIndex = new GraphAdjacencyIndex(this.storage);
+            this.storage.invalidateGraphIndex();
+            this.graphIndex = await this.storage.getGraphIndex();
             // v5.7.7: Reset lazy loading state when switching branches
             // Indexes contain data from previous branch, must rebuild for new branch
             this.lazyRebuildCompleted = false;
             // Rebuild indexes from new branch data (force=true to override disableAutoRebuild)
             await this.rebuildIndexesIfNeeded(true);
-            // Re-initialize VFS for new branch
+            // v6.3.0: Clear VFS caches before recreating VFS for new branch
+            // UnifiedCache is global, so old branch's VFS path cache entries would persist
             if (this._vfs) {
+                // Clear old PathResolver's caches including UnifiedCache entries
+                if (this._vfs.pathResolver?.invalidateAllCaches) {
+                    this._vfs.pathResolver.invalidateAllCaches();
+                }
+                // Recreate VFS for new branch
                 this._vfs = new VirtualFileSystem(this);
                 await this._vfs.init();
             }

package/dist/graph/graphAdjacencyIndex.d.ts

@@ -51,8 +51,15 @@ export declare class GraphAdjacencyIndex {
     constructor(storage: StorageAdapter, config?: GraphIndexConfig);
     /**
      * Initialize the graph index (lazy initialization)
+     * v6.3.0: Added defensive auto-rebuild check for verbIdSet consistency
      */
     private ensureInitialized;
+    /**
+     * Populate verbIdSet from storage without full rebuild (v6.3.0)
+     * Lighter weight than full rebuild - only loads verb IDs, not all verb data
+     * @private
+     */
+    private populateVerbIdSetFromStorage;
     /**
      * Core API - Neighbor lookup with LSM-tree storage
      *

package/dist/graph/graphAdjacencyIndex.js

@@ -72,6 +72,7 @@ export class GraphAdjacencyIndex {
     }
     /**
      * Initialize the graph index (lazy initialization)
+     * v6.3.0: Added defensive auto-rebuild check for verbIdSet consistency
      */
     async ensureInitialized() {
         if (this.initialized) {
@@ -81,10 +82,51 @@ export class GraphAdjacencyIndex {
         await this.lsmTreeTarget.init();
         await this.lsmTreeVerbsBySource.init();
         await this.lsmTreeVerbsByTarget.init();
+        // v6.3.0: Defensive check - if LSM-trees have data but verbIdSet is empty,
+        // the index was created without proper rebuild (shouldn't happen with singleton
+        // pattern but protects against edge cases and future refactoring)
+        const lsmTreeSize = this.lsmTreeVerbsBySource.size();
+        if (lsmTreeSize > 0 && this.verbIdSet.size === 0) {
+            prodLog.warn(`GraphAdjacencyIndex: LSM-trees have ${lsmTreeSize} relationships but verbIdSet is empty. ` +
+                `Triggering auto-rebuild to restore consistency.`);
+            // Note: We don't await rebuild() here to avoid infinite loop
+            // (rebuild calls ensureInitialized). Instead, we'll populate verbIdSet
+            // by loading all verb IDs from storage.
+            await this.populateVerbIdSetFromStorage();
+        }
         // Start auto-flush timer after initialization
         this.startAutoFlush();
         this.initialized = true;
     }
+    /**
+     * Populate verbIdSet from storage without full rebuild (v6.3.0)
+     * Lighter weight than full rebuild - only loads verb IDs, not all verb data
+     * @private
+     */
+    async populateVerbIdSetFromStorage() {
+        prodLog.info('GraphAdjacencyIndex: Populating verbIdSet from storage...');
+        const startTime = Date.now();
+        // Use pagination to load all verb IDs
+        let hasMore = true;
+        let cursor = undefined;
+        let count = 0;
+        while (hasMore) {
+            const result = await this.storage.getVerbs({
+                pagination: { limit: 10000, cursor }
+            });
+            for (const verb of result.items) {
+                this.verbIdSet.add(verb.id);
+                // Also update counts
+                const verbType = verb.verb || 'unknown';
+                this.relationshipCountsByType.set(verbType, (this.relationshipCountsByType.get(verbType) || 0) + 1);
+                count++;
+            }
+            hasMore = result.hasMore;
+            cursor = result.nextCursor;
+        }
+        const elapsed = Date.now() - startTime;
+        prodLog.info(`GraphAdjacencyIndex: Populated verbIdSet with ${count} verb IDs in ${elapsed}ms`);
+    }
     /**
      * Core API - Neighbor lookup with LSM-tree storage
      *

package/dist/storage/baseStorage.d.ts

@@ -82,6 +82,13 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * @public
      */
     rebuildGraphIndex(): Promise<void>;
+    /**
+     * Invalidate GraphAdjacencyIndex (v6.3.0)
+     * Call this when switching branches or clearing data to force re-creation
+     * The next getGraphIndex() call will create a fresh instance and rebuild
+     * @public
+     */
+    invalidateGraphIndex(): void;
     /**
      * Ensure the storage adapter is initialized
      */

package/dist/storage/baseStorage.js

@@ -185,18 +185,12 @@ export class BaseStorage extends BaseStorageAdapter {
         try {
             // Load type statistics from storage (if they exist)
             await this.loadTypeStatistics();
-            // v6.0.0: Create GraphAdjacencyIndex (lazy-loaded, no rebuild)
-            // LSM-trees are initialized on first use via ensureInitialized()
-            // Index is populated incrementally as verbs are added via addVerb()
-            try {
-                prodLog.debug('[BaseStorage] Creating GraphAdjacencyIndex...');
-                this.graphIndex = new GraphAdjacencyIndex(this);
-                prodLog.debug(`[BaseStorage] GraphAdjacencyIndex instantiated (lazy-loaded), graphIndex=${!!this.graphIndex}`);
-            }
-            catch (error) {
-                prodLog.error('[BaseStorage] Failed to create GraphAdjacencyIndex:', error);
-                throw error;
-            }
+            // v6.3.0: GraphAdjacencyIndex is now SINGLETON via getGraphIndex()
+            // - Removed direct creation here to fix dual-ownership bug
+            // - GraphAdjacencyIndex will be created lazily on first getGraphIndex() call
+            // - This ensures there's only ONE instance per storage adapter
+            // - See: https://github.com/soulcraftlabs/brainy/issues/vfs-corruption
+            prodLog.debug('[BaseStorage] init() complete - GraphAdjacencyIndex will be created via getGraphIndex()');
         }
         catch (error) {
             // Reset flag on failure to allow retry
@@ -210,13 +204,28 @@ export class BaseStorage extends BaseStorageAdapter {
      * @public
      */
     async rebuildGraphIndex() {
-        if (!this.graphIndex) {
-            throw new Error('GraphAdjacencyIndex not initialized');
-        }
+        const index = await this.getGraphIndex();
         prodLog.info('[BaseStorage] Rebuilding graph index from existing data...');
-        await this.graphIndex.rebuild();
+        await index.rebuild();
         prodLog.info('[BaseStorage] Graph index rebuild complete');
     }
+    /**
+     * Invalidate GraphAdjacencyIndex (v6.3.0)
+     * Call this when switching branches or clearing data to force re-creation
+     * The next getGraphIndex() call will create a fresh instance and rebuild
+     * @public
+     */
+    invalidateGraphIndex() {
+        if (this.graphIndex) {
+            prodLog.info('[BaseStorage] Invalidating GraphAdjacencyIndex for branch switch/clear');
+            // Stop any pending operations
+            if (typeof this.graphIndex.stopAutoFlush === 'function') {
+                this.graphIndex.stopAutoFlush();
+            }
+            this.graphIndex = undefined;
+            this.graphIndexPromise = undefined;
+        }
+    }
     /**
      * Ensure the storage adapter is initialized
      */
@@ -1276,6 +1285,42 @@ export class BaseStorage extends BaseStorageAdapter {
                     nextCursor
                 };
             }
+            // v6.2.9: Fast path for SINGLE sourceId + verbType combo (common VFS pattern)
+            // This avoids the slow type-iteration fallback for VFS operations
+            // NOTE: Only use fast path for single sourceId to avoid incomplete results
+            const isSingleSourceId = options.filter.sourceId &&
+                !Array.isArray(options.filter.sourceId);
+            if (isSingleSourceId &&
+                options.filter.verbType &&
+                !options.filter.targetId &&
+                !options.filter.service &&
+                !options.filter.metadata) {
+                const sourceId = options.filter.sourceId;
+                const verbTypes = Array.isArray(options.filter.verbType)
+                    ? options.filter.verbType
+                    : [options.filter.verbType];
+                prodLog.debug(`[BaseStorage] getVerbs: Using fast path for sourceId=${sourceId}, verbTypes=${verbTypes.join(',')}`);
+                // Get verbs by source (uses GraphAdjacencyIndex if available)
+                const verbsBySource = await this.getVerbsBySource_internal(sourceId);
+                // Filter by verbType in memory (fast - usually small number of verbs per source)
+                const filtered = verbsBySource.filter(v => verbTypes.includes(v.verb));
+                // Apply pagination
+                const paginatedVerbs = filtered.slice(offset, offset + limit);
+                const hasMore = offset + limit < filtered.length;
+                // Set next cursor if there are more items
+                let nextCursor = undefined;
+                if (hasMore && paginatedVerbs.length > 0) {
+                    const lastItem = paginatedVerbs[paginatedVerbs.length - 1];
+                    nextCursor = lastItem.id;
+                }
+                prodLog.debug(`[BaseStorage] getVerbs: Fast path returned ${filtered.length} verbs (${paginatedVerbs.length} after pagination)`);
+                return {
+                    items: paginatedVerbs,
+                    totalCount: filtered.length,
+                    hasMore,
+                    nextCursor
+                };
+            }
         }
         // For more complex filtering or no filtering, use a paginated approach
         // that avoids loading all verbs into memory at once
@@ -1336,14 +1381,32 @@ export class BaseStorage extends BaseStorageAdapter {
             // Only use type-skipping optimization if counts are non-zero (reliable)
             const totalVerbCountFromArray = this.verbCountsByType.reduce((sum, c) => sum + c, 0);
             const useOptimization = totalVerbCountFromArray > 0;
+            // v6.2.9 BUG FIX: Pre-compute requested verb types to avoid skipping them
+            // When a specific verbType filter is provided, we MUST check that type
+            // even if verbCountsByType shows 0 (counts can be stale after restart)
+            const requestedVerbTypes = options?.filter?.verbType;
+            const requestedVerbTypesSet = requestedVerbTypes
+                ? new Set(Array.isArray(requestedVerbTypes) ? requestedVerbTypes : [requestedVerbTypes])
+                : null;
             // Iterate through all 127 verb types (Stage 3 CANONICAL) with early termination
             // OPTIMIZATION: Skip types with zero count (only if counts are reliable)
             for (let i = 0; i < VERB_TYPE_COUNT && collectedVerbs.length < targetCount; i++) {
-                // Skip empty types for performance (but only if optimization is enabled)
-                if (useOptimization && this.verbCountsByType[i] === 0) {
+                const type = TypeUtils.getVerbFromIndex(i);
+                // v6.2.9 FIX: Never skip a type that's explicitly requested in the filter
+                // This fixes VFS bug where Contains relationships were skipped after restart
+                // when verbCountsByType[Contains] was 0 due to stale statistics
+                const isRequestedType = requestedVerbTypesSet?.has(type) ?? false;
+                const countIsZero = this.verbCountsByType[i] === 0;
+                // Skip empty types for performance (but only if optimization is enabled AND not requested)
+                if (useOptimization && countIsZero && !isRequestedType) {
                     continue;
                 }
-                const type = TypeUtils.getVerbFromIndex(i);
+                // v6.2.9: Log when we DON'T skip a requested type that would have been skipped
+                // This helps diagnose stale statistics issues in production
+                if (useOptimization && countIsZero && isRequestedType) {
+                    prodLog.debug(`[BaseStorage] getVerbs: NOT skipping type=${type} despite count=0 (type was explicitly requested). ` +
+                        `Statistics may be stale - consider running rebuildTypeCounts().`);
+                }
                 try {
                     const verbsOfType = await this.getVerbsByType_internal(type);
                     // Apply filtering inline (memory efficient)
@@ -2169,8 +2232,11 @@ export class BaseStorage extends BaseStorageAdapter {
         this.nounCountsByType[typeIndex]++;
         // COW-aware write (v5.0.1): Use COW helper for branch isolation
         await this.writeObjectToBranch(path, noun);
-        // Periodically save statistics (every 100 saves)
-        if (this.nounCountsByType[typeIndex] % 100 === 0) {
+        // Periodically save statistics
+        // v6.2.9: Also save on first noun of each type to ensure low-count types are tracked
+        const shouldSave = this.nounCountsByType[typeIndex] === 1 || // First noun of type
+            this.nounCountsByType[typeIndex] % 100 === 0; // Every 100th
+        if (shouldSave) {
             await this.saveTypeStatistics();
         }
     }
@@ -2254,30 +2320,18 @@ export class BaseStorage extends BaseStorageAdapter {
         this.verbCountsByType[typeIndex]++;
         // COW-aware write (v5.0.1): Use COW helper for branch isolation
         await this.writeObjectToBranch(path, verb);
-        // v6.0.0: Update GraphAdjacencyIndex incrementally (always available after init())
-        // GraphAdjacencyIndex.addVerb() calls ensureInitialized() automatically
-        if (this.graphIndex) {
-            prodLog.debug(`[BaseStorage] Updating GraphAdjacencyIndex with verb ${verb.id}`);
-            await this.graphIndex.addVerb({
-                id: verb.id,
-                sourceId: verb.sourceId,
-                targetId: verb.targetId,
-                vector: verb.vector,
-                source: verb.sourceId,
-                target: verb.targetId,
-                verb: verb.verb,
-                type: verb.verb,
-                createdAt: { seconds: Math.floor(Date.now() / 1000), nanoseconds: 0 },
-                updatedAt: { seconds: Math.floor(Date.now() / 1000), nanoseconds: 0 },
-                createdBy: { augmentation: 'storage', version: '6.0.0' }
-            });
-            prodLog.debug(`[BaseStorage] GraphAdjacencyIndex updated successfully`);
-        }
-        else {
-            prodLog.warn(`[BaseStorage] graphIndex is null, cannot update index for verb ${verb.id}`);
-        }
+        // v6.3.0: GraphAdjacencyIndex updates are now handled EXCLUSIVELY by Brainy.relate()
+        // via AddToGraphIndexOperation in the transaction system. This provides:
+        // 1. Singleton pattern - only one graphIndex instance exists (via getGraphIndex())
+        // 2. Transaction rollback - if relate() fails, index update is rolled back
+        // 3. No double-counting - prevents duplicate addVerb() calls
+        // REMOVED: Direct graphIndex.addVerb() call that caused dual-ownership bugs
         // Periodically save statistics
-        if (this.verbCountsByType[typeIndex] % 100 === 0) {
+        // v6.2.9: Also save on first verb of each type to ensure low-count types are tracked
+        // This prevents stale statistics after restart for types with < 100 verbs (common for VFS)
+        const shouldSave = this.verbCountsByType[typeIndex] === 1 || // First verb of type
+            this.verbCountsByType[typeIndex] % 100 === 0; // Every 100th
+        if (shouldSave) {
             await this.saveTypeStatistics();
         }
     }

package/dist/utils/unifiedCache.d.ts

@@ -92,6 +92,13 @@ export declare class UnifiedCache {
      * Delete specific item from cache
      */
     delete(key: string): boolean;
+    /**
+     * Delete all items with keys starting with the given prefix
+     * v6.2.9: Added for VFS cache invalidation (fixes stale parent ID bug)
+     * @param prefix - The key prefix to match
+     * @returns Number of items deleted
+     */
+    deleteByPrefix(prefix: string): number;
     /**
      * Clear cache or specific type
      */

package/dist/utils/unifiedCache.js

@@ -304,6 +304,23 @@ export class UnifiedCache {
         }
         return false;
     }
+    /**
+     * Delete all items with keys starting with the given prefix
+     * v6.2.9: Added for VFS cache invalidation (fixes stale parent ID bug)
+     * @param prefix - The key prefix to match
+     * @returns Number of items deleted
+     */
+    deleteByPrefix(prefix) {
+        let deleted = 0;
+        for (const [key, item] of this.cache) {
+            if (key.startsWith(prefix)) {
+                this.currentSize -= item.size;
+                this.cache.delete(key);
+                deleted++;
+            }
+        }
+        return deleted;
+    }
     /**
      * Clear cache or specific type
      */

package/dist/vfs/PathResolver.d.ts

@@ -62,8 +62,16 @@ export declare class PathResolver {
      * Create a new path entry (for mkdir/writeFile)
      */
     createPath(path: string, entityId: string): Promise<void>;
+    /**
+     * Invalidate ALL caches (v6.3.0)
+     * Call this when switching branches (checkout), clearing data (clear), or forking
+     * This ensures no stale data from previous branch/state remains in cache
+     */
+    invalidateAllCaches(): void;
     /**
      * Invalidate cache entries for a path and its children
+     * v6.2.9 FIX: Also invalidates UnifiedCache to prevent stale entity IDs
+     * This fixes the "Source entity not found" bug after delete+recreate operations
      */
     invalidatePath(path: string, recursive?: boolean): void;
     /**

package/dist/vfs/PathResolver.js

@@ -252,28 +252,60 @@ export class PathResolver {
             this.parentCache.get(parentId).add(name);
         }
     }
+    /**
+     * Invalidate ALL caches (v6.3.0)
+     * Call this when switching branches (checkout), clearing data (clear), or forking
+     * This ensures no stale data from previous branch/state remains in cache
+     */
+    invalidateAllCaches() {
+        // Clear all local caches
+        this.pathCache.clear();
+        this.parentCache.clear();
+        this.hotPaths.clear();
+        // Clear all VFS entries from UnifiedCache
+        getGlobalCache().deleteByPrefix('vfs:path:');
+        // Reset statistics (optional but helpful for debugging)
+        this.cacheHits = 0;
+        this.cacheMisses = 0;
+        this.metadataIndexHits = 0;
+        this.metadataIndexMisses = 0;
+        this.graphTraversalFallbacks = 0;
+        prodLog.info('[PathResolver] All caches invalidated');
+    }
     /**
      * Invalidate cache entries for a path and its children
+     * v6.2.9 FIX: Also invalidates UnifiedCache to prevent stale entity IDs
+     * This fixes the "Source entity not found" bug after delete+recreate operations
      */
     invalidatePath(path, recursive = false) {
         const normalizedPath = this.normalizePath(path);
-        // Remove from all caches
+        // v6.2.9 FIX: Clear parent cache BEFORE deleting from pathCache
+        // (we need the entityId from the cache entry)
+        const cached = this.pathCache.get(normalizedPath);
+        if (cached) {
+            this.parentCache.delete(cached.entityId);
+        }
+        // Remove from local caches
         this.pathCache.delete(normalizedPath);
         this.hotPaths.delete(normalizedPath);
+        // v6.2.9 CRITICAL FIX: Also invalidate UnifiedCache (global LRU cache)
+        // This was missing before, causing stale entity IDs to be returned after delete
+        const cacheKey = `vfs:path:${normalizedPath}`;
+        getGlobalCache().delete(cacheKey);
         if (recursive) {
             // Remove all paths that start with this path
             const prefix = normalizedPath.endsWith('/') ? normalizedPath : normalizedPath + '/';
-            for (const [cachedPath] of this.pathCache) {
+            for (const [cachedPath, entry] of this.pathCache) {
                 if (cachedPath.startsWith(prefix)) {
                     this.pathCache.delete(cachedPath);
                     this.hotPaths.delete(cachedPath);
+                    // v6.2.9: Also clear parent cache for this entry
+                    this.parentCache.delete(entry.entityId);
                 }
             }
-        }
-        // Clear parent cache for the entity
-        const cached = this.pathCache.get(normalizedPath);
-        if (cached) {
-            this.parentCache.delete(cached.entityId);
+            // v6.2.9 CRITICAL FIX: Also invalidate UnifiedCache entries with this prefix
+            const globalCachePrefix = `vfs:path:${prefix}`;
+            getGlobalCache().deleteByPrefix(globalCachePrefix);
         }
     }
     /**

package/dist/vfs/semantic/SemanticPathResolver.d.ts

@@ -92,6 +92,12 @@ export declare class SemanticPathResolver {
      * Uses UnifiedCache's clear method
      */
     invalidateSemanticCache(): void;
+    /**
+     * Invalidate ALL caches (v6.3.0)
+     * Clears both traditional path cache AND semantic cache
+     * Call this when switching branches, clearing data, or forking
+     */
+    invalidateAllCaches(): void;
     /**
      * Cleanup resources
      */

package/dist/vfs/semantic/SemanticPathResolver.js

@@ -231,6 +231,17 @@ export class SemanticPathResolver {
     invalidateSemanticCache() {
         this.cache.clear();
     }
+    /**
+     * Invalidate ALL caches (v6.3.0)
+     * Clears both traditional path cache AND semantic cache
+     * Call this when switching branches, clearing data, or forking
+     */
+    invalidateAllCaches() {
+        // Clear traditional PathResolver caches (including UnifiedCache VFS entries)
+        this.pathResolver.invalidateAllCaches();
+        // Clear semantic cache
+        this.cache.clear();
+    }
     /**
      * Cleanup resources
      */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "6.2.8",
+  "version": "6.3.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",