@soulcraft/brainy 6.2.9 → 6.3.1

package/CHANGELOG.md

@@ -2,6 +2,27 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [6.3.1](https://github.com/soulcraftlabs/brainy/compare/v6.3.0...v6.3.1) (2025-12-09)
+
+- fix(versioning): clean architecture with index pollution prevention (f145fa1)
+- chore(release): 6.3.0 - singleton GraphAdjacencyIndex architecture fix (292be1b)
+- fix(architecture): singleton GraphAdjacencyIndex via storage.getGraphIndex() (v6.3.0) (c15892e)
+- chore(release): 6.2.9 - fix critical VFS bugs (directory corruption) (810b756)
+- fix(vfs): resolve two critical VFS bugs causing directory listing corruption (2ba69ec)
+- chore(release): 6.2.8 - deferred HNSW persistence for 30-50× faster cloud adds (1da6048)
+- perf(hnsw): deferred persistence mode for 30-50× faster cloud storage adds (4d1d567)
+- chore(release): 6.2.7 - simplify cloud storage to always-on write buffering (a33b759)
+- perf(storage): simplify cloud adapters to always-on write buffering (26510ce)
+- chore(release): 6.2.6 - fix cloud storage read-after-write consistency (6449bb1)
+- fix(storage): populate cache before write buffer for read-after-write consistency (2d27bd0)
+- chore(release): 6.2.5 - fix counts.byType() accumulation bug (e4bbd7f)
+- fix(counts): counts.byType() returns inflated values due to accumulation bug (9456c2c)
+- chore(release): 6.2.4 - fix asOf() COW property name mismatch (ea53c11)
+- fix(cow): asOf() fails with "COW not enabled" due to property name mismatch (b3ae18b)
+- chore(release): 6.2.3 - fix counts.byType({ excludeVFS: true }) returning empty (0ba6da4)
+- fix(counts): counts.byType({ excludeVFS: true }) now returns correct type counts (9b2ff2d)
+
+
 ### [6.2.2](https://github.com/soulcraftlabs/brainy/compare/v6.2.1...v6.2.2) (2025-11-25)
 
 - refactor: remove 3,700+ LOC of unused HNSW implementations (e3146ce)

package/dist/brainy.d.ts

@@ -4,6 +4,7 @@
  * Beautiful, Professional, Planet-Scale, Fun to Use
  * NO STUBS, NO MOCKS, REAL IMPLEMENTATION
  */
+import { BaseStorage } from './storage/baseStorage.js';
 import { Vector } from './coreTypes.js';
 import { ImprovedNeuralAPI } from './neural/improvedNeuralAPI.js';
 import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js';
@@ -1629,6 +1630,60 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
         };
         density: number;
     }>;
+    /**
+     * Search entities by metadata filters (internal API)
+     * Used by versioning system for querying version metadata
+     *
+     * @param filters - Metadata filter object (same format as `where` in find())
+     * @returns Array of matching entities
+     * @internal
+     */
+    searchByMetadata(filters: Record<string, any>): Promise<any[]>;
+    /**
+     * Get raw noun metadata (internal API)
+     * Used by versioning system for reading entity state
+     *
+     * @param id - Entity ID
+     * @returns Noun metadata or null if not found
+     * @internal
+     */
+    getNounMetadata(id: string): Promise<any | null>;
+    /**
+     * Save raw noun metadata (internal API)
+     * Used by versioning system for storing version index entries
+     *
+     * @param id - Entity ID
+     * @param data - Metadata to save
+     * @internal
+     */
+    saveNounMetadata(id: string, data: any): Promise<void>;
+    /**
+     * Delete noun metadata (internal API)
+     * Used by versioning system for removing version index entries
+     *
+     * @param id - Entity ID
+     * @internal
+     */
+    deleteNounMetadata(id: string): Promise<void>;
+    /**
+     * Current branch name (internal API)
+     * Used by versioning system for branch-aware operations
+     * @internal
+     */
+    get currentBranch(): string;
+    /**
+     * Reference manager for COW commits (internal API)
+     * Used by versioning system for commit operations
+     * @internal
+     */
+    get refManager(): any;
+    /**
+     * Storage adapter (internal API)
+     * Used by versioning system for direct storage access
+     * Returns BaseStorage which has saveMetadata/getMetadata for key-value storage
+     * @internal
+     */
+    get storageAdapter(): BaseStorage;
     /**
      * Parse natural language query using advanced NLP with 220+ patterns
      * The embedding model is always available as it's core to Brainy's functionality

package/dist/brainy.js

@@ -18,7 +18,6 @@ import { TripleIntelligenceSystem } from './triple/TripleIntelligenceSystem.js';
 import { VirtualFileSystem } from './vfs/VirtualFileSystem.js';
 import { VersioningAPI } from './versioning/VersioningAPI.js';
 import { MetadataIndexManager } from './utils/metadataIndex.js';
-import { GraphAdjacencyIndex } from './graph/graphAdjacencyIndex.js';
 import { CommitBuilder } from './storage/cow/CommitObject.js';
 import { NULL_HASH } from './storage/cow/constants.js';
 import { createPipeline } from './streaming/pipeline.js';
@@ -127,8 +126,11 @@ export class Brainy {
             // Initialize core metadata index
             this.metadataIndex = new MetadataIndexManager(this.storage);
             await this.metadataIndex.init();
-            // Initialize core graph index
-            this.graphIndex = new GraphAdjacencyIndex(this.storage);
+            // v6.3.0: Get GraphAdjacencyIndex from storage (SINGLETON pattern)
+            // Storage owns the single instance, Brainy accesses it via getGraphIndex()
+            // This fixes the dual-ownership bug where Brainy and Storage had separate instances
+            // causing verbIdSet to be out of sync and VFS tree queries to fail
+            this.graphIndex = await this.storage.getGraphIndex();
             // Rebuild indexes if needed for existing data
             await this.rebuildIndexesIfNeeded();
             // Initialize augmentations
@@ -2198,8 +2200,10 @@ export class Brainy {
             // Fast rebuild for small indexes from COW storage (Metadata/Graph are fast)
             clone.metadataIndex = new MetadataIndexManager(clone.storage);
             await clone.metadataIndex.init();
-            clone.graphIndex = new GraphAdjacencyIndex(clone.storage);
-            await clone.graphIndex.rebuild();
+            clone.storage.graphIndex = undefined;
+            clone.storage.graphIndexPromise = undefined;
+            clone.graphIndex = await clone.storage.getGraphIndex();
+            // getGraphIndex() will rebuild automatically if data exists (via _initializeGraphIndex)
             // Setup augmentations
             clone.augmentationRegistry = this.setupAugmentations();
             await clone.augmentationRegistry.initializeAll({
@@ -2289,14 +2293,21 @@ export class Brainy {
             this.index = this.setupIndex();
             this.metadataIndex = new MetadataIndexManager(this.storage);
             await this.metadataIndex.init();
-            this.graphIndex = new GraphAdjacencyIndex(this.storage);
+            this.storage.invalidateGraphIndex();
+            this.graphIndex = await this.storage.getGraphIndex();
             // v5.7.7: Reset lazy loading state when switching branches
             // Indexes contain data from previous branch, must rebuild for new branch
             this.lazyRebuildCompleted = false;
             // Rebuild indexes from new branch data (force=true to override disableAutoRebuild)
             await this.rebuildIndexesIfNeeded(true);
-            // Re-initialize VFS for new branch
+            // v6.3.0: Clear VFS caches before recreating VFS for new branch
+            // UnifiedCache is global, so old branch's VFS path cache entries would persist
             if (this._vfs) {
+                // Clear old PathResolver's caches including UnifiedCache entries
+                if (this._vfs.pathResolver?.invalidateAllCaches) {
+                    this._vfs.pathResolver.invalidateAllCaches();
+                }
+                // Recreate VFS for new branch
                 this._vfs = new VirtualFileSystem(this);
                 await this._vfs.init();
             }
@@ -3543,6 +3554,92 @@ export class Brainy {
     async getStats(options) {
         return this.counts.getStats(options);
     }
+    // ============= INTERNAL VERSIONING API =============
+    // These methods are used by the versioning system (brain.versions.*)
+    // They expose internal storage and index operations needed for entity versioning
+    /**
+     * Search entities by metadata filters (internal API)
+     * Used by versioning system for querying version metadata
+     *
+     * @param filters - Metadata filter object (same format as `where` in find())
+     * @returns Array of matching entities
+     * @internal
+     */
+    async searchByMetadata(filters) {
+        await this.ensureInitialized();
+        const ids = await this.metadataIndex.getIdsForFilter(filters);
+        if (ids.length === 0)
+            return [];
+        const results = [];
+        const entitiesMap = await this.batchGet(ids);
+        for (const id of ids) {
+            const entity = entitiesMap.get(id);
+            if (entity) {
+                results.push(entity);
+            }
+        }
+        return results;
+    }
+    /**
+     * Get raw noun metadata (internal API)
+     * Used by versioning system for reading entity state
+     *
+     * @param id - Entity ID
+     * @returns Noun metadata or null if not found
+     * @internal
+     */
+    async getNounMetadata(id) {
+        await this.ensureInitialized();
+        return this.storage.getNounMetadata(id);
+    }
+    /**
+     * Save raw noun metadata (internal API)
+     * Used by versioning system for storing version index entries
+     *
+     * @param id - Entity ID
+     * @param data - Metadata to save
+     * @internal
+     */
+    async saveNounMetadata(id, data) {
+        await this.ensureInitialized();
+        await this.storage.saveNounMetadata(id, data);
+    }
+    /**
+     * Delete noun metadata (internal API)
+     * Used by versioning system for removing version index entries
+     *
+     * @param id - Entity ID
+     * @internal
+     */
+    async deleteNounMetadata(id) {
+        await this.ensureInitialized();
+        await this.storage.deleteNounMetadata(id);
+    }
+    /**
+     * Current branch name (internal API)
+     * Used by versioning system for branch-aware operations
+     * @internal
+     */
+    get currentBranch() {
+        return this.storage.currentBranch || 'main';
+    }
+    /**
+     * Reference manager for COW commits (internal API)
+     * Used by versioning system for commit operations
+     * @internal
+     */
+    get refManager() {
+        return this.storage.refManager;
+    }
+    /**
+     * Storage adapter (internal API)
+     * Used by versioning system for direct storage access
+     * Returns BaseStorage which has saveMetadata/getMetadata for key-value storage
+     * @internal
+     */
+    get storageAdapter() {
+        return this.storage;
+    }
     // ============= HELPER METHODS =============
     /**
      * Parse natural language query using advanced NLP with 220+ patterns

package/dist/graph/graphAdjacencyIndex.d.ts

@@ -51,8 +51,15 @@ export declare class GraphAdjacencyIndex {
     constructor(storage: StorageAdapter, config?: GraphIndexConfig);
     /**
      * Initialize the graph index (lazy initialization)
+     * v6.3.0: Added defensive auto-rebuild check for verbIdSet consistency
      */
     private ensureInitialized;
+    /**
+     * Populate verbIdSet from storage without full rebuild (v6.3.0)
+     * Lighter weight than full rebuild - only loads verb IDs, not all verb data
+     * @private
+     */
+    private populateVerbIdSetFromStorage;
     /**
      * Core API - Neighbor lookup with LSM-tree storage
      *

package/dist/graph/graphAdjacencyIndex.js

@@ -72,6 +72,7 @@ export class GraphAdjacencyIndex {
     }
     /**
      * Initialize the graph index (lazy initialization)
+     * v6.3.0: Added defensive auto-rebuild check for verbIdSet consistency
      */
     async ensureInitialized() {
         if (this.initialized) {
@@ -81,10 +82,51 @@ export class GraphAdjacencyIndex {
         await this.lsmTreeTarget.init();
         await this.lsmTreeVerbsBySource.init();
         await this.lsmTreeVerbsByTarget.init();
+        // v6.3.0: Defensive check - if LSM-trees have data but verbIdSet is empty,
+        // the index was created without proper rebuild (shouldn't happen with singleton
+        // pattern but protects against edge cases and future refactoring)
+        const lsmTreeSize = this.lsmTreeVerbsBySource.size();
+        if (lsmTreeSize > 0 && this.verbIdSet.size === 0) {
+            prodLog.warn(`GraphAdjacencyIndex: LSM-trees have ${lsmTreeSize} relationships but verbIdSet is empty. ` +
+                `Triggering auto-rebuild to restore consistency.`);
+            // Note: We don't await rebuild() here to avoid infinite loop
+            // (rebuild calls ensureInitialized). Instead, we'll populate verbIdSet
+            // by loading all verb IDs from storage.
+            await this.populateVerbIdSetFromStorage();
+        }
         // Start auto-flush timer after initialization
         this.startAutoFlush();
         this.initialized = true;
     }
+    /**
+     * Populate verbIdSet from storage without full rebuild (v6.3.0)
+     * Lighter weight than full rebuild - only loads verb IDs, not all verb data
+     * @private
+     */
+    async populateVerbIdSetFromStorage() {
+        prodLog.info('GraphAdjacencyIndex: Populating verbIdSet from storage...');
+        const startTime = Date.now();
+        // Use pagination to load all verb IDs
+        let hasMore = true;
+        let cursor = undefined;
+        let count = 0;
+        while (hasMore) {
+            const result = await this.storage.getVerbs({
+                pagination: { limit: 10000, cursor }
+            });
+            for (const verb of result.items) {
+                this.verbIdSet.add(verb.id);
+                // Also update counts
+                const verbType = verb.verb || 'unknown';
+                this.relationshipCountsByType.set(verbType, (this.relationshipCountsByType.get(verbType) || 0) + 1);
+                count++;
+            }
+            hasMore = result.hasMore;
+            cursor = result.nextCursor;
+        }
+        const elapsed = Date.now() - startTime;
+        prodLog.info(`GraphAdjacencyIndex: Populated verbIdSet with ${count} verb IDs in ${elapsed}ms`);
+    }
     /**
      * Core API - Neighbor lookup with LSM-tree storage
      *

package/dist/storage/baseStorage.d.ts

@@ -82,6 +82,13 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * @public
      */
     rebuildGraphIndex(): Promise<void>;
+    /**
+     * Invalidate GraphAdjacencyIndex (v6.3.0)
+     * Call this when switching branches or clearing data to force re-creation
+     * The next getGraphIndex() call will create a fresh instance and rebuild
+     * @public
+     */
+    invalidateGraphIndex(): void;
     /**
      * Ensure the storage adapter is initialized
      */

package/dist/storage/baseStorage.js

@@ -185,18 +185,12 @@ export class BaseStorage extends BaseStorageAdapter {
         try {
             // Load type statistics from storage (if they exist)
             await this.loadTypeStatistics();
-            // v6.0.0: Create GraphAdjacencyIndex (lazy-loaded, no rebuild)
-            // LSM-trees are initialized on first use via ensureInitialized()
-            // Index is populated incrementally as verbs are added via addVerb()
-            try {
-                prodLog.debug('[BaseStorage] Creating GraphAdjacencyIndex...');
-                this.graphIndex = new GraphAdjacencyIndex(this);
-                prodLog.debug(`[BaseStorage] GraphAdjacencyIndex instantiated (lazy-loaded), graphIndex=${!!this.graphIndex}`);
-            }
-            catch (error) {
-                prodLog.error('[BaseStorage] Failed to create GraphAdjacencyIndex:', error);
-                throw error;
-            }
+            // v6.3.0: GraphAdjacencyIndex is now SINGLETON via getGraphIndex()
+            // - Removed direct creation here to fix dual-ownership bug
+            // - GraphAdjacencyIndex will be created lazily on first getGraphIndex() call
+            // - This ensures there's only ONE instance per storage adapter
+            // - See: https://github.com/soulcraftlabs/brainy/issues/vfs-corruption
+            prodLog.debug('[BaseStorage] init() complete - GraphAdjacencyIndex will be created via getGraphIndex()');
         }
         catch (error) {
             // Reset flag on failure to allow retry
@@ -210,13 +204,28 @@ export class BaseStorage extends BaseStorageAdapter {
      * @public
      */
     async rebuildGraphIndex() {
-        if (!this.graphIndex) {
-            throw new Error('GraphAdjacencyIndex not initialized');
-        }
+        const index = await this.getGraphIndex();
         prodLog.info('[BaseStorage] Rebuilding graph index from existing data...');
-        await this.graphIndex.rebuild();
+        await index.rebuild();
         prodLog.info('[BaseStorage] Graph index rebuild complete');
     }
+    /**
+     * Invalidate GraphAdjacencyIndex (v6.3.0)
+     * Call this when switching branches or clearing data to force re-creation
+     * The next getGraphIndex() call will create a fresh instance and rebuild
+     * @public
+     */
+    invalidateGraphIndex() {
+        if (this.graphIndex) {
+            prodLog.info('[BaseStorage] Invalidating GraphAdjacencyIndex for branch switch/clear');
+            // Stop any pending operations
+            if (typeof this.graphIndex.stopAutoFlush === 'function') {
+                this.graphIndex.stopAutoFlush();
+            }
+            this.graphIndex = undefined;
+            this.graphIndexPromise = undefined;
+        }
+    }
     /**
      * Ensure the storage adapter is initialized
      */
@@ -2311,28 +2320,12 @@ export class BaseStorage extends BaseStorageAdapter {
         this.verbCountsByType[typeIndex]++;
         // COW-aware write (v5.0.1): Use COW helper for branch isolation
         await this.writeObjectToBranch(path, verb);
-        // v6.0.0: Update GraphAdjacencyIndex incrementally (always available after init())
-        // GraphAdjacencyIndex.addVerb() calls ensureInitialized() automatically
-        if (this.graphIndex) {
-            prodLog.debug(`[BaseStorage] Updating GraphAdjacencyIndex with verb ${verb.id}`);
-            await this.graphIndex.addVerb({
-                id: verb.id,
-                sourceId: verb.sourceId,
-                targetId: verb.targetId,
-                vector: verb.vector,
-                source: verb.sourceId,
-                target: verb.targetId,
-                verb: verb.verb,
-                type: verb.verb,
-                createdAt: { seconds: Math.floor(Date.now() / 1000), nanoseconds: 0 },
-                updatedAt: { seconds: Math.floor(Date.now() / 1000), nanoseconds: 0 },
-                createdBy: { augmentation: 'storage', version: '6.0.0' }
-            });
-            prodLog.debug(`[BaseStorage] GraphAdjacencyIndex updated successfully`);
-        }
-        else {
-            prodLog.warn(`[BaseStorage] graphIndex is null, cannot update index for verb ${verb.id}`);
-        }
+        // v6.3.0: GraphAdjacencyIndex updates are now handled EXCLUSIVELY by Brainy.relate()
+        // via AddToGraphIndexOperation in the transaction system. This provides:
+        // 1. Singleton pattern - only one graphIndex instance exists (via getGraphIndex())
+        // 2. Transaction rollback - if relate() fails, index update is rolled back
+        // 3. No double-counting - prevents duplicate addVerb() calls
+        // REMOVED: Direct graphIndex.addVerb() call that caused dual-ownership bugs
         // Periodically save statistics
         // v6.2.9: Also save on first verb of each type to ensure low-count types are tracked
         // This prevents stale statistics after restart for types with < 100 verbs (common for VFS)

package/dist/versioning/VersionIndex.d.ts

@@ -1,33 +1,24 @@
 /**
- * VersionIndex - Fast Version Lookup Using Existing Index Infrastructure (v5.3.0)
+ * VersionIndex - Pure Key-Value Version Storage (v6.3.0)
  *
- * Integrates with Brainy's existing index system:
- * - Uses MetadataIndexManager for field indexing
- * - Leverages UnifiedCache for memory management
- * - Uses EntityIdMapper for efficient ID handling
- * - Uses ChunkManager for adaptive chunking
- * - Leverages Roaring Bitmaps for fast set operations
+ * Stores version metadata using simple key-value storage:
+ * - Version list per entity: __version_meta_{entityId}_{branch}
+ * - Content stored separately by VersionStorage
  *
- * Version metadata is stored as regular entities with type='_version'
- * This allows us to use existing index infrastructure without modification!
- *
- * Fields indexed:
- * - versionEntityId: Entity being versioned
- * - versionBranch: Branch version was created on
- * - versionNumber: Version number
- * - versionTag: Optional user tag
- * - versionTimestamp: Creation timestamp
- * - versionCommitHash: Commit hash
+ * Key Design Decisions:
+ * - Versions are NOT entities (no brain.add())
+ * - Versions do NOT pollute find() results
+ * - Simple O(1) lookups per entity
+ * - Versions per entity is typically small (10-1000)
  *
  * NO MOCKS - Production implementation
  */
-import type { EntityVersion } from './VersionManager.js';
-import type { VersionQuery } from './VersionManager.js';
+import type { EntityVersion, VersionQuery } from './VersionManager.js';
 /**
- * VersionIndex - Version lookup and querying using existing indexes
+ * VersionIndex - Pure key-value version metadata storage
  *
- * Strategy: Store version metadata as special entities with type='_version'
- * This leverages ALL existing index infrastructure automatically!
+ * Uses simple JSON storage instead of creating entities.
+ * This ensures versions never appear in find() results.
  */
 export declare class VersionIndex {
     private brain;
@@ -35,25 +26,21 @@ export declare class VersionIndex {
     constructor(brain: any);
     /**
      * Initialize version index
-     *
-     * No special setup needed - we use existing entity storage and indexes!
      */
     initialize(): Promise<void>;
     /**
      * Add version to index
      *
-     * Stores version metadata as a special entity with type='_version'
-     * This automatically indexes it using existing MetadataIndexManager!
+     * Stores version entry in key-value storage.
+     * Handles deduplication by content hash.
      *
-     * @param version Version metadata
+     * @param version Version metadata to store
      */
     addVersion(version: EntityVersion): Promise<void>;
     /**
      * Get versions for an entity
      *
-     * Uses existing MetadataIndexManager to query efficiently!
-     *
-     * @param query Version query
+     * @param query Version query with filters
      * @returns List of versions (newest first)
      */
     getVersions(query: VersionQuery): Promise<EntityVersion[]>;
@@ -92,35 +79,43 @@ export declare class VersionIndex {
      */
     removeVersion(entityId: string, version: number, branch: string): Promise<void>;
     /**
-     * Convert entity to EntityVersion format
-     *
-     * @param entity Entity from storage
-     * @returns EntityVersion or null if invalid
-     */
-    private entityToVersion;
-    /**
-     * Generate unique ID for version entity
-     *
-     * Format: _version:{entityId}:{version}:{branch}
+     * Clear all versions for an entity
      *
      * @param entityId Entity ID
-     * @param version Version number
      * @param branch Branch name
-     * @returns Version entity ID
+     * @returns Number of versions deleted
      */
-    private getVersionEntityId;
+    clearVersions(entityId: string, branch: string): Promise<number>;
     /**
      * Get all versioned entities (for cleanup/debugging)
      *
-     * @returns List of entity IDs that have versions
+     * Note: This is an expensive operation that requires scanning.
+     * In the simple key-value approach, we don't maintain a global index.
+     * This method returns an empty array - use storage-level scanning if needed.
+     *
+     * @returns Empty array (not supported in simple approach)
      */
     getVersionedEntities(): Promise<string[]>;
     /**
-     * Clear all versions for an entity
+     * Generate storage key for version metadata
      *
      * @param entityId Entity ID
      * @param branch Branch name
-     * @returns Number of versions deleted
+     * @returns Storage key
      */
-    clearVersions(entityId: string, branch: string): Promise<number>;
+    private getMetaKey;
+    /**
+     * Load version store from storage
+     *
+     * @param key Storage key
+     * @returns Version store or null
+     */
+    private loadStore;
+    /**
+     * Save version store to storage
+     *
+     * @param key Storage key
+     * @param store Version store
+     */
+    private saveStore;
 }