@soulcraft/brainy 6.3.0 → 6.3.2

package/dist/versioning/VersionIndex.js

@@ -1,31 +1,23 @@
 /**
- * VersionIndex - Fast Version Lookup Using Existing Index Infrastructure (v5.3.0)
+ * VersionIndex - Pure Key-Value Version Storage (v6.3.0)
  *
- * Integrates with Brainy's existing index system:
- * - Uses MetadataIndexManager for field indexing
- * - Leverages UnifiedCache for memory management
- * - Uses EntityIdMapper for efficient ID handling
- * - Uses ChunkManager for adaptive chunking
- * - Leverages Roaring Bitmaps for fast set operations
+ * Stores version metadata using simple key-value storage:
+ * - Version list per entity: __version_meta_{entityId}_{branch}
+ * - Content stored separately by VersionStorage
  *
- * Version metadata is stored as regular entities with type='_version'
- * This allows us to use existing index infrastructure without modification!
- *
- * Fields indexed:
- * - versionEntityId: Entity being versioned
- * - versionBranch: Branch version was created on
- * - versionNumber: Version number
- * - versionTag: Optional user tag
- * - versionTimestamp: Creation timestamp
- * - versionCommitHash: Commit hash
+ * Key Design Decisions:
+ * - Versions are NOT entities (no brain.add())
+ * - Versions do NOT pollute find() results
+ * - Simple O(1) lookups per entity
+ * - Versions per entity is typically small (10-1000)
  *
  * NO MOCKS - Production implementation
  */
 /**
- * VersionIndex - Version lookup and querying using existing indexes
+ * VersionIndex - Pure key-value version metadata storage
  *
- * Strategy: Store version metadata as special entities with type='_version'
- * This leverages ALL existing index infrastructure automatically!
+ * Uses simple JSON storage instead of creating entities.
+ * This ensures versions never appear in find() results.
  */
 export class VersionIndex {
     constructor(brain) {
@@ -34,8 +26,6 @@ export class VersionIndex {
     }
     /**
      * Initialize version index
-     *
-     * No special setup needed - we use existing entity storage and indexes!
      */
     async initialize() {
         if (this.initialized)
@@ -45,75 +35,85 @@ export class VersionIndex {
     /**
      * Add version to index
      *
-     * Stores version metadata as a special entity with type='_version'
-     * This automatically indexes it using existing MetadataIndexManager!
+     * Stores version entry in key-value storage.
+     * Handles deduplication by content hash.
      *
-     * @param version Version metadata
+     * @param version Version metadata to store
      */
     async addVersion(version) {
         await this.initialize();
-        // Generate unique ID for version entity
-        const versionEntityId = this.getVersionEntityId(version.entityId, version.version, version.branch);
-        // Store as special entity with type='state' (version is a snapshot/state)
-        // This automatically gets indexed by MetadataIndexManager!
-        await this.brain.saveNounMetadata(versionEntityId, {
-            id: versionEntityId,
-            type: 'state', // Use standard 'state' type (version = snapshot state)
-            name: `Version ${version.version} of ${version.entityId}`,
-            metadata: {
-                // Flag to identify as version metadata
-                _isVersion: true,
-                // These fields are automatically indexed by MetadataIndexManager
-                versionEntityId: version.entityId, // Entity being versioned
-                versionBranch: version.branch, // Branch
-                versionNumber: version.version, // Version number
-                versionTag: version.tag, // Optional tag
-                versionTimestamp: version.timestamp, // Timestamp (indexed with bucketing)
-                versionCommitHash: version.commitHash, // Commit hash
-                versionContentHash: version.contentHash, // Content hash
-                versionAuthor: version.author, // Author
-                versionDescription: version.description, // Description
-                versionMetadata: version.metadata // Additional metadata
+        const key = this.getMetaKey(version.entityId, version.branch);
+        const store = await this.loadStore(key) || {
+            entityId: version.entityId,
+            branch: version.branch,
+            versions: []
+        };
+        // Check for duplicate content hash (deduplication)
+        const existing = store.versions.find(v => v.contentHash === version.contentHash);
+        if (existing) {
+            // Update tag/description if provided on duplicate save
+            let updated = false;
+            if (version.tag && version.tag !== existing.tag) {
+                existing.tag = version.tag;
+                updated = true;
+            }
+            if (version.description && version.description !== existing.description) {
+                existing.description = version.description;
+                updated = true;
             }
+            if (updated) {
+                await this.saveStore(key, store);
+            }
+            return;
+        }
+        // Add new version entry
+        store.versions.push({
+            version: version.version,
+            timestamp: version.timestamp,
+            contentHash: version.contentHash,
+            commitHash: version.commitHash,
+            tag: version.tag,
+            description: version.description,
+            author: version.author
         });
+        await this.saveStore(key, store);
     }
     /**
      * Get versions for an entity
      *
-     * Uses existing MetadataIndexManager to query efficiently!
-     *
-     * @param query Version query
+     * @param query Version query with filters
      * @returns List of versions (newest first)
      */
     async getVersions(query) {
         await this.initialize();
-        // Build metadata filter using existing query system
-        const filters = {
-            type: 'state',
-            _isVersion: true,
-            versionEntityId: query.entityId,
-            versionBranch: query.branch
-        };
-        // Add optional filters
+        const branch = query.branch || this.brain.currentBranch;
+        const key = this.getMetaKey(query.entityId, branch);
+        const store = await this.loadStore(key);
+        if (!store)
+            return [];
+        // Convert entries to EntityVersion format
+        let versions = store.versions.map(entry => ({
+            version: entry.version,
+            entityId: store.entityId,
+            branch: store.branch,
+            timestamp: entry.timestamp,
+            contentHash: entry.contentHash,
+            commitHash: entry.commitHash || '',
+            tag: entry.tag,
+            description: entry.description,
+            author: entry.author
+        }));
+        // Apply filters
         if (query.tag) {
-            filters.versionTag = query.tag;
+            versions = versions.filter(v => v.tag === query.tag);
         }
-        // Query using existing search infrastructure
-        const results = await this.brain.searchByMetadata(filters);
-        // Convert entities back to EntityVersion format
-        const versions = [];
-        for (const entity of results) {
-            const version = this.entityToVersion(entity);
-            if (version) {
-                // Filter by date range if specified
-                if (query.startDate && version.timestamp < query.startDate)
-                    continue;
-                if (query.endDate && version.timestamp > query.endDate)
-                    continue;
-                versions.push(version);
-            }
+        if (query.startDate) {
+            versions = versions.filter(v => v.timestamp >= query.startDate);
         }
-        // Sort by version number (newest first)
+        if (query.endDate) {
+            versions = versions.filter(v => v.timestamp <= query.endDate);
+        }
+        // Sort newest first (highest version number first)
         versions.sort((a, b) => b.version - a.version);
         // Apply pagination
         const start = query.offset || 0;
@@ -129,12 +129,8 @@ export class VersionIndex {
      * @returns Version metadata or null
      */
     async getVersion(entityId, version, branch) {
-        await this.initialize();
-        const versionEntityId = this.getVersionEntityId(entityId, version, branch);
-        const entity = await this.brain.getNounMetadata(versionEntityId);
-        if (!entity)
-            return null;
-        return this.entityToVersion(entity);
+        const versions = await this.getVersions({ entityId, branch });
+        return versions.find(v => v.version === version) || null;
     }
     /**
      * Get version by tag
@@ -145,19 +141,8 @@ export class VersionIndex {
      * @returns Version metadata or null
      */
     async getVersionByTag(entityId, tag, branch) {
-        await this.initialize();
-        // Query using existing metadata index
-        const results = await this.brain.searchByMetadata({
-            type: 'state',
-            _isVersion: true,
-            versionEntityId: entityId,
-            versionBranch: branch,
-            versionTag: tag
-        });
-        if (results.length === 0)
-            return null;
-        // Return first match (tags should be unique per entity/branch)
-        return this.entityToVersion(results[0]);
+        const versions = await this.getVersions({ entityId, branch, tag });
+        return versions[0] || null;
     }
     /**
      * Get version count for entity
@@ -167,15 +152,9 @@ export class VersionIndex {
      * @returns Number of versions
      */
     async getVersionCount(entityId, branch) {
-        await this.initialize();
-        // Use existing search infrastructure
-        const results = await this.brain.searchByMetadata({
-            type: 'state',
-            _isVersion: true,
-            versionEntityId: entityId,
-            versionBranch: branch
-        });
-        return results.length;
+        const key = this.getMetaKey(entityId, branch);
+        const store = await this.loadStore(key);
+        return store?.versions.length || 0;
     }
     /**
      * Remove version from index
@@ -186,90 +165,86 @@ export class VersionIndex {
      */
     async removeVersion(entityId, version, branch) {
         await this.initialize();
-        const versionEntityId = this.getVersionEntityId(entityId, version, branch);
-        // Delete version entity (automatically removed from indexes)
-        await this.brain.deleteNounMetadata(versionEntityId);
-    }
-    /**
-     * Convert entity to EntityVersion format
-     *
-     * @param entity Entity from storage
-     * @returns EntityVersion or null if invalid
-     */
-    entityToVersion(entity) {
-        if (!entity || !entity.metadata)
-            return null;
-        const m = entity.metadata;
-        if (!m.versionEntityId ||
-            !m.versionBranch ||
-            m.versionNumber === undefined ||
-            !m.versionCommitHash ||
-            !m.versionContentHash ||
-            !m.versionTimestamp) {
-            return null;
+        const key = this.getMetaKey(entityId, branch);
+        const store = await this.loadStore(key);
+        if (!store)
+            return;
+        const initialLength = store.versions.length;
+        store.versions = store.versions.filter(v => v.version !== version);
+        // Only save if something was removed
+        if (store.versions.length < initialLength) {
+            await this.saveStore(key, store);
         }
-        return {
-            version: m.versionNumber,
-            entityId: m.versionEntityId,
-            branch: m.versionBranch,
-            commitHash: m.versionCommitHash,
-            timestamp: m.versionTimestamp,
-            contentHash: m.versionContentHash,
-            tag: m.versionTag,
-            description: m.versionDescription,
-            author: m.versionAuthor,
-            metadata: m.versionMetadata
-        };
     }
     /**
-     * Generate unique ID for version entity
-     *
-     * Format: _version:{entityId}:{version}:{branch}
+     * Clear all versions for an entity
      *
      * @param entityId Entity ID
-     * @param version Version number
      * @param branch Branch name
-     * @returns Version entity ID
+     * @returns Number of versions deleted
      */
-    getVersionEntityId(entityId, version, branch) {
-        return `_version:${entityId}:${version}:${branch}`;
+    async clearVersions(entityId, branch) {
+        const key = this.getMetaKey(entityId, branch);
+        const store = await this.loadStore(key);
+        if (!store)
+            return 0;
+        const count = store.versions.length;
+        // Delete the store by saving null/empty
+        await this.saveStore(key, { entityId, branch, versions: [] });
+        return count;
     }
     /**
      * Get all versioned entities (for cleanup/debugging)
      *
-     * @returns List of entity IDs that have versions
+     * Note: This is an expensive operation that requires scanning.
+     * In the simple key-value approach, we don't maintain a global index.
+     * This method returns an empty array - use storage-level scanning if needed.
+     *
+     * @returns Empty array (not supported in simple approach)
      */
     async getVersionedEntities() {
-        await this.initialize();
-        // Query all version entities
-        const results = await this.brain.searchByMetadata({
-            type: 'state',
-            _isVersion: true
-        });
-        // Extract unique entity IDs
-        const entityIds = new Set();
-        for (const entity of results) {
-            const version = this.entityToVersion(entity);
-            if (version) {
-                entityIds.add(version.entityId);
-            }
-        }
-        return Array.from(entityIds);
+        // In the simple key-value approach, we don't maintain a global index
+        // of all versioned entities. This would require scanning storage.
+        // For most use cases, you know which entities you've versioned.
+        return [];
     }
+    // ============= Private Helpers =============
     /**
-     * Clear all versions for an entity
+     * Generate storage key for version metadata
      *
      * @param entityId Entity ID
      * @param branch Branch name
-     * @returns Number of versions deleted
+     * @returns Storage key
      */
-    async clearVersions(entityId, branch) {
-        await this.initialize();
-        const versions = await this.getVersions({ entityId, branch });
-        for (const version of versions) {
-            await this.removeVersion(entityId, version.version, branch);
+    getMetaKey(entityId, branch) {
+        return `__version_meta_${entityId}_${branch}`;
+    }
+    /**
+     * Load version store from storage
+     *
+     * @param key Storage key
+     * @returns Version store or null
+     */
+    async loadStore(key) {
+        try {
+            const store = await this.brain.storageAdapter.getMetadata(key);
+            // Handle empty store
+            if (!store || !store.versions)
+                return null;
+            return store;
         }
-        return versions.length;
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Save version store to storage
+     *
+     * @param key Storage key
+     * @param store Version store
+     */
+    async saveStore(key, store) {
+        await this.brain.storageAdapter.saveMetadata(key, store);
     }
 }
 //# sourceMappingURL=VersionIndex.js.map

package/dist/versioning/VersionManager.d.ts

@@ -1,20 +1,26 @@
 /**
- * VersionManager - Entity-Level Versioning Engine (v5.3.0)
+ * VersionManager - Entity-Level Versioning Engine (v5.3.0, v6.3.0 fix)
  *
  * Provides entity-level version control with:
  * - save() - Create entity version
- * - restore() - Restore entity to specific version
+ * - restore() - Restore entity to specific version (v6.3.0: now updates all indexes)
  * - list() - List all versions of an entity
  * - compare() - Deep diff between versions
  * - prune() - Remove old versions (retention policies)
  *
- * Architecture:
- * - Hybrid storage: COW commits for full snapshots + version index for fast queries
+ * Architecture (v6.3.0 - Clean Key-Value Storage):
+ * - Versions stored as key-value pairs, NOT as entities (no index pollution)
  * - Content-addressable: SHA-256 hashing for deduplication
- * - Space-efficient: Only stores changed data
+ * - Space-efficient: Only stores unique content
  * - Branch-aware: Versions tied to current branch
+ * - restore() uses brain.update() to refresh ALL indexes (HNSW, metadata, graph)
  *
- * NO MOCKS - Production implementation
+ * Storage keys:
+ * - Version metadata: __version_meta_{entityId}_{branch}
+ * - Version content:  __system_version_{entityId}_{contentHash}
+ *
+ * ZERO-CONFIG - Works automatically with existing storage infrastructure.
+ * NO MOCKS - Production implementation.
  */
 import { VersionDiff } from './VersionDiff.js';
 export interface EntityVersion {
@@ -93,6 +99,20 @@ export declare class VersionManager {
     private versionIndex;
     private initialized;
     constructor(brain: any);
+    /**
+     * Check if an entity is a VFS file
+     * VFS files store content in BlobStorage, not in entity.data
+     *
+     * @param entity Entity metadata object
+     * @returns True if entity is a VFS file
+     */
+    private isVFSFile;
+    /**
+     * Check if content is text-based for encoding decisions
+     * @param mimeType MIME type of the content
+     * @returns True if content should be stored as UTF-8 string
+     */
+    private isTextContent;
     /**
      * Initialize versioning system (lazy)
      */

package/dist/versioning/VersionManager.js

@@ -1,20 +1,26 @@
 /**
- * VersionManager - Entity-Level Versioning Engine (v5.3.0)
+ * VersionManager - Entity-Level Versioning Engine (v5.3.0, v6.3.0 fix)
  *
  * Provides entity-level version control with:
  * - save() - Create entity version
- * - restore() - Restore entity to specific version
+ * - restore() - Restore entity to specific version (v6.3.0: now updates all indexes)
  * - list() - List all versions of an entity
  * - compare() - Deep diff between versions
  * - prune() - Remove old versions (retention policies)
  *
- * Architecture:
- * - Hybrid storage: COW commits for full snapshots + version index for fast queries
+ * Architecture (v6.3.0 - Clean Key-Value Storage):
+ * - Versions stored as key-value pairs, NOT as entities (no index pollution)
  * - Content-addressable: SHA-256 hashing for deduplication
- * - Space-efficient: Only stores changed data
+ * - Space-efficient: Only stores unique content
  * - Branch-aware: Versions tied to current branch
+ * - restore() uses brain.update() to refresh ALL indexes (HNSW, metadata, graph)
  *
- * NO MOCKS - Production implementation
+ * Storage keys:
+ * - Version metadata: __version_meta_{entityId}_{branch}
+ * - Version content:  __system_version_{entityId}_{contentHash}
+ *
+ * ZERO-CONFIG - Works automatically with existing storage infrastructure.
+ * NO MOCKS - Production implementation.
  */
 import { VersionStorage } from './VersionStorage.js';
 import { VersionIndex } from './VersionIndex.js';
@@ -29,6 +35,34 @@ export class VersionManager {
         this.versionStorage = new VersionStorage(brain);
         this.versionIndex = new VersionIndex(brain);
     }
+    /**
+     * Check if an entity is a VFS file
+     * VFS files store content in BlobStorage, not in entity.data
+     *
+     * @param entity Entity metadata object
+     * @returns True if entity is a VFS file
+     */
+    isVFSFile(entity) {
+        return (entity?.isVFS === true &&
+            entity?.vfsType === 'file' &&
+            typeof entity?.path === 'string');
+    }
+    /**
+     * Check if content is text-based for encoding decisions
+     * @param mimeType MIME type of the content
+     * @returns True if content should be stored as UTF-8 string
+     */
+    isTextContent(mimeType) {
+        if (!mimeType)
+            return false;
+        return (mimeType.startsWith('text/') ||
+            mimeType === 'application/json' ||
+            mimeType === 'application/javascript' ||
+            mimeType === 'application/typescript' ||
+            mimeType === 'application/xml' ||
+            mimeType.includes('+xml') ||
+            mimeType.includes('+json'));
+    }
     /**
      * Initialize versioning system (lazy)
      */
@@ -56,6 +90,27 @@ export class VersionManager {
         if (!entity) {
             throw new Error(`Entity ${entityId} not found`);
         }
+        // v6.3.2 FIX: For VFS file entities, fetch current content from blob storage
+        // The entity.data field contains stale embedding text, not actual file content
+        // VFS files store their real content in BlobStorage (content-addressable)
+        if (this.isVFSFile(entity)) {
+            if (!this.brain.vfs) {
+                throw new Error(`Cannot version VFS file ${entityId}: VFS not initialized. ` +
+                    `Ensure brain.vfs is available before versioning VFS files.`);
+            }
+            // Read fresh content from blob storage via VFS
+            const freshContent = await this.brain.vfs.readFile(entity.path);
+            // Store content with appropriate encoding
+            // Text files as UTF-8 string (readable, smaller)
+            // Binary files as base64 (safe for JSON serialization)
+            if (this.isTextContent(entity.mimeType)) {
+                entity.data = freshContent.toString('utf-8');
+            }
+            else {
+                entity.data = freshContent.toString('base64');
+                entity._vfsEncoding = 'base64'; // Flag for restore to decode
+            }
+        }
         // Get current branch
         const currentBranch = this.brain.currentBranch;
         // Get next version number
@@ -201,8 +256,46 @@ export class VersionManager {
         if (!versionedEntity) {
             throw new Error(`Version data not found for entity ${entityId} version ${version}`);
         }
-        // Restore entity in storage
-        await this.brain.saveNounMetadata(entityId, versionedEntity);
+        // v6.3.2 FIX: For VFS file entities, write content back to blob storage
+        // The versioned data contains the actual file content (not stale embedding text)
+        // Using vfs.writeFile() ensures proper blob creation and metadata update
+        if (this.isVFSFile(versionedEntity)) {
+            if (!this.brain.vfs) {
+                throw new Error(`Cannot restore VFS file ${entityId}: VFS not initialized. ` +
+                    `Ensure brain.vfs is available before restoring VFS files.`);
+            }
+            // Decode content based on how it was stored
+            let content;
+            if (versionedEntity._vfsEncoding === 'base64') {
+                // Binary file stored as base64
+                content = Buffer.from(versionedEntity.data, 'base64');
+            }
+            else {
+                // Text file stored as UTF-8 string
+                content = Buffer.from(versionedEntity.data, 'utf-8');
+            }
+            // Write content back to VFS - this handles:
+            // - BlobStorage write (new hash)
+            // - Entity metadata update
+            // - Path resolver cache update
+            await this.brain.vfs.writeFile(versionedEntity.path, content);
+            return targetVersion;
+        }
+        // For non-VFS entities, use existing brain.update() logic
+        // Extract standard fields vs custom metadata
+        // NounMetadata has: noun, data, createdAt, updatedAt, createdBy, service, confidence, weight
+        const { noun, data, createdAt, updatedAt, createdBy, service, confidence, weight, ...customMetadata } = versionedEntity;
+        // Use brain.update() to restore - this updates ALL indexes (HNSW, metadata, graph)
+        // This is critical: saveNounMetadata() only saves to storage without updating indexes
+        await this.brain.update({
+            id: entityId,
+            data: data,
+            type: noun,
+            metadata: customMetadata,
+            confidence: confidence,
+            weight: weight,
+            merge: false // Replace entirely, don't merge with existing metadata
+        });
         return targetVersion;
     }
     /**