@soulcraft/brainy 5.0.0 → 5.1.0

package/dist/storage/adapters/typeAwareStorageAdapter.js

@@ -36,7 +36,7 @@
  * @since Phase 1 - Type-First Implementation
  */
 import { BaseStorage } from '../baseStorage.js';
-import { TypeUtils, NOUN_TYPE_COUNT, VERB_TYPE_COUNT } from '../../types/graphTypes.js';
+import { NounType, TypeUtils, NOUN_TYPE_COUNT, VERB_TYPE_COUNT } from '../../types/graphTypes.js';
 import { getShardIdFromUuid } from '../sharding.js';
 /**
  * Type-first storage paths
@@ -201,8 +201,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
         const typeIndex = TypeUtils.getNounIndex(type);
         this.nounCountsByType[typeIndex]++;
         this.nounTypeCache.set(noun.id, type);
-        // Delegate to underlying storage
-        await this.u.writeObjectToPath(path, noun);
+        // COW-aware write (v5.0.1): Use COW helper for branch isolation
+        await this.writeObjectToBranch(path, noun);
         // Periodically save statistics (every 100 saves)
         if (this.nounCountsByType[typeIndex] % 100 === 0) {
             await this.saveTypeStatistics();
@@ -216,14 +216,16 @@ export class TypeAwareStorageAdapter extends BaseStorage {
         const cachedType = this.nounTypeCache.get(id);
         if (cachedType) {
             const path = getNounVectorPath(cachedType, id);
-            return await this.u.readObjectFromPath(path);
+            // COW-aware read (v5.0.1): Use COW helper for branch isolation
+            return await this.readWithInheritance(path);
         }
         // Need to search across all types (expensive, but cached after first access)
         for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
             const type = TypeUtils.getNounFromIndex(i);
             const path = getNounVectorPath(type, id);
             try {
-                const noun = await this.u.readObjectFromPath(path);
+                // COW-aware read (v5.0.1): Use COW helper for branch isolation
+                const noun = await this.readWithInheritance(path);
                 if (noun) {
                     // Cache the type for next time
                     this.nounTypeCache.set(id, type);
@@ -242,13 +244,14 @@ export class TypeAwareStorageAdapter extends BaseStorage {
     async getNounsByNounType_internal(nounType) {
         const type = nounType;
         const prefix = `entities/nouns/${type}/vectors/`;
-        // List all files under this type's directory
-        const paths = await this.u.listObjectsUnderPath(prefix);
+        // COW-aware list (v5.0.1): Use COW helper for branch isolation
+        const paths = await this.listObjectsInBranch(prefix);
         // Load all nouns of this type
         const nouns = [];
         for (const path of paths) {
             try {
-                const noun = await this.u.readObjectFromPath(path);
+                // COW-aware read (v5.0.1): Use COW helper for branch isolation
+                const noun = await this.readWithInheritance(path);
                 if (noun) {
                     nouns.push(noun);
                     // Cache the type
@@ -269,7 +272,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
         const cachedType = this.nounTypeCache.get(id);
         if (cachedType) {
             const path = getNounVectorPath(cachedType, id);
-            await this.u.deleteObjectFromPath(path);
+            // COW-aware delete (v5.0.1): Use COW helper for branch isolation
+            await this.deleteObjectFromBranch(path);
             // Update counts
             const typeIndex = TypeUtils.getNounIndex(cachedType);
             if (this.nounCountsByType[typeIndex] > 0) {
@@ -283,7 +287,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
             const type = TypeUtils.getNounFromIndex(i);
             const path = getNounVectorPath(type, id);
             try {
-                await this.u.deleteObjectFromPath(path);
+                // COW-aware delete (v5.0.1): Use COW helper for branch isolation
+                await this.deleteObjectFromBranch(path);
                 // Update counts
                 if (this.nounCountsByType[i] > 0) {
                     this.nounCountsByType[i]--;
@@ -310,8 +315,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
         const typeIndex = TypeUtils.getVerbIndex(type);
         this.verbCountsByType[typeIndex]++;
         this.verbTypeCache.set(verb.id, type);
-        // Delegate to underlying storage
-        await this.u.writeObjectToPath(path, verb);
+        // COW-aware write (v5.0.1): Use COW helper for branch isolation
+        await this.writeObjectToBranch(path, verb);
         // Periodically save statistics
         if (this.verbCountsByType[typeIndex] % 100 === 0) {
             await this.saveTypeStatistics();
@@ -328,7 +333,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
         const cachedType = this.verbTypeCache.get(id);
         if (cachedType) {
             const path = getVerbVectorPath(cachedType, id);
-            const verb = await this.u.readObjectFromPath(path);
+            // COW-aware read (v5.0.1): Use COW helper for branch isolation
+            const verb = await this.readWithInheritance(path);
             return verb;
         }
         // Search across all types (only on first access)
@@ -336,7 +342,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
             const type = TypeUtils.getVerbFromIndex(i);
             const path = getVerbVectorPath(type, id);
             try {
-                const verb = await this.u.readObjectFromPath(path);
+                // COW-aware read (v5.0.1): Use COW helper for branch isolation
+                const verb = await this.readWithInheritance(path);
                 if (verb) {
                     // Cache the type for next time (read from verb.verb field)
                     this.verbTypeCache.set(id, verb.verb);
@@ -353,27 +360,36 @@ export class TypeAwareStorageAdapter extends BaseStorage {
      * Get verbs by source
      */
     async getVerbsBySource_internal(sourceId) {
-        // v4.8.1 PERFORMANCE FIX: Delegate to underlying storage instead of scanning all files
-        // Previous implementation was O(total_verbs) - scanned ALL 40 verb types and ALL verb files
-        // This was the root cause of the 11-version VFS bug (timeouts/zero results)
+        // v5.0.1 COW FIX: Use getVerbsWithPagination which is COW-aware
+        // Previous v4.8.1 implementation delegated to underlying storage, which bypasses COW!
+        // The underlying storage delegates to GraphAdjacencyIndex, which is shared between forks.
+        // This caused getRelations() to return 0 results for fork-created relationships.
         //
-        // Underlying storage adapters have optimized implementations:
-        // - FileSystemStorage: Uses getVerbsWithPagination with sourceId filter
-        // - GcsStorage: Uses batch queries with prefix filtering
-        // - S3Storage: Uses listObjects with sourceId-based filtering
+        // Now we use getVerbsWithPagination with sourceId filter, which:
+        // - Searches across all verb types using COW-aware listObjectsInBranch()
+        // - Reads verbs using COW-aware readWithInheritance()
+        // - Properly isolates fork data from parent
         //
-        // Phase 1b TODO: Add graph adjacency index query for O(1) lookups:
-        // const verbIds = await this.graphIndex?.getOutgoingEdges(sourceId) || []
-        // return Promise.all(verbIds.map(id => this.getVerb(id)))
-        return this.underlying.getVerbsBySource(sourceId);
+        // Performance: Still efficient because sourceId filter reduces iteration
+        const result = await this.getVerbsWithPagination({
+            limit: 10000, // High limit to get all verbs for this source
+            offset: 0,
+            filter: { sourceId }
+        });
+        return result.items;
     }
     /**
      * Get verbs by target
      */
     async getVerbsByTarget_internal(targetId) {
-        // v4.8.1 PERFORMANCE FIX: Delegate to underlying storage (same as getVerbsBySource fix)
-        // Previous implementation was O(total_verbs) - scanned ALL 40 verb types and ALL verb files
-        return this.underlying.getVerbsByTarget(targetId);
+        // v5.0.1 COW FIX: Use getVerbsWithPagination which is COW-aware
+        // Same fix as getVerbsBySource_internal - delegating to underlying bypasses COW
+        const result = await this.getVerbsWithPagination({
+            limit: 10000, // High limit to get all verbs for this target
+            offset: 0,
+            filter: { targetId }
+        });
+        return result.items;
     }
     /**
      * Get verbs by type (O(1) with type-first paths!)
@@ -383,11 +399,13 @@ export class TypeAwareStorageAdapter extends BaseStorage {
     async getVerbsByType_internal(verbType) {
         const type = verbType;
         const prefix = `entities/verbs/${type}/vectors/`;
-        const paths = await this.u.listObjectsUnderPath(prefix);
+        // COW-aware list (v5.0.1): Use COW helper for branch isolation
+        const paths = await this.listObjectsInBranch(prefix);
         const verbs = [];
         for (const path of paths) {
             try {
-                const hnswVerb = await this.u.readObjectFromPath(path);
+                // COW-aware read (v5.0.1): Use COW helper for branch isolation
+                const hnswVerb = await this.readWithInheritance(path);
                 if (!hnswVerb)
                     continue;
                 // Cache type from HNSWVerb for future O(1) retrievals
@@ -438,7 +456,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
         const cachedType = this.verbTypeCache.get(id);
         if (cachedType) {
             const path = getVerbVectorPath(cachedType, id);
-            await this.u.deleteObjectFromPath(path);
+            // COW-aware delete (v5.0.1): Use COW helper for branch isolation
+            await this.deleteObjectFromBranch(path);
             const typeIndex = TypeUtils.getVerbIndex(cachedType);
             if (this.verbCountsByType[typeIndex] > 0) {
                 this.verbCountsByType[typeIndex]--;
@@ -451,7 +470,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
             const type = TypeUtils.getVerbFromIndex(i);
             const path = getVerbVectorPath(type, id);
             try {
-                await this.u.deleteObjectFromPath(path);
+                // COW-aware delete (v5.0.1): Use COW helper for branch isolation
+                await this.deleteObjectFromBranch(path);
                 if (this.verbCountsByType[i] > 0) {
                     this.verbCountsByType[i]--;
                 }
@@ -472,9 +492,9 @@ export class TypeAwareStorageAdapter extends BaseStorage {
         // Extract and cache the type
         const type = (metadata.noun || 'thing');
         this.nounTypeCache.set(id, type);
-        // Save to type-aware path
+        // COW-aware write (v5.0.1): Use COW helper for branch isolation
         const path = getNounMetadataPath(type, id);
-        await this.u.writeObjectToPath(path, metadata);
+        await this.writeObjectToBranch(path, metadata);
     }
     /**
      * Get noun metadata (override to use type-aware paths)
@@ -484,14 +504,16 @@ export class TypeAwareStorageAdapter extends BaseStorage {
         const cachedType = this.nounTypeCache.get(id);
         if (cachedType) {
             const path = getNounMetadataPath(cachedType, id);
-            return await this.u.readObjectFromPath(path);
+            // COW-aware read (v5.0.1): Use COW helper for branch isolation
+            return await this.readWithInheritance(path);
         }
         // Search across all types
         for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
             const type = TypeUtils.getNounFromIndex(i);
             const path = getNounMetadataPath(type, id);
             try {
-                const metadata = await this.u.readObjectFromPath(path);
+                // COW-aware read (v5.0.1): Use COW helper for branch isolation
+                const metadata = await this.readWithInheritance(path);
                 if (metadata) {
                     // Cache the type for next time
                     const metadataType = (metadata.noun || 'thing');
@@ -512,7 +534,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
         const cachedType = this.nounTypeCache.get(id);
         if (cachedType) {
             const path = getNounMetadataPath(cachedType, id);
-            await this.u.deleteObjectFromPath(path);
+            // COW-aware delete (v5.0.1): Use COW helper for branch isolation
+            await this.deleteObjectFromBranch(path);
             return;
         }
         // Search across all types
@@ -520,7 +543,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
             const type = TypeUtils.getNounFromIndex(i);
             const path = getNounMetadataPath(type, id);
             try {
-                await this.u.deleteObjectFromPath(path);
+                // COW-aware delete (v5.0.1): Use COW helper for branch isolation
+                await this.deleteObjectFromBranch(path);
                 return;
             }
             catch (error) {
@@ -550,7 +574,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
         }
         // Save to type-aware path
         const path = getVerbMetadataPath(type, id);
-        await this.u.writeObjectToPath(path, metadata);
+        // COW-aware write (v5.0.1): Use COW helper for branch isolation
+        await this.writeObjectToBranch(path, metadata);
     }
     /**
      * Get verb metadata (override to use type-aware paths)
@@ -560,14 +585,16 @@ export class TypeAwareStorageAdapter extends BaseStorage {
         const cachedType = this.verbTypeCache.get(id);
         if (cachedType) {
             const path = getVerbMetadataPath(cachedType, id);
-            return await this.u.readObjectFromPath(path);
+            // COW-aware read (v5.0.1): Use COW helper for branch isolation
+            return await this.readWithInheritance(path);
         }
         // Search across all types
         for (let i = 0; i < VERB_TYPE_COUNT; i++) {
             const type = TypeUtils.getVerbFromIndex(i);
             const path = getVerbMetadataPath(type, id);
             try {
-                const metadata = await this.u.readObjectFromPath(path);
+                // COW-aware read (v5.0.1): Use COW helper for branch isolation
+                const metadata = await this.readWithInheritance(path);
                 if (metadata) {
                     // Cache the type for next time
                     this.verbTypeCache.set(id, type);
@@ -587,7 +614,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
         const cachedType = this.verbTypeCache.get(id);
         if (cachedType) {
             const path = getVerbMetadataPath(cachedType, id);
-            await this.u.deleteObjectFromPath(path);
+            // COW-aware delete (v5.0.1): Use COW helper for branch isolation
+            await this.deleteObjectFromBranch(path);
             return;
         }
         // Search across all types
@@ -595,7 +623,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
             const type = TypeUtils.getVerbFromIndex(i);
             const path = getVerbMetadataPath(type, id);
             try {
-                await this.u.deleteObjectFromPath(path);
+                // COW-aware delete (v5.0.1): Use COW helper for branch isolation
+                await this.deleteObjectFromBranch(path);
                 return;
             }
             catch (error) {
@@ -742,6 +771,206 @@ export class TypeAwareStorageAdapter extends BaseStorage {
         }
         return null;
     }
+    /**
+     * Get nouns with pagination (v5.0.1: COW-aware)
+     * Required for find() to work with TypeAwareStorage
+     */
+    async getNounsWithPagination(options) {
+        const limit = options.limit || 100;
+        const offset = options.offset || 0;
+        const filter = options.filter || {};
+        // Determine which types to search
+        let typesToSearch;
+        if (filter.nounType) {
+            typesToSearch = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
+        }
+        else {
+            // Search all 31 types
+            typesToSearch = [];
+            for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+                const type = TypeUtils.getNounFromIndex(i);
+                typesToSearch.push(type);
+            }
+        }
+        // Collect all matching nouns across types (COW-aware!)
+        const allNouns = [];
+        for (const type of typesToSearch) {
+            const prefix = `entities/nouns/${type}/vectors/`;
+            // COW-aware list with inheritance (v5.0.1): Fork sees parent's nouns too!
+            const paths = await this.listObjectsWithInheritance(prefix);
+            for (const path of paths) {
+                try {
+                    // COW-aware read with inheritance
+                    const noun = await this.readWithInheritance(path);
+                    if (!noun)
+                        continue;
+                    // Get metadata separately
+                    const metadata = await this.getNounMetadata(noun.id);
+                    if (!metadata)
+                        continue;
+                    // Filter by service if specified
+                    if (filter.service && metadata.service !== filter.service)
+                        continue;
+                    // Filter by custom metadata if specified
+                    if (filter.metadata) {
+                        let matches = true;
+                        for (const [key, value] of Object.entries(filter.metadata)) {
+                            if (metadata[key] !== value) {
+                                matches = false;
+                                break;
+                            }
+                        }
+                        if (!matches)
+                            continue;
+                    }
+                    // Extract standard fields from metadata
+                    const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadata;
+                    // Create HNSWNounWithMetadata (v4.8.0 format)
+                    const nounWithMetadata = {
+                        id: noun.id,
+                        vector: noun.vector,
+                        connections: noun.connections,
+                        level: noun.level || 0,
+                        type: nounType || NounType.Thing,
+                        createdAt: createdAt || Date.now(),
+                        updatedAt: updatedAt || Date.now(),
+                        confidence,
+                        weight,
+                        service,
+                        data,
+                        createdBy,
+                        metadata: customMetadata
+                    };
+                    allNouns.push(nounWithMetadata);
+                }
+                catch (error) {
+                    // Skip entities with errors
+                    continue;
+                }
+            }
+        }
+        // Apply pagination
+        const totalCount = allNouns.length;
+        const paginatedNouns = allNouns.slice(offset, offset + limit);
+        const hasMore = offset + limit < totalCount;
+        // Generate cursor if more results exist
+        let nextCursor;
+        if (hasMore && paginatedNouns.length > 0) {
+            nextCursor = paginatedNouns[paginatedNouns.length - 1].id;
+        }
+        return {
+            items: paginatedNouns,
+            totalCount,
+            hasMore,
+            nextCursor
+        };
+    }
+    /**
+     * Get verbs with pagination (v5.0.1: COW-aware)
+     * Required for GraphAdjacencyIndex rebuild and find() to work
+     */
+    async getVerbsWithPagination(options) {
+        const limit = options.limit || 100;
+        const offset = options.offset || 0;
+        const filter = options.filter || {};
+        // Determine which types to search
+        let typesToSearch;
+        if (filter.verbType) {
+            typesToSearch = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType];
+        }
+        else {
+            // Search all 40 verb types
+            typesToSearch = [];
+            for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+                const type = TypeUtils.getVerbFromIndex(i);
+                typesToSearch.push(type);
+            }
+        }
+        // Collect all matching verbs across types (COW-aware!)
+        const allVerbs = [];
+        for (const type of typesToSearch) {
+            const prefix = `entities/verbs/${type}/vectors/`;
+            // COW-aware list with inheritance (v5.0.1): Fork sees parent's verbs too!
+            const paths = await this.listObjectsWithInheritance(prefix);
+            for (const path of paths) {
+                try {
+                    // COW-aware read with inheritance
+                    const verb = await this.readWithInheritance(path);
+                    if (!verb)
+                        continue;
+                    // Filter by sourceId if specified
+                    if (filter.sourceId) {
+                        const sourceIds = Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId];
+                        if (!sourceIds.includes(verb.sourceId))
+                            continue;
+                    }
+                    // Filter by targetId if specified
+                    if (filter.targetId) {
+                        const targetIds = Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId];
+                        if (!targetIds.includes(verb.targetId))
+                            continue;
+                    }
+                    // Get metadata separately
+                    const metadata = await this.getVerbMetadata(verb.id);
+                    // Filter by service if specified
+                    if (filter.service && metadata && metadata.service !== filter.service)
+                        continue;
+                    // Filter by custom metadata if specified
+                    if (filter.metadata && metadata) {
+                        let matches = true;
+                        for (const [key, value] of Object.entries(filter.metadata)) {
+                            if (metadata[key] !== value) {
+                                matches = false;
+                                break;
+                            }
+                        }
+                        if (!matches)
+                            continue;
+                    }
+                    // Extract standard fields from metadata
+                    const metadataObj = metadata || {};
+                    const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
+                    // Create HNSWVerbWithMetadata (v4.8.0 format)
+                    const verbWithMetadata = {
+                        id: verb.id,
+                        vector: verb.vector,
+                        connections: verb.connections,
+                        verb: verb.verb,
+                        sourceId: verb.sourceId,
+                        targetId: verb.targetId,
+                        createdAt: createdAt || Date.now(),
+                        updatedAt: updatedAt || Date.now(),
+                        confidence,
+                        weight,
+                        service,
+                        data,
+                        createdBy,
+                        metadata: customMetadata
+                    };
+                    allVerbs.push(verbWithMetadata);
+                }
+                catch (error) {
+                    // Skip verbs with errors
+                    continue;
+                }
+            }
+        }
+        // Apply pagination
+        const totalCount = allVerbs.length;
+        const paginatedVerbs = allVerbs.slice(offset, offset + limit);
+        const hasMore = offset + limit < totalCount;
+        // Generate cursor if more results exist
+        let nextCursor;
+        if (hasMore && paginatedVerbs.length > 0) {
+            nextCursor = paginatedVerbs[paginatedVerbs.length - 1].id;
+        }
+        return {
+            items: paginatedVerbs,
+            totalCount,
+            hasMore,
+            nextCursor
+        };
+    }
     /**
      * Save HNSW system data (entry point, max level)
      */

package/dist/storage/baseStorage.d.ts

@@ -73,19 +73,62 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * Ensure the storage adapter is initialized
      */
     protected ensureInitialized(): Promise<void>;
+    /**
+     * Lightweight COW enablement - just enables branch-scoped paths
+     * Called during init() to ensure all data is stored with branch prefixes from the start
+     * RefManager/BlobStorage/CommitLog are lazy-initialized on first fork()
+     * @param branch - Branch name to use (default: 'main')
+     */
+    enableCOWLightweight(branch?: string): void;
     /**
      * Initialize COW (Copy-on-Write) support
      * Creates RefManager and BlobStorage for instant fork() capability
      *
+     * v5.0.1: Now called automatically by storageFactory (zero-config)
+     *
      * @param options - COW initialization options
      * @param options.branch - Initial branch name (default: 'main')
      * @param options.enableCompression - Enable zstd compression for blobs (default: true)
      * @returns Promise that resolves when COW is initialized
      */
-    protected initializeCOW(options?: {
+    initializeCOW(options?: {
         branch?: string;
         enableCompression?: boolean;
     }): Promise<void>;
+    /**
+     * Resolve branch-scoped path for COW isolation
+     * @protected - Available to subclasses for COW implementation
+     */
+    protected resolveBranchPath(basePath: string, branch?: string): string;
+    /**
+     * Write object to branch-specific path (COW layer)
+     * @protected - Available to subclasses for COW implementation
+     */
+    protected writeObjectToBranch(path: string, data: any, branch?: string): Promise<void>;
+    /**
+     * Read object with inheritance from parent branches (COW layer)
+     * Tries current branch first, then walks commit history
+     * @protected - Available to subclasses for COW implementation
+     */
+    protected readWithInheritance(path: string, branch?: string): Promise<any | null>;
+    /**
+     * Delete object from branch-specific path (COW layer)
+     * @protected - Available to subclasses for COW implementation
+     */
+    protected deleteObjectFromBranch(path: string, branch?: string): Promise<void>;
+    /**
+     * List objects under path in branch (COW layer)
+     * @protected - Available to subclasses for COW implementation
+     */
+    protected listObjectsInBranch(prefix: string, branch?: string): Promise<string[]>;
+    /**
+     * List objects with inheritance (v5.0.1)
+     * Lists objects from current branch AND main branch, returns unique paths
+     * This enables fork to see parent's data in pagination operations
+     *
+     * Simplified approach: All branches inherit from main
+     */
+    protected listObjectsWithInheritance(prefix: string, branch?: string): Promise<string[]>;
     /**
      * Save a noun to storage (v4.0.0: vector only, metadata saved separately)
      * @param noun Pure HNSW vector data (no metadata)