@soulcraft/brainy 4.11.2 → 5.1.0

package/dist/storage/baseStorage.js

@@ -7,6 +7,9 @@ import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js';
 import { validateNounType, validateVerbType } from '../utils/typeValidation.js';
 import { NounType } from '../types/graphTypes.js';
 import { getShardIdFromUuid } from './sharding.js';
+import { RefManager } from './cow/RefManager.js';
+import { BlobStorage } from './cow/BlobStorage.js';
+import { CommitLog } from './cow/CommitLog.js';
 // Clean directory structure (v4.7.2+)
 // All storage adapters use this consistent structure
 export const NOUNS_METADATA_DIR = 'entities/nouns/metadata';
@@ -38,6 +41,8 @@ export class BaseStorage extends BaseStorageAdapter {
         super(...arguments);
         this.isInitialized = false;
         this.readOnly = false;
+        this.currentBranch = 'main';
+        this.cowEnabled = false;
     }
     /**
      * Analyze a storage key to determine its routing and path
@@ -119,6 +124,241 @@ export class BaseStorage extends BaseStorageAdapter {
             await this.init();
         }
     }
+    /**
+     * Lightweight COW enablement - just enables branch-scoped paths
+     * Called during init() to ensure all data is stored with branch prefixes from the start
+     * RefManager/BlobStorage/CommitLog are lazy-initialized on first fork()
+     * @param branch - Branch name to use (default: 'main')
+     */
+    enableCOWLightweight(branch = 'main') {
+        if (this.cowEnabled) {
+            return;
+        }
+        this.currentBranch = branch;
+        this.cowEnabled = true;
+        // RefManager/BlobStorage/CommitLog remain undefined until first fork()
+    }
+    /**
+     * Initialize COW (Copy-on-Write) support
+     * Creates RefManager and BlobStorage for instant fork() capability
+     *
+     * v5.0.1: Now called automatically by storageFactory (zero-config)
+     *
+     * @param options - COW initialization options
+     * @param options.branch - Initial branch name (default: 'main')
+     * @param options.enableCompression - Enable zstd compression for blobs (default: true)
+     * @returns Promise that resolves when COW is initialized
+     */
+    async initializeCOW(options) {
+        // Check if RefManager already initialized (full COW setup complete)
+        if (this.refManager) {
+            return;
+        }
+        // Enable lightweight COW if not already enabled
+        if (!this.cowEnabled) {
+            this.currentBranch = options?.branch || 'main';
+            this.cowEnabled = true;
+        }
+        // Create COWStorageAdapter bridge
+        // This adapts BaseStorage's methods to the simple key-value interface
+        const cowAdapter = {
+            get: async (key) => {
+                try {
+                    const data = await this.readObjectFromPath(`_cow/${key}`);
+                    if (data === null) {
+                        return undefined;
+                    }
+                    // Convert to Buffer
+                    if (Buffer.isBuffer(data)) {
+                        return data;
+                    }
+                    return Buffer.from(JSON.stringify(data));
+                }
+                catch (error) {
+                    return undefined;
+                }
+            },
+            put: async (key, data) => {
+                // Store as Buffer (for blob data) or parse JSON (for metadata)
+                let obj;
+                try {
+                    // Try to parse as JSON first (for metadata)
+                    obj = JSON.parse(data.toString());
+                }
+                catch {
+                    // Not JSON, store as binary (base64 encoded for JSON storage)
+                    obj = { _binary: true, data: data.toString('base64') };
+                }
+                await this.writeObjectToPath(`_cow/${key}`, obj);
+            },
+            delete: async (key) => {
+                try {
+                    await this.deleteObjectFromPath(`_cow/${key}`);
+                }
+                catch (error) {
+                    // Ignore if doesn't exist
+                }
+            },
+            list: async (prefix) => {
+                try {
+                    const paths = await this.listObjectsUnderPath(`_cow/${prefix}`);
+                    // Remove _cow/ prefix and return relative keys
+                    return paths.map(p => p.replace(/^_cow\//, ''));
+                }
+                catch (error) {
+                    return [];
+                }
+            }
+        };
+        // Initialize RefManager
+        this.refManager = new RefManager(cowAdapter);
+        // Initialize BlobStorage
+        this.blobStorage = new BlobStorage(cowAdapter, {
+            enableCompression: options?.enableCompression !== false
+        });
+        // Initialize CommitLog
+        this.commitLog = new CommitLog(this.blobStorage, this.refManager);
+        // Check if main branch exists, create if not
+        const mainRef = await this.refManager.getRef('main');
+        if (!mainRef) {
+            // Create initial commit (empty tree)
+            const emptyTreeHash = '0000000000000000000000000000000000000000000000000000000000000000';
+            await this.refManager.createBranch('main', emptyTreeHash, {
+                description: 'Initial branch',
+                author: 'system'
+            });
+        }
+        // Set HEAD to current branch
+        const currentRef = await this.refManager.getRef(this.currentBranch);
+        if (currentRef) {
+            await this.refManager.setHead(this.currentBranch);
+        }
+        else {
+            // Branch doesn't exist, create it from main
+            const mainCommit = await this.refManager.resolveRef('main');
+            if (mainCommit) {
+                await this.refManager.createBranch(this.currentBranch, mainCommit, {
+                    description: `Branch created from main`,
+                    author: 'system'
+                });
+                await this.refManager.setHead(this.currentBranch);
+            }
+        }
+        this.cowEnabled = true;
+    }
+    /**
+     * Resolve branch-scoped path for COW isolation
+     * @protected - Available to subclasses for COW implementation
+     */
+    resolveBranchPath(basePath, branch) {
+        if (!this.cowEnabled) {
+            return basePath; // COW disabled, use direct path
+        }
+        const targetBranch = branch || this.currentBranch || 'main';
+        // Branch-scoped path: branches/<branch>/<basePath>
+        return `branches/${targetBranch}/${basePath}`;
+    }
+    /**
+     * Write object to branch-specific path (COW layer)
+     * @protected - Available to subclasses for COW implementation
+     */
+    async writeObjectToBranch(path, data, branch) {
+        const branchPath = this.resolveBranchPath(path, branch);
+        return this.writeObjectToPath(branchPath, data);
+    }
+    /**
+     * Read object with inheritance from parent branches (COW layer)
+     * Tries current branch first, then walks commit history
+     * @protected - Available to subclasses for COW implementation
+     */
+    async readWithInheritance(path, branch) {
+        if (!this.cowEnabled) {
+            // COW disabled, direct read
+            return this.readObjectFromPath(path);
+        }
+        const targetBranch = branch || this.currentBranch || 'main';
+        // Try current branch first
+        const branchPath = this.resolveBranchPath(path, targetBranch);
+        let data = await this.readObjectFromPath(branchPath);
+        if (data !== null) {
+            return data; // Found in current branch
+        }
+        // Not in branch, check if we're on main (no inheritance needed)
+        if (targetBranch === 'main') {
+            return null;
+        }
+        // Not in branch, walk commit history to find in parent
+        if (this.refManager && this.commitLog) {
+            try {
+                const commitHash = await this.refManager.resolveRef(targetBranch);
+                if (commitHash) {
+                    // Walk parent commits until we find the data
+                    for await (const commit of this.commitLog.walk(commitHash)) {
+                        // Try reading from parent's branch path
+                        const parentBranch = commit.metadata?.branch || 'main';
+                        if (parentBranch === targetBranch)
+                            continue; // Skip self
+                        const parentPath = this.resolveBranchPath(path, parentBranch);
+                        data = await this.readObjectFromPath(parentPath);
+                        if (data !== null) {
+                            return data; // Found in ancestor
+                        }
+                    }
+                }
+            }
+            catch (error) {
+                // Commit walk failed, fall back to main
+                const mainPath = this.resolveBranchPath(path, 'main');
+                return this.readObjectFromPath(mainPath);
+            }
+        }
+        // Last fallback: try main branch
+        const mainPath = this.resolveBranchPath(path, 'main');
+        return this.readObjectFromPath(mainPath);
+    }
+    /**
+     * Delete object from branch-specific path (COW layer)
+     * @protected - Available to subclasses for COW implementation
+     */
+    async deleteObjectFromBranch(path, branch) {
+        const branchPath = this.resolveBranchPath(path, branch);
+        return this.deleteObjectFromPath(branchPath);
+    }
+    /**
+     * List objects under path in branch (COW layer)
+     * @protected - Available to subclasses for COW implementation
+     */
+    async listObjectsInBranch(prefix, branch) {
+        const branchPrefix = this.resolveBranchPath(prefix, branch);
+        const paths = await this.listObjectsUnderPath(branchPrefix);
+        // Remove branch prefix from results
+        const targetBranch = branch || this.currentBranch || 'main';
+        const prefixToRemove = `branches/${targetBranch}/`;
+        return paths.map(p => p.startsWith(prefixToRemove) ? p.substring(prefixToRemove.length) : p);
+    }
+    /**
+     * List objects with inheritance (v5.0.1)
+     * Lists objects from current branch AND main branch, returns unique paths
+     * This enables fork to see parent's data in pagination operations
+     *
+     * Simplified approach: All branches inherit from main
+     */
+    async listObjectsWithInheritance(prefix, branch) {
+        if (!this.cowEnabled) {
+            return this.listObjectsInBranch(prefix, branch);
+        }
+        const targetBranch = branch || this.currentBranch || 'main';
+        // Collect paths from current branch
+        const pathsSet = new Set();
+        const currentBranchPaths = await this.listObjectsInBranch(prefix, targetBranch);
+        currentBranchPaths.forEach(p => pathsSet.add(p));
+        // If not on main, also list from main (all branches inherit from main)
+        if (targetBranch !== 'main') {
+            const mainPaths = await this.listObjectsInBranch(prefix, 'main');
+            mainPaths.forEach(p => pathsSet.add(p));
+        }
+        return Array.from(pathsSet);
+    }
     /**
      * Save a noun to storage (v4.0.0: vector only, metadata saved separately)
      * @param noun Pure HNSW vector data (no metadata)
@@ -731,7 +971,7 @@ export class BaseStorage extends BaseStorageAdapter {
     async saveMetadata(id, metadata) {
         await this.ensureInitialized();
         const keyInfo = this.analyzeKey(id, 'system');
-        return this.writeObjectToPath(keyInfo.fullPath, metadata);
+        return this.writeObjectToBranch(keyInfo.fullPath, metadata);
     }
     /**
      * Get metadata from storage (v4.0.0: now typed)
@@ -740,7 +980,7 @@ export class BaseStorage extends BaseStorageAdapter {
     async getMetadata(id) {
         await this.ensureInitialized();
         const keyInfo = this.analyzeKey(id, 'system');
-        return this.readObjectFromPath(keyInfo.fullPath);
+        return this.readWithInheritance(keyInfo.fullPath);
     }
     /**
      * Save noun metadata to storage (v4.0.0: now typed)
@@ -765,10 +1005,10 @@ export class BaseStorage extends BaseStorageAdapter {
         await this.ensureInitialized();
         // Determine if this is a new entity by checking if metadata already exists
         const keyInfo = this.analyzeKey(id, 'noun-metadata');
-        const existingMetadata = await this.readObjectFromPath(keyInfo.fullPath);
+        const existingMetadata = await this.readWithInheritance(keyInfo.fullPath);
         const isNew = !existingMetadata;
-        // Save the metadata
-        await this.writeObjectToPath(keyInfo.fullPath, metadata);
+        // Save the metadata (COW-aware - writes to branch-specific path)
+        await this.writeObjectToBranch(keyInfo.fullPath, metadata);
         // CRITICAL FIX (v4.1.2): Increment count for new entities
         // This runs AFTER metadata is saved, guaranteeing type information is available
         // Uses synchronous increment since storage operations are already serialized
@@ -788,7 +1028,7 @@ export class BaseStorage extends BaseStorageAdapter {
     async getNounMetadata(id) {
         await this.ensureInitialized();
         const keyInfo = this.analyzeKey(id, 'noun-metadata');
-        return this.readObjectFromPath(keyInfo.fullPath);
+        return this.readWithInheritance(keyInfo.fullPath);
     }
     /**
      * Delete noun metadata from storage
@@ -797,7 +1037,7 @@ export class BaseStorage extends BaseStorageAdapter {
     async deleteNounMetadata(id) {
         await this.ensureInitialized();
         const keyInfo = this.analyzeKey(id, 'noun-metadata');
-        return this.deleteObjectFromPath(keyInfo.fullPath);
+        return this.deleteObjectFromBranch(keyInfo.fullPath);
     }
     /**
      * Save verb metadata to storage (v4.0.0: now typed)
@@ -823,10 +1063,10 @@ export class BaseStorage extends BaseStorageAdapter {
         await this.ensureInitialized();
         // Determine if this is a new verb by checking if metadata already exists
         const keyInfo = this.analyzeKey(id, 'verb-metadata');
-        const existingMetadata = await this.readObjectFromPath(keyInfo.fullPath);
+        const existingMetadata = await this.readWithInheritance(keyInfo.fullPath);
         const isNew = !existingMetadata;
-        // Save the metadata
-        await this.writeObjectToPath(keyInfo.fullPath, metadata);
+        // Save the metadata (COW-aware - writes to branch-specific path)
+        await this.writeObjectToBranch(keyInfo.fullPath, metadata);
         // CRITICAL FIX (v4.1.2): Increment verb count for new relationships
         // This runs AFTER metadata is saved
         // Verb type is now stored in metadata (as of v4.1.2) to avoid loading HNSWVerb
@@ -847,7 +1087,7 @@ export class BaseStorage extends BaseStorageAdapter {
     async getVerbMetadata(id) {
         await this.ensureInitialized();
         const keyInfo = this.analyzeKey(id, 'verb-metadata');
-        return this.readObjectFromPath(keyInfo.fullPath);
+        return this.readWithInheritance(keyInfo.fullPath);
     }
     /**
      * Delete verb metadata from storage
@@ -856,7 +1096,7 @@ export class BaseStorage extends BaseStorageAdapter {
     async deleteVerbMetadata(id) {
         await this.ensureInitialized();
         const keyInfo = this.analyzeKey(id, 'verb-metadata');
-        return this.deleteObjectFromPath(keyInfo.fullPath);
+        return this.deleteObjectFromBranch(keyInfo.fullPath);
     }
     /**
      * Helper method to convert a Map to a plain object for serialization

package/dist/storage/cow/BlobStorage.d.ts

@@ -0,0 +1,232 @@
+/**
+ * BlobStorage: Content-Addressable Blob Storage for COW (Copy-on-Write)
+ *
+ * State-of-the-art implementation featuring:
+ * - Content-addressable: SHA-256 hashing
+ * - Type-aware chunking: Separate vectors, metadata, relationships
+ * - Compression: zstd for JSON, optimized for vectors
+ * - LRU caching: Hot blob performance
+ * - Streaming: Multipart upload for large blobs
+ * - Batch operations: Parallel I/O
+ * - Integrity: Cryptographic verification
+ * - Observability: Metrics and tracing
+ *
+ * @module storage/cow/BlobStorage
+ */
+/**
+ * Simple key-value storage interface for COW primitives
+ * This will be implemented by BaseStorage when COW is integrated
+ */
+export interface COWStorageAdapter {
+    get(key: string): Promise<Buffer | undefined>;
+    put(key: string, data: Buffer): Promise<void>;
+    delete(key: string): Promise<void>;
+    list(prefix: string): Promise<string[]>;
+}
+/**
+ * Blob metadata stored alongside blob data
+ */
+export interface BlobMetadata {
+    hash: string;
+    size: number;
+    compressedSize: number;
+    compression: 'none' | 'zstd';
+    type: 'vector' | 'metadata' | 'tree' | 'commit' | 'raw';
+    createdAt: number;
+    refCount: number;
+}
+/**
+ * Blob write options
+ */
+export interface BlobWriteOptions {
+    compression?: 'none' | 'zstd' | 'auto';
+    type?: 'vector' | 'metadata' | 'tree' | 'commit' | 'raw';
+    skipVerification?: boolean;
+}
+/**
+ * Blob read options
+ */
+export interface BlobReadOptions {
+    skipDecompression?: boolean;
+    skipCache?: boolean;
+    skipVerification?: boolean;
+}
+/**
+ * Blob statistics for observability
+ */
+export interface BlobStats {
+    totalBlobs: number;
+    totalSize: number;
+    compressedSize: number;
+    cacheHits: number;
+    cacheMisses: number;
+    compressionRatio: number;
+    avgBlobSize: number;
+    dedupSavings: number;
+}
+/**
+ * State-of-the-art content-addressable blob storage
+ *
+ * Features:
+ * - Content addressing via SHA-256
+ * - Type-aware compression (zstd, vector-optimized)
+ * - LRU caching with memory limits
+ * - Streaming for large blobs
+ * - Batch operations
+ * - Integrity verification
+ * - Observability metrics
+ */
+export declare class BlobStorage {
+    private adapter;
+    private cache;
+    private cacheMaxSize;
+    private currentCacheSize;
+    private stats;
+    private zstdCompress?;
+    private zstdDecompress?;
+    private readonly CACHE_MAX_SIZE;
+    private readonly MULTIPART_THRESHOLD;
+    private readonly COMPRESSION_THRESHOLD;
+    constructor(adapter: COWStorageAdapter, options?: {
+        cacheMaxSize?: number;
+        enableCompression?: boolean;
+    });
+    /**
+     * Lazy load zstd compression module
+     * (Avoids loading if not needed)
+     */
+    private initCompression;
+    /**
+     * Compute SHA-256 hash of data
+     *
+     * @param data - Data to hash
+     * @returns SHA-256 hash as hex string
+     */
+    static hash(data: Buffer): string;
+    /**
+     * Write a blob to storage
+     *
+     * Features:
+     * - Content-addressable: hash determines storage key
+     * - Deduplication: existing blob not rewritten
+     * - Compression: auto-compress based on type
+     * - Multipart: for large blobs (>5MB)
+     * - Verification: hash verification
+     * - Caching: write-through cache
+     *
+     * @param data - Blob data to write
+     * @param options - Write options
+     * @returns Blob hash
+     */
+    write(data: Buffer, options?: BlobWriteOptions): Promise<string>;
+    /**
+     * Read a blob from storage
+     *
+     * Features:
+     * - Cache lookup first (LRU)
+     * - Decompression (if compressed)
+     * - Verification (optional hash check)
+     * - Streaming for large blobs
+     *
+     * @param hash - Blob hash
+     * @param options - Read options
+     * @returns Blob data
+     */
+    read(hash: string, options?: BlobReadOptions): Promise<Buffer>;
+    /**
+     * Check if blob exists
+     *
+     * @param hash - Blob hash
+     * @returns True if blob exists
+     */
+    has(hash: string): Promise<boolean>;
+    /**
+     * Delete a blob from storage
+     *
+     * Features:
+     * - Reference counting: only delete if refCount = 0
+     * - Cascade: delete metadata too
+     * - Cache invalidation
+     *
+     * @param hash - Blob hash
+     */
+    delete(hash: string): Promise<void>;
+    /**
+     * Get blob metadata without reading full blob
+     *
+     * @param hash - Blob hash
+     * @returns Blob metadata
+     */
+    getMetadata(hash: string): Promise<BlobMetadata | undefined>;
+    /**
+     * Batch write multiple blobs in parallel
+     *
+     * @param blobs - Array of [data, options] tuples
+     * @returns Array of blob hashes
+     */
+    writeBatch(blobs: Array<[Buffer, BlobWriteOptions?]>): Promise<string[]>;
+    /**
+     * Batch read multiple blobs in parallel
+     *
+     * @param hashes - Array of blob hashes
+     * @param options - Read options
+     * @returns Array of blob data
+     */
+    readBatch(hashes: string[], options?: BlobReadOptions): Promise<Buffer[]>;
+    /**
+     * List all blobs (for garbage collection, debugging)
+     *
+     * @returns Array of blob hashes
+     */
+    listBlobs(): Promise<string[]>;
+    /**
+     * Get storage statistics
+     *
+     * @returns Blob statistics
+     */
+    getStats(): BlobStats;
+    /**
+     * Clear cache (useful for testing, memory pressure)
+     */
+    clearCache(): void;
+    /**
+     * Garbage collect unreferenced blobs
+     *
+     * @param referencedHashes - Set of hashes that should be kept
+     * @returns Number of blobs deleted
+     */
+    garbageCollect(referencedHashes: Set<string>): Promise<number>;
+    /**
+     * Select compression strategy based on data and options
+     */
+    private selectCompression;
+    /**
+     * Write large blob using multipart upload
+     * (Future enhancement: stream to adapter if supported)
+     */
+    private writeMultipart;
+    /**
+     * Increment reference count for a blob
+     */
+    private incrementRefCount;
+    /**
+     * Decrement reference count for a blob
+     */
+    private decrementRefCount;
+    /**
+     * Add blob to LRU cache
+     */
+    private addToCache;
+    /**
+     * Get blob from cache
+     */
+    private getFromCache;
+    /**
+     * Remove blob from cache
+     */
+    private removeFromCache;
+    /**
+     * Evict least recently used entry from cache
+     */
+    private evictLRU;
+}