@soulcraft/brainy 5.10.4 → 5.11.1

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -89,6 +89,16 @@ export class S3CompatibleStorage extends BaseStorage {
         this.hnswLocks = new Map();
         // Node cache to avoid redundant API calls
         this.nodeCache = new Map();
+        /**
+         * Check if COW has been explicitly disabled via clear()
+         * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
+         * @returns true if marker object exists, false otherwise
+         * @protected
+         */
+        /**
+         * v5.11.0: Removed checkClearMarker() and createClearMarker() methods
+         * COW is now always enabled - marker files are no longer used
+         */
         // Batch update timer ID
         this.statisticsBatchUpdateTimerId = null;
         // Flag to indicate if statistics have been modified since last save
@@ -1611,32 +1621,18 @@ export class S3CompatibleStorage extends BaseStorage {
                     }
                 }
             };
-            // Delete all objects in the nouns directory
-            await deleteObjectsWithPrefix(this.nounPrefix);
-            // Delete all objects in the verbs directory
-            await deleteObjectsWithPrefix(this.verbPrefix);
-            // Delete all objects in the noun metadata directory
-            await deleteObjectsWithPrefix(this.metadataPrefix);
-            // Delete all objects in the verb metadata directory
-            await deleteObjectsWithPrefix(this.verbMetadataPrefix);
-            // Delete all objects in the index directory
-            await deleteObjectsWithPrefix(this.indexPrefix);
-            // v5.6.1: Delete COW (copy-on-write) version control data
-            // This includes all git-like versioning data (commits, trees, blobs, refs)
-            // Must be deleted to fully clear all data including version history
+            // v5.11.0: Clear ALL data using correct paths
+            // Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
+            await deleteObjectsWithPrefix('branches/');
+            // Delete COW version control data
             await deleteObjectsWithPrefix('_cow/');
-            // CRITICAL: Reset COW state to prevent automatic reinitialization
-            // When COW data is cleared, we must also clear the COW managers
-            // Otherwise initializeCOW() will auto-recreate initial commit on next operation
+            // Delete system metadata
+            await deleteObjectsWithPrefix('_system/');
+            // v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
+            // COW will re-initialize automatically on next use
             this.refManager = undefined;
             this.blobStorage = undefined;
             this.commitLog = undefined;
-            this.cowEnabled = false;
-            // v5.10.4: Create persistent marker object (CRITICAL FIX)
-            // Bug: cowEnabled = false only affects current instance, not future instances
-            // Fix: Create marker object that persists across instance restarts
-            // When new instance calls initializeCOW(), it checks for this marker
-            await this.createClearMarker();
             // Clear the statistics cache
             this.statisticsCache = null;
             this.statisticsModified = false;
@@ -1736,54 +1732,6 @@ export class S3CompatibleStorage extends BaseStorage {
             };
         }
     }
-    /**
-     * Check if COW has been explicitly disabled via clear()
-     * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
-     * @returns true if marker object exists, false otherwise
-     * @protected
-     */
-    async checkClearMarker() {
-        await this.ensureInitialized();
-        try {
-            const { HeadObjectCommand } = await import('@aws-sdk/client-s3');
-            const markerKey = `${this.systemPrefix}cow-disabled`;
-            await this.s3Client.send(new HeadObjectCommand({
-                Bucket: this.bucketName,
-                Key: markerKey
-            }));
-            return true; // Marker exists
-        }
-        catch (error) {
-            if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) {
-                return false; // Marker doesn't exist
-            }
-            prodLog.warn('S3CompatibleStorage.checkClearMarker: Error checking marker', error);
-            return false;
-        }
-    }
-    /**
-     * Create marker indicating COW has been explicitly disabled
-     * v5.10.4: Called by clear() to prevent COW reinitialization on new instances
-     * @protected
-     */
-    async createClearMarker() {
-        await this.ensureInitialized();
-        try {
-            const { PutObjectCommand } = await import('@aws-sdk/client-s3');
-            const markerKey = `${this.systemPrefix}cow-disabled`;
-            // Create empty marker object
-            await this.s3Client.send(new PutObjectCommand({
-                Bucket: this.bucketName,
-                Key: markerKey,
-                Body: Buffer.from(''),
-                ContentType: 'text/plain'
-            }));
-        }
-        catch (error) {
-            prodLog.error('S3CompatibleStorage.createClearMarker: Failed to create marker object', error);
-            // Don't throw - marker creation failure shouldn't break clear()
-        }
-    }
     /**
      * Get the statistics key for a specific date
      * @param date The date to get the key for

package/dist/storage/baseStorage.d.ts

@@ -58,7 +58,6 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
     blobStorage?: BlobStorage;
     commitLog?: CommitLog;
     currentBranch: string;
-    protected cowEnabled: boolean;
     protected nounCountsByType: Uint32Array<ArrayBuffer>;
     protected verbCountsByType: Uint32Array<ArrayBuffer>;
     protected nounTypeCache: Map<string, NounType>;
@@ -88,6 +87,8 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * Called during init() to ensure all data is stored with branch prefixes from the start
      * RefManager/BlobStorage/CommitLog are lazy-initialized on first fork()
      * @param branch - Branch name to use (default: 'main')
+     *
+     * v5.11.0: COW is always enabled - this method now just sets the branch name (idempotent)
      */
     enableCOWLightweight(branch?: string): void;
     /**
@@ -119,6 +120,8 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * Read object with inheritance from parent branches (COW layer)
      * Tries current branch first, then walks commit history
      * @protected - Available to subclasses for COW implementation
+     *
+     * v5.11.0: COW is always enabled - always use branch-scoped paths with inheritance
      */
     protected readWithInheritance(path: string, branch?: string): Promise<any | null>;
     /**
@@ -137,6 +140,8 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * This enables fork to see parent's data in pagination operations
      *
      * Simplified approach: All branches inherit from main
+     *
+     * v5.11.0: COW is always enabled - always use inheritance
      */
     protected listObjectsWithInheritance(prefix: string, branch?: string): Promise<string[]>;
     /**
@@ -328,20 +333,9 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      */
     abstract clear(): Promise<void>;
     /**
-     * Check if COW has been explicitly disabled via clear()
-     * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
-     * Each adapter checks for a marker file/object (e.g., "_system/cow-disabled")
-     * @returns true if COW was disabled by clear(), false otherwise
-     * @protected
-     */
-    protected abstract checkClearMarker(): Promise<boolean>;
-    /**
-     * Create marker indicating COW has been explicitly disabled
-     * v5.10.4: Called by clear() to prevent COW reinitialization on new instances
-     * Each adapter creates a marker file/object (e.g., "_system/cow-disabled")
-     * @protected
+     * v5.11.0: Removed checkClearMarker() and createClearMarker() abstract methods
+     * COW is now always enabled - marker files are no longer used
      */
-    protected abstract createClearMarker(): Promise<void>;
     /**
      * Get information about storage usage and capacity
      * This method should be implemented by each specific adapter
@@ -410,8 +404,44 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      */
     protected saveNounMetadata_internal(id: string, metadata: NounMetadata): Promise<void>;
     /**
-     * Get noun metadata from storage (v4.0.0: now typed)
-     * v5.4.0: Uses type-first paths (must match saveNounMetadata_internal)
+     * Get noun metadata from storage (METADATA-ONLY, NO VECTORS)
+     *
+     * **Performance (v5.11.1)**: Fast path for metadata-only reads
+     * - **Speed**: 10ms vs 43ms (76-81% faster than getNoun)
+     * - **Bandwidth**: 300 bytes vs 6KB (95% less)
+     * - **Memory**: 300 bytes vs 6KB (87% less)
+     *
+     * **What's included**:
+     * - All entity metadata (data, type, timestamps, confidence, weight)
+     * - Custom user fields
+     * - VFS metadata (_vfs.path, _vfs.size, etc.)
+     *
+     * **What's excluded**:
+     * - 384-dimensional vector embeddings
+     * - HNSW graph connections
+     *
+     * **Usage**:
+     * - VFS operations (readFile, stat, readdir) - 100% of cases
+     * - Existence checks: `if (await storage.getNounMetadata(id))`
+     * - Metadata inspection: `metadata.data`, `metadata.noun` (type)
+     * - Relationship traversal: Just need IDs, not vectors
+     *
+     * **When to use getNoun() instead**:
+     * - Computing similarity on this specific entity
+     * - Manual vector operations
+     * - HNSW graph traversal
+     *
+     * @param id - Entity ID to retrieve metadata for
+     * @returns Metadata or null if not found
+     *
+     * @performance
+     * - Type cache O(1) lookup for cached entities
+     * - Type scan O(N_types) for cache misses (typically <100ms)
+     * - Uses readWithInheritance() for COW branch support
+     *
+     * @since v4.0.0
+     * @since v5.4.0 - Type-first paths
+     * @since v5.11.1 - Promoted to fast path for brain.get() optimization
      */
     getNounMetadata(id: string): Promise<NounMetadata | null>;
     /**

package/dist/storage/baseStorage.js

@@ -83,7 +83,7 @@ export class BaseStorage extends BaseStorageAdapter {
         // Memory footprint: Bounded by batch size (typically <1000 items during imports)
         this.writeCache = new Map();
         this.currentBranch = 'main';
-        this.cowEnabled = false;
+        // v5.11.0: Removed cowEnabled flag - COW is ALWAYS enabled (mandatory, cannot be disabled)
         // Type-first indexing support (v5.4.0)
         // Built into all storage adapters for billion-scale efficiency
         this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT); // 168 bytes (Stage 3: 42 types)
@@ -191,13 +191,11 @@ export class BaseStorage extends BaseStorageAdapter {
      * Called during init() to ensure all data is stored with branch prefixes from the start
      * RefManager/BlobStorage/CommitLog are lazy-initialized on first fork()
      * @param branch - Branch name to use (default: 'main')
+     *
+     * v5.11.0: COW is always enabled - this method now just sets the branch name (idempotent)
      */
     enableCOWLightweight(branch = 'main') {
-        if (this.cowEnabled) {
-            return;
-        }
         this.currentBranch = branch;
-        this.cowEnabled = true;
         // RefManager/BlobStorage/CommitLog remain undefined until first fork()
     }
     /**
@@ -212,27 +210,15 @@ export class BaseStorage extends BaseStorageAdapter {
      * @returns Promise that resolves when COW is initialized
      */
     async initializeCOW(options) {
-        // v5.10.4: Check for persistent marker file (CRITICAL FIX)
-        // Bug: Setting cowEnabled = false on OLD instance doesn't affect NEW instances
-        // Fix: Check storage for persistent marker created by clear()
-        // If marker exists, COW was explicitly disabled and should NOT reinitialize
-        const markerExists = await this.checkClearMarker();
-        if (markerExists) {
-            return; // COW was disabled by clear() - don't recreate _cow/ directory
-        }
-        // v5.6.1: If COW was explicitly disabled (e.g., via clear()), don't reinitialize
-        // This prevents automatic recreation of COW data after clear() operations
-        if (this.cowEnabled === false) {
-            return;
-        }
-        // Check if RefManager already initialized (full COW setup complete)
-        if (this.refManager) {
+        // v5.11.0: COW is ALWAYS enabled - idempotent initialization only
+        // Removed marker file check (cowEnabled flag removed, COW is mandatory)
+        // Check if RefManager already initialized (idempotent)
+        if (this.refManager && this.blobStorage && this.commitLog) {
             return;
         }
-        // Enable lightweight COW if not already enabled
-        if (!this.cowEnabled) {
-            this.currentBranch = options?.branch || 'main';
-            this.cowEnabled = true;
+        // Set current branch if provided
+        if (options?.branch) {
+            this.currentBranch = options.branch;
         }
         // Create COWStorageAdapter bridge
         // This adapts BaseStorage's methods to the simple key-value interface
@@ -334,7 +320,7 @@ export class BaseStorage extends BaseStorageAdapter {
                 await this.refManager.setHead(this.currentBranch);
             }
         }
-        this.cowEnabled = true;
+        // v5.11.0: COW is always enabled - no flag to set
     }
     /**
      * Resolve branch-scoped path for COW isolation
@@ -348,9 +334,7 @@ export class BaseStorage extends BaseStorageAdapter {
         if (basePath.startsWith('_cow/')) {
             return basePath; // COW metadata is global across all branches
         }
-        if (!this.cowEnabled) {
-            return basePath; // COW disabled, use direct path
-        }
+        // v5.11.0: COW is always enabled - always use branch-scoped paths
         const targetBranch = branch || this.currentBranch || 'main';
         // Branch-scoped path: branches/<branch>/<basePath>
         return `branches/${targetBranch}/${basePath}`;
@@ -374,17 +358,10 @@ export class BaseStorage extends BaseStorageAdapter {
      * Read object with inheritance from parent branches (COW layer)
      * Tries current branch first, then walks commit history
      * @protected - Available to subclasses for COW implementation
+     *
+     * v5.11.0: COW is always enabled - always use branch-scoped paths with inheritance
      */
     async readWithInheritance(path, branch) {
-        if (!this.cowEnabled) {
-            // COW disabled: check write cache, then direct read
-            // v5.7.2: Check cache first for read-after-write consistency
-            const cachedData = this.writeCache.get(path);
-            if (cachedData !== undefined) {
-                return cachedData;
-            }
-            return this.readObjectFromPath(path);
-        }
         const targetBranch = branch || this.currentBranch || 'main';
         const branchPath = this.resolveBranchPath(path, targetBranch);
         // v5.7.2: Check write cache FIRST (synchronous, instant)
@@ -461,11 +438,10 @@ export class BaseStorage extends BaseStorageAdapter {
      * This enables fork to see parent's data in pagination operations
      *
      * Simplified approach: All branches inherit from main
+     *
+     * v5.11.0: COW is always enabled - always use inheritance
      */
     async listObjectsWithInheritance(prefix, branch) {
-        if (!this.cowEnabled) {
-            return this.listObjectsInBranch(prefix, branch);
-        }
         const targetBranch = branch || this.currentBranch || 'main';
         // Collect paths from current branch
         const pathsSet = new Set();
@@ -1433,8 +1409,44 @@ export class BaseStorage extends BaseStorageAdapter {
         }
     }
     /**
-     * Get noun metadata from storage (v4.0.0: now typed)
-     * v5.4.0: Uses type-first paths (must match saveNounMetadata_internal)
+     * Get noun metadata from storage (METADATA-ONLY, NO VECTORS)
+     *
+     * **Performance (v5.11.1)**: Fast path for metadata-only reads
+     * - **Speed**: 10ms vs 43ms (76-81% faster than getNoun)
+     * - **Bandwidth**: 300 bytes vs 6KB (95% less)
+     * - **Memory**: 300 bytes vs 6KB (87% less)
+     *
+     * **What's included**:
+     * - All entity metadata (data, type, timestamps, confidence, weight)
+     * - Custom user fields
+     * - VFS metadata (_vfs.path, _vfs.size, etc.)
+     *
+     * **What's excluded**:
+     * - 384-dimensional vector embeddings
+     * - HNSW graph connections
+     *
+     * **Usage**:
+     * - VFS operations (readFile, stat, readdir) - 100% of cases
+     * - Existence checks: `if (await storage.getNounMetadata(id))`
+     * - Metadata inspection: `metadata.data`, `metadata.noun` (type)
+     * - Relationship traversal: Just need IDs, not vectors
+     *
+     * **When to use getNoun() instead**:
+     * - Computing similarity on this specific entity
+     * - Manual vector operations
+     * - HNSW graph traversal
+     *
+     * @param id - Entity ID to retrieve metadata for
+     * @returns Metadata or null if not found
+     *
+     * @performance
+     * - Type cache O(1) lookup for cached entities
+     * - Type scan O(N_types) for cache misses (typically <100ms)
+     * - Uses readWithInheritance() for COW branch support
+     *
+     * @since v4.0.0
+     * @since v5.4.0 - Type-first paths
+     * @since v5.11.1 - Promoted to fast path for brain.get() optimization
      */
     async getNounMetadata(id) {
         await this.ensureInitialized();

package/dist/storage/cow/CommitLog.d.ts

@@ -115,6 +115,30 @@ export declare class CommitLog {
         since?: number;
         until?: number;
     }): Promise<CommitObject[]>;
+    /**
+     * Stream commit history (memory-efficient for large histories)
+     *
+     * Yields commits one at a time without accumulating in memory.
+     * Use this for large commit histories (1000s of commits) where
+     * memory efficiency is important.
+     *
+     * @param ref - Starting ref
+     * @param options - Walk options
+     * @yields Commits in reverse chronological order (newest first)
+     *
+     * @example
+     * ```typescript
+     * // Stream all commits without memory accumulation
+     * for await (const commit of commitLog.streamHistory('main', { maxCount: 10000 })) {
+     *   console.log(commit.message)
+     * }
+     * ```
+     */
+    streamHistory(ref: string, options?: {
+        maxCount?: number;
+        since?: number;
+        until?: number;
+    }): AsyncIterableIterator<CommitObject>;
     /**
      * Count commits between two commits
      *

package/dist/storage/cow/CommitLog.js

@@ -183,6 +183,43 @@ export class CommitLog {
         }
         return commits;
     }
+    /**
+     * Stream commit history (memory-efficient for large histories)
+     *
+     * Yields commits one at a time without accumulating in memory.
+     * Use this for large commit histories (1000s of commits) where
+     * memory efficiency is important.
+     *
+     * @param ref - Starting ref
+     * @param options - Walk options
+     * @yields Commits in reverse chronological order (newest first)
+     *
+     * @example
+     * ```typescript
+     * // Stream all commits without memory accumulation
+     * for await (const commit of commitLog.streamHistory('main', { maxCount: 10000 })) {
+     *   console.log(commit.message)
+     * }
+     * ```
+     */
+    async *streamHistory(ref, options) {
+        let count = 0;
+        for await (const commit of this.walk(ref, {
+            maxDepth: options?.maxCount,
+            until: options?.until
+        })) {
+            // Filter by since timestamp if provided
+            if (options?.since && commit.timestamp < options.since) {
+                continue;
+            }
+            yield commit;
+            count++;
+            // Stop after maxCount commits
+            if (options?.maxCount && count >= options.maxCount) {
+                break;
+            }
+        }
+    }
     /**
      * Count commits between two commits
      *

package/dist/types/brainy.types.d.ts

@@ -421,6 +421,63 @@ export interface ImportResult {
         error?: any;
     }>;
 }
+/**
+ * Options for brain.get() entity retrieval
+ *
+ * **Performance Optimization (v5.11.1)**:
+ * By default, brain.get() loads ONLY metadata (not vectors), resulting in:
+ * - **76-81% faster** reads (10ms vs 43ms for metadata-only)
+ * - **95% less bandwidth** (300 bytes vs 6KB per entity)
+ * - **87% less memory** (optimal for VFS and large-scale operations)
+ *
+ * **When to use includeVectors**:
+ * - Computing similarity on a specific entity (not search): `brain.similar({ to: entity.vector })`
+ * - Manual vector operations: `cosineSimilarity(entity.vector, otherVector)`
+ * - Inspecting embeddings for debugging
+ *
+ * **When NOT to use includeVectors** (metadata-only is sufficient):
+ * - VFS operations (readFile, stat, readdir) - 100% of cases
+ * - Existence checks: `if (await brain.get(id))`
+ * - Metadata inspection: `entity.metadata`, `entity.data`, `entity.type`
+ * - Relationship traversal: `brain.getRelations({ from: id })`
+ * - Search operations: `brain.find()` generates embeddings automatically
+ *
+ * @example
+ * ```typescript
+ * // ✅ FAST (default): Metadata-only - 10ms, 300 bytes
+ * const entity = await brain.get(id)
+ * console.log(entity.data, entity.metadata)  // ✅ Available
+ * console.log(entity.vector)  // Empty Float32Array (stub)
+ *
+ * // ✅ FULL: Load vectors when needed - 43ms, 6KB
+ * const fullEntity = await brain.get(id, { includeVectors: true })
+ * const similarity = cosineSimilarity(fullEntity.vector, otherVector)
+ *
+ * // ✅ VFS automatically uses fast path (no change needed)
+ * await vfs.readFile('/file.txt')  // 53ms → 10ms (81% faster)
+ * ```
+ *
+ * @since v5.11.1
+ */
+export interface GetOptions {
+    /**
+     * Include 384-dimensional vector embeddings in the response
+     *
+     * **Default: false** (metadata-only for 76-81% speedup)
+     *
+     * Set to `true` when you need to:
+     * - Compute similarity on this specific entity's vector
+     * - Perform manual vector operations
+     * - Inspect embeddings for debugging
+     *
+     * **Note**: Search operations (`brain.find()`) generate vectors automatically,
+     * so you don't need this flag for search. Only for direct vector operations
+     * on a retrieved entity.
+     *
+     * @default false
+     */
+    includeVectors?: boolean;
+}
 /**
  * Graph traversal parameters
  */
@@ -494,6 +551,8 @@ export interface BrainyConfig {
     disableAutoOptimize?: boolean;
     batchWrites?: boolean;
     maxConcurrentOperations?: number;
+    maxQueryLimit?: number;
+    reservedQueryMemory?: number;
     verbose?: boolean;
     silent?: boolean;
 }

package/dist/utils/paramValidation.d.ts

@@ -5,6 +5,49 @@
  * Only enforces universal truths, learns everything else
  */
 import { FindParams, AddParams, UpdateParams, RelateParams } from '../types/brainy.types.js';
+/**
+ * Configuration options for ValidationConfig
+ */
+export interface ValidationConfigOptions {
+    /**
+     * Explicit maximum query limit override
+     * Bypasses all auto-detection
+     */
+    maxQueryLimit?: number;
+    /**
+     * Memory reserved for query operations (in bytes)
+     * Bypasses auto-detection but still applies safety limits
+     */
+    reservedQueryMemory?: number;
+}
+/**
+ * Auto-configured limits based on system resources
+ * These adapt to available memory and observed performance
+ */
+export declare class ValidationConfig {
+    private static instance;
+    maxLimit: number;
+    maxQueryLength: number;
+    maxVectorDimensions: number;
+    limitBasis: 'override' | 'reservedMemory' | 'containerMemory' | 'freeMemory';
+    detectedContainerLimit: number | null;
+    private avgQueryTime;
+    private queryCount;
+    private constructor();
+    static getInstance(options?: ValidationConfigOptions): ValidationConfig;
+    /**
+     * Reset singleton (for testing or reconfiguration)
+     */
+    static reset(): void;
+    /**
+     * Reconfigure with new options
+     */
+    static reconfigure(options: ValidationConfigOptions): ValidationConfig;
+    /**
+     * Learn from actual usage to adjust limits
+     */
+    recordQuery(duration: number, resultCount: number): void;
+}
 /**
  * Universal validations - things that are always invalid
  * These are mathematical/logical truths, not configuration