@soulcraft/brainy 0.31.0 → 0.32.0

package/dist/unified.js

@@ -5766,9 +5766,12 @@ class BaseStorage extends BaseStorageAdapter {
     }
     /**
      * Get all nouns from storage
+     * @deprecated This method is deprecated and will be removed in a future version.
+     * It can cause memory issues with large datasets. Use getNouns() with pagination instead.
      */
     async getAllNouns() {
         await this.ensureInitialized();
+        console.warn('WARNING: getAllNouns() is deprecated and will be removed in a future version. Use getNouns() with pagination instead.');
         return this.getAllNouns_internal();
     }
     /**
@@ -5803,9 +5806,12 @@ class BaseStorage extends BaseStorageAdapter {
     }
     /**
      * Get all verbs from storage
+     * @deprecated This method is deprecated and will be removed in a future version.
+     * It can cause memory issues with large datasets. Use getVerbs() with pagination instead.
      */
     async getAllVerbs() {
         await this.ensureInitialized();
+        console.warn('WARNING: getAllVerbs() is deprecated and will be removed in a future version. Use getVerbs() with pagination instead.');
         return this.getAllVerbs_internal();
     }
     /**
@@ -5840,10 +5846,13 @@ class BaseStorage extends BaseStorageAdapter {
         const pagination = options?.pagination || {};
         const limit = pagination.limit || 100;
         const offset = pagination.offset || 0;
+        const cursor = pagination.cursor;
         // Optimize for common filter cases to avoid loading all nouns
         if (options?.filter) {
             // If filtering by nounType only, use the optimized method
-            if (options.filter.nounType && !options.filter.service && !options.filter.metadata) {
+            if (options.filter.nounType &&
+                !options.filter.service &&
+                !options.filter.metadata) {
                 const nounType = Array.isArray(options.filter.nounType)
                     ? options.filter.nounType[0]
                     : options.filter.nounType;
@@ -5866,81 +5875,124 @@ class BaseStorage extends BaseStorageAdapter {
                 };
             }
         }
-        // For more complex filtering or no filtering, we need to get all nouns
-        // but limit the number we load to avoid memory issues
-        const maxNouns = offset + limit + 1; // Get one extra to check if there are more
-        let allNouns = [];
+        // For more complex filtering or no filtering, use a paginated approach
+        // that avoids loading all nouns into memory at once
         try {
-            // Try to get only the nouns we need
-            allNouns = await this.getAllNouns_internal();
-            // If we have too many nouns, truncate the array to avoid memory issues
-            if (allNouns.length > maxNouns * 10) {
-                console.warn(`Large number of nouns (${allNouns.length}), truncating to ${maxNouns * 10} for filtering`);
-                allNouns = allNouns.slice(0, maxNouns * 10);
+            // First, try to get a count of total nouns (if the adapter supports it)
+            let totalCount = undefined;
+            try {
+                // This is an optional method that adapters may implement
+                if (typeof this.countNouns === 'function') {
+                    totalCount = await this.countNouns(options?.filter);
+                }
+            }
+            catch (countError) {
+                // Ignore errors from count method, it's optional
+                console.warn('Error getting noun count:', countError);
+            }
+            // Check if the adapter has a paginated method for getting nouns
+            if (typeof this.getNounsWithPagination === 'function') {
+                // Use the adapter's paginated method
+                const result = await this.getNounsWithPagination({
+                    limit,
+                    cursor,
+                    filter: options?.filter
+                });
+                // Apply offset if needed (some adapters might not support offset)
+                const items = result.items.slice(offset);
+                return {
+                    items,
+                    totalCount: result.totalCount || totalCount,
+                    hasMore: result.hasMore,
+                    nextCursor: result.nextCursor
+                };
+            }
+            // If the adapter doesn't have a paginated method, fall back to the old approach
+            // but with a warning and a reasonable limit
+            console.warn('Storage adapter does not support pagination, falling back to loading all nouns. This may cause performance issues with large datasets.');
+            // Get nouns with a reasonable limit to avoid memory issues
+            const maxNouns = Math.min(offset + limit + 100, 1000); // Reasonable limit
+            let allNouns = [];
+            try {
+                // Try to get only the nouns we need
+                allNouns = await this.getAllNouns_internal();
+                // If we have too many nouns, truncate the array to avoid memory issues
+                if (allNouns.length > maxNouns) {
+                    console.warn(`Large number of nouns (${allNouns.length}), truncating to ${maxNouns} for filtering`);
+                    allNouns = allNouns.slice(0, maxNouns);
+                }
+            }
+            catch (error) {
+                console.error('Error getting all nouns:', error);
+                // Return empty result on error
+                return {
+                    items: [],
+                    totalCount: 0,
+                    hasMore: false
+                };
+            }
+            // Apply filtering if needed
+            let filteredNouns = allNouns;
+            if (options?.filter) {
+                // Filter by noun type
+                if (options.filter.nounType) {
+                    const nounTypes = Array.isArray(options.filter.nounType)
+                        ? options.filter.nounType
+                        : [options.filter.nounType];
+                    filteredNouns = filteredNouns.filter((noun) => {
+                        // HNSWNoun doesn't have a type property directly, check metadata
+                        const nounType = noun.metadata?.type;
+                        return typeof nounType === 'string' && nounTypes.includes(nounType);
+                    });
+                }
+                // Filter by service
+                if (options.filter.service) {
+                    const services = Array.isArray(options.filter.service)
+                        ? options.filter.service
+                        : [options.filter.service];
+                    filteredNouns = filteredNouns.filter((noun) => {
+                        // HNSWNoun doesn't have a service property directly, check metadata
+                        const service = noun.metadata?.service;
+                        return typeof service === 'string' && services.includes(service);
+                    });
+                }
+                // Filter by metadata
+                if (options.filter.metadata) {
+                    const metadataFilter = options.filter.metadata;
+                    filteredNouns = filteredNouns.filter((noun) => {
+                        if (!noun.metadata)
+                            return false;
+                        // Check if all metadata keys match
+                        return Object.entries(metadataFilter).every(([key, value]) => noun.metadata && noun.metadata[key] === value);
+                    });
+                }
+            }
+            // Get total count before pagination
+            totalCount = totalCount || filteredNouns.length;
+            // Apply pagination
+            const paginatedNouns = filteredNouns.slice(offset, offset + limit);
+            const hasMore = offset + limit < filteredNouns.length || filteredNouns.length >= maxNouns;
+            // Set next cursor if there are more items
+            let nextCursor = undefined;
+            if (hasMore && paginatedNouns.length > 0) {
+                const lastItem = paginatedNouns[paginatedNouns.length - 1];
+                nextCursor = lastItem.id;
             }
+            return {
+                items: paginatedNouns,
+                totalCount,
+                hasMore,
+                nextCursor
+            };
         }
         catch (error) {
-            console.error('Error getting all nouns:', error);
-            // Return empty result on error
+            console.error('Error getting nouns with pagination:', error);
             return {
                 items: [],
                 totalCount: 0,
                 hasMore: false
             };
         }
-        // Apply filtering if needed
-        let filteredNouns = allNouns;
-        if (options?.filter) {
-            // Filter by noun type
-            if (options.filter.nounType) {
-                const nounTypes = Array.isArray(options.filter.nounType)
-                    ? options.filter.nounType
-                    : [options.filter.nounType];
-                filteredNouns = filteredNouns.filter(noun => {
-                    // HNSWNoun doesn't have a type property directly, check metadata
-                    const nounType = noun.metadata?.type;
-                    return typeof nounType === 'string' && nounTypes.includes(nounType);
-                });
-            }
-            // Filter by service
-            if (options.filter.service) {
-                const services = Array.isArray(options.filter.service)
-                    ? options.filter.service
-                    : [options.filter.service];
-                filteredNouns = filteredNouns.filter(noun => {
-                    // HNSWNoun doesn't have a service property directly, check metadata
-                    const service = noun.metadata?.service;
-                    return typeof service === 'string' && services.includes(service);
-                });
-            }
-            // Filter by metadata
-            if (options.filter.metadata) {
-                const metadataFilter = options.filter.metadata;
-                filteredNouns = filteredNouns.filter(noun => {
-                    if (!noun.metadata)
-                        return false;
-                    // Check if all metadata keys match
-                    return Object.entries(metadataFilter).every(([key, value]) => noun.metadata && noun.metadata[key] === value);
-                });
-            }
-        }
-        // Get total count before pagination
-        const totalCount = filteredNouns.length;
-        // Apply pagination
-        const paginatedNouns = filteredNouns.slice(offset, offset + limit);
-        const hasMore = offset + limit < totalCount;
-        // Set next cursor if there are more items
-        let nextCursor = undefined;
-        if (hasMore && paginatedNouns.length > 0) {
-            const lastItem = paginatedNouns[paginatedNouns.length - 1];
-            nextCursor = lastItem.id;
-        }
-        return {
-            items: paginatedNouns,
-            totalCount,
-            hasMore,
-            nextCursor
-        };
     }
     /**
      * Get verbs with pagination and filtering
@@ -5953,11 +6005,14 @@ class BaseStorage extends BaseStorageAdapter {
         const pagination = options?.pagination || {};
         const limit = pagination.limit || 100;
         const offset = pagination.offset || 0;
+        const cursor = pagination.cursor;
         // Optimize for common filter cases to avoid loading all verbs
         if (options?.filter) {
             // If filtering by sourceId only, use the optimized method
-            if (options.filter.sourceId && !options.filter.verbType &&
-                !options.filter.targetId && !options.filter.service &&
+            if (options.filter.sourceId &&
+                !options.filter.verbType &&
+                !options.filter.targetId &&
+                !options.filter.service &&
                 !options.filter.metadata) {
                 const sourceId = Array.isArray(options.filter.sourceId)
                     ? options.filter.sourceId[0]
@@ -5981,8 +6036,10 @@ class BaseStorage extends BaseStorageAdapter {
                 };
             }
             // If filtering by targetId only, use the optimized method
-            if (options.filter.targetId && !options.filter.verbType &&
-                !options.filter.sourceId && !options.filter.service &&
+            if (options.filter.targetId &&
+                !options.filter.verbType &&
+                !options.filter.sourceId &&
+                !options.filter.service &&
                 !options.filter.metadata) {
                 const targetId = Array.isArray(options.filter.targetId)
                     ? options.filter.targetId[0]
@@ -6006,8 +6063,10 @@ class BaseStorage extends BaseStorageAdapter {
                 };
             }
             // If filtering by verbType only, use the optimized method
-            if (options.filter.verbType && !options.filter.sourceId &&
-                !options.filter.targetId && !options.filter.service &&
+            if (options.filter.verbType &&
+                !options.filter.sourceId &&
+                !options.filter.targetId &&
+                !options.filter.service &&
                 !options.filter.metadata) {
                 const verbType = Array.isArray(options.filter.verbType)
                     ? options.filter.verbType[0]
@@ -6031,91 +6090,134 @@ class BaseStorage extends BaseStorageAdapter {
                 };
             }
         }
-        // For more complex filtering or no filtering, we need to get all verbs
-        // but limit the number we load to avoid memory issues
-        const maxVerbs = offset + limit + 1; // Get one extra to check if there are more
-        let allVerbs = [];
+        // For more complex filtering or no filtering, use a paginated approach
+        // that avoids loading all verbs into memory at once
         try {
-            // Try to get only the verbs we need
-            allVerbs = await this.getAllVerbs_internal();
-            // If we have too many verbs, truncate the array to avoid memory issues
-            if (allVerbs.length > maxVerbs * 10) {
-                console.warn(`Large number of verbs (${allVerbs.length}), truncating to ${maxVerbs * 10} for filtering`);
-                allVerbs = allVerbs.slice(0, maxVerbs * 10);
+            // First, try to get a count of total verbs (if the adapter supports it)
+            let totalCount = undefined;
+            try {
+                // This is an optional method that adapters may implement
+                if (typeof this.countVerbs === 'function') {
+                    totalCount = await this.countVerbs(options?.filter);
+                }
+            }
+            catch (countError) {
+                // Ignore errors from count method, it's optional
+                console.warn('Error getting verb count:', countError);
+            }
+            // Check if the adapter has a paginated method for getting verbs
+            if (typeof this.getVerbsWithPagination === 'function') {
+                // Use the adapter's paginated method
+                const result = await this.getVerbsWithPagination({
+                    limit,
+                    cursor,
+                    filter: options?.filter
+                });
+                // Apply offset if needed (some adapters might not support offset)
+                const items = result.items.slice(offset);
+                return {
+                    items,
+                    totalCount: result.totalCount || totalCount,
+                    hasMore: result.hasMore,
+                    nextCursor: result.nextCursor
+                };
+            }
+            // If the adapter doesn't have a paginated method, fall back to the old approach
+            // but with a warning and a reasonable limit
+            console.warn('Storage adapter does not support pagination, falling back to loading all verbs. This may cause performance issues with large datasets.');
+            // Get verbs with a reasonable limit to avoid memory issues
+            const maxVerbs = Math.min(offset + limit + 100, 1000); // Reasonable limit
+            let allVerbs = [];
+            try {
+                // Try to get only the verbs we need
+                allVerbs = await this.getAllVerbs_internal();
+                // If we have too many verbs, truncate the array to avoid memory issues
+                if (allVerbs.length > maxVerbs) {
+                    console.warn(`Large number of verbs (${allVerbs.length}), truncating to ${maxVerbs} for filtering`);
+                    allVerbs = allVerbs.slice(0, maxVerbs);
+                }
             }
+            catch (error) {
+                console.error('Error getting all verbs:', error);
+                // Return empty result on error
+                return {
+                    items: [],
+                    totalCount: 0,
+                    hasMore: false
+                };
+            }
+            // Apply filtering if needed
+            let filteredVerbs = allVerbs;
+            if (options?.filter) {
+                // Filter by verb type
+                if (options.filter.verbType) {
+                    const verbTypes = Array.isArray(options.filter.verbType)
+                        ? options.filter.verbType
+                        : [options.filter.verbType];
+                    filteredVerbs = filteredVerbs.filter((verb) => verb.type !== undefined && verbTypes.includes(verb.type));
+                }
+                // Filter by source ID
+                if (options.filter.sourceId) {
+                    const sourceIds = Array.isArray(options.filter.sourceId)
+                        ? options.filter.sourceId
+                        : [options.filter.sourceId];
+                    filteredVerbs = filteredVerbs.filter((verb) => verb.sourceId !== undefined && sourceIds.includes(verb.sourceId));
+                }
+                // Filter by target ID
+                if (options.filter.targetId) {
+                    const targetIds = Array.isArray(options.filter.targetId)
+                        ? options.filter.targetId
+                        : [options.filter.targetId];
+                    filteredVerbs = filteredVerbs.filter((verb) => verb.targetId !== undefined && targetIds.includes(verb.targetId));
+                }
+                // Filter by service
+                if (options.filter.service) {
+                    const services = Array.isArray(options.filter.service)
+                        ? options.filter.service
+                        : [options.filter.service];
+                    filteredVerbs = filteredVerbs.filter((verb) => {
+                        // GraphVerb doesn't have a service property directly, check metadata
+                        const service = verb.metadata?.service;
+                        return typeof service === 'string' && services.includes(service);
+                    });
+                }
+                // Filter by metadata
+                if (options.filter.metadata) {
+                    const metadataFilter = options.filter.metadata;
+                    filteredVerbs = filteredVerbs.filter((verb) => {
+                        if (!verb.metadata)
+                            return false;
+                        // Check if all metadata keys match
+                        return Object.entries(metadataFilter).every(([key, value]) => verb.metadata && verb.metadata[key] === value);
+                    });
+                }
+            }
+            // Get total count before pagination
+            totalCount = totalCount || filteredVerbs.length;
+            // Apply pagination
+            const paginatedVerbs = filteredVerbs.slice(offset, offset + limit);
+            const hasMore = offset + limit < filteredVerbs.length || filteredVerbs.length >= maxVerbs;
+            // Set next cursor if there are more items
+            let nextCursor = undefined;
+            if (hasMore && paginatedVerbs.length > 0) {
+                const lastItem = paginatedVerbs[paginatedVerbs.length - 1];
+                nextCursor = lastItem.id;
+            }
+            return {
+                items: paginatedVerbs,
+                totalCount,
+                hasMore,
+                nextCursor
+            };
         }
         catch (error) {
-            console.error('Error getting all verbs:', error);
-            // Return empty result on error
+            console.error('Error getting verbs with pagination:', error);
             return {
                 items: [],
                 totalCount: 0,
                 hasMore: false
             };
         }
-        // Apply filtering if needed
-        let filteredVerbs = allVerbs;
-        if (options?.filter) {
-            // Filter by verb type
-            if (options.filter.verbType) {
-                const verbTypes = Array.isArray(options.filter.verbType)
-                    ? options.filter.verbType
-                    : [options.filter.verbType];
-                filteredVerbs = filteredVerbs.filter(verb => verb.type !== undefined && verbTypes.includes(verb.type));
-            }
-            // Filter by source ID
-            if (options.filter.sourceId) {
-                const sourceIds = Array.isArray(options.filter.sourceId)
-                    ? options.filter.sourceId
-                    : [options.filter.sourceId];
-                filteredVerbs = filteredVerbs.filter(verb => verb.sourceId !== undefined && sourceIds.includes(verb.sourceId));
-            }
-            // Filter by target ID
-            if (options.filter.targetId) {
-                const targetIds = Array.isArray(options.filter.targetId)
-                    ? options.filter.targetId
-                    : [options.filter.targetId];
-                filteredVerbs = filteredVerbs.filter(verb => verb.targetId !== undefined && targetIds.includes(verb.targetId));
-            }
-            // Filter by service
-            if (options.filter.service) {
-                const services = Array.isArray(options.filter.service)
-                    ? options.filter.service
-                    : [options.filter.service];
-                filteredVerbs = filteredVerbs.filter(verb => {
-                    // GraphVerb doesn't have a service property directly, check metadata
-                    const service = verb.metadata?.service;
-                    return typeof service === 'string' && services.includes(service);
-                });
-            }
-            // Filter by metadata
-            if (options.filter.metadata) {
-                const metadataFilter = options.filter.metadata;
-                filteredVerbs = filteredVerbs.filter(verb => {
-                    if (!verb.metadata)
-                        return false;
-                    // Check if all metadata keys match
-                    return Object.entries(metadataFilter).every(([key, value]) => verb.metadata && verb.metadata[key] === value);
-                });
-            }
-        }
-        // Get total count before pagination
-        const totalCount = filteredVerbs.length;
-        // Apply pagination
-        const paginatedVerbs = filteredVerbs.slice(offset, offset + limit);
-        const hasMore = offset + limit < totalCount;
-        // Set next cursor if there are more items
-        let nextCursor = undefined;
-        if (hasMore && paginatedVerbs.length > 0) {
-            const lastItem = paginatedVerbs[paginatedVerbs.length - 1];
-            nextCursor = lastItem.id;
-        }
-        return {
-            items: paginatedVerbs,
-            totalCount,
-            hasMore,
-            nextCursor
-        };
     }
     /**
      * Delete a verb from storage
@@ -7912,22 +8014,853 @@ class StorageOperationExecutors {
         this.deleteExecutor = createOperationExecutor('delete', config);
     }
     /**
-     * Execute a get operation with timeout and retry
+     * Execute a get operation with timeout and retry
+     */
+    async executeGet(operation, operationName) {
+        return this.getExecutor(operation, operationName);
+    }
+    /**
+     * Execute an add operation with timeout and retry
+     */
+    async executeAdd(operation, operationName) {
+        return this.addExecutor(operation, operationName);
+    }
+    /**
+     * Execute a delete operation with timeout and retry
+     */
+    async executeDelete(operation, operationName) {
+        return this.deleteExecutor(operation, operationName);
+    }
+}
+
+/**
+ * Multi-level Cache Manager
+ *
+ * Implements a three-level caching strategy:
+ * - Level 1: Hot cache (most accessed nodes) - RAM (automatically detecting and adjusting in each environment)
+ * - Level 2: Warm cache (recent nodes) - OPFS, Filesystem or S3 depending on environment
+ * - Level 3: Cold storage (all nodes) - OPFS, Filesystem or S3 depending on environment
+ */
+// Environment detection for storage selection
+var Environment$1;
+(function (Environment) {
+    Environment[Environment["BROWSER"] = 0] = "BROWSER";
+    Environment[Environment["NODE"] = 1] = "NODE";
+    Environment[Environment["WORKER"] = 2] = "WORKER";
+})(Environment$1 || (Environment$1 = {}));
+// Storage type for warm and cold caches
+var StorageType;
+(function (StorageType) {
+    StorageType[StorageType["MEMORY"] = 0] = "MEMORY";
+    StorageType[StorageType["OPFS"] = 1] = "OPFS";
+    StorageType[StorageType["FILESYSTEM"] = 2] = "FILESYSTEM";
+    StorageType[StorageType["S3"] = 3] = "S3";
+})(StorageType || (StorageType = {}));
+/**
+ * Multi-level cache manager for efficient data access
+ */
+class CacheManager {
+    /**
+     * Initialize the cache manager
+     * @param options Configuration options
+     */
+    constructor(options = {}) {
+        // Hot cache (RAM)
+        this.hotCache = new Map();
+        // Cache statistics
+        this.stats = {
+            hits: 0,
+            misses: 0,
+            evictions: 0,
+            size: 0,
+            maxSize: 0
+        };
+        this.lastAutoTuneTime = 0;
+        this.autoTuneInterval = 5 * 60 * 1000; // 5 minutes
+        this.storageStatistics = null;
+        // Detect environment
+        this.environment = this.detectEnvironment();
+        // Set storage types based on environment
+        this.warmStorageType = this.detectWarmStorageType();
+        this.coldStorageType = this.detectColdStorageType();
+        // Initialize storage adapters
+        this.warmStorage = options.warmStorage || this.initializeWarmStorage();
+        this.coldStorage = options.coldStorage || this.initializeColdStorage();
+        // Set auto-tuning flag
+        this.autoTune = options.autoTune !== undefined ? options.autoTune : true;
+        // Set default values or use provided values
+        this.hotCacheMaxSize = options.hotCacheMaxSize || this.detectOptimalCacheSize();
+        this.hotCacheEvictionThreshold = options.hotCacheEvictionThreshold || 0.8;
+        this.warmCacheTTL = options.warmCacheTTL || 24 * 60 * 60 * 1000; // 24 hours
+        this.batchSize = options.batchSize || 10;
+        // If auto-tuning is enabled, perform initial tuning
+        if (this.autoTune) {
+            this.tuneParameters();
+        }
+        // Log configuration
+        if (process.env.DEBUG) {
+            console.log('Cache Manager initialized with configuration:', {
+                environment: Environment$1[this.environment],
+                hotCacheMaxSize: this.hotCacheMaxSize,
+                hotCacheEvictionThreshold: this.hotCacheEvictionThreshold,
+                warmCacheTTL: this.warmCacheTTL,
+                batchSize: this.batchSize,
+                autoTune: this.autoTune,
+                warmStorageType: StorageType[this.warmStorageType],
+                coldStorageType: StorageType[this.coldStorageType]
+            });
+        }
+    }
+    /**
+     * Detect the current environment
+     */
+    detectEnvironment() {
+        if (typeof window !== 'undefined' && typeof document !== 'undefined') {
+            return Environment$1.BROWSER;
+        }
+        else if (typeof self !== 'undefined' && typeof window === 'undefined') {
+            // In a worker environment, self is defined but window is not
+            return Environment$1.WORKER;
+        }
+        else {
+            return Environment$1.NODE;
+        }
+    }
+    /**
+     * Detect the optimal cache size based on available memory
+     */
+    detectOptimalCacheSize() {
+        try {
+            // Default to a conservative value
+            const defaultSize = 1000;
+            // In Node.js, use available system memory
+            if (this.environment === Environment$1.NODE) {
+                try {
+                    // Use dynamic import to avoid ESLint warning
+                    const getOS = () => {
+                        // eslint-disable-next-line @typescript-eslint/no-var-requires
+                        return require('os');
+                    };
+                    const os = getOS();
+                    const freeMemory = os.freemem();
+                    // Estimate average entry size (in bytes)
+                    // This is a conservative estimate for complex objects with vectors
+                    const ESTIMATED_BYTES_PER_ENTRY = 1024; // 1KB per entry
+                    // Use 10% of free memory, with a minimum of 1000 entries
+                    const optimalSize = Math.max(Math.floor(freeMemory * 0.1 / ESTIMATED_BYTES_PER_ENTRY), 1000);
+                    return optimalSize;
+                }
+                catch (error) {
+                    console.warn('Failed to detect optimal cache size:', error);
+                    return defaultSize;
+                }
+            }
+            // In browser, use navigator.deviceMemory if available
+            if (this.environment === Environment$1.BROWSER && navigator.deviceMemory) {
+                // deviceMemory is in GB, scale accordingly
+                return Math.max(navigator.deviceMemory * 500, 1000);
+            }
+            return defaultSize;
+        }
+        catch (error) {
+            console.warn('Error detecting optimal cache size:', error);
+            return 1000; // Conservative default
+        }
+    }
+    /**
+     * Tune cache parameters based on statistics and environment
+     * This method is called periodically if auto-tuning is enabled
+     *
+     * The auto-tuning process:
+     * 1. Retrieves storage statistics if available
+     * 2. Tunes each parameter based on statistics and environment
+     * 3. Logs the tuned parameters if debug is enabled
+     *
+     * Auto-tuning helps optimize cache performance by adapting to:
+     * - The current environment (Node.js, browser, worker)
+     * - Available system resources (memory, CPU)
+     * - Usage patterns (read-heavy vs. write-heavy workloads)
+     * - Cache efficiency (hit/miss ratios)
+     */
+    async tuneParameters() {
+        // Skip if auto-tuning is disabled
+        if (!this.autoTune)
+            return;
+        // Check if it's time to tune parameters
+        const now = Date.now();
+        if (now - this.lastAutoTuneTime < this.autoTuneInterval)
+            return;
+        // Update last tune time
+        this.lastAutoTuneTime = now;
+        try {
+            // Get storage statistics if available
+            if (this.coldStorage && typeof this.coldStorage.getStatistics === 'function') {
+                this.storageStatistics = await this.coldStorage.getStatistics();
+            }
+            // Tune hot cache size
+            this.tuneHotCacheSize();
+            // Tune eviction threshold
+            this.tuneEvictionThreshold();
+            // Tune warm cache TTL
+            this.tuneWarmCacheTTL();
+            // Tune batch size
+            this.tuneBatchSize();
+            // Log tuned parameters if debug is enabled
+            if (process.env.DEBUG) {
+                console.log('Cache parameters auto-tuned:', {
+                    hotCacheMaxSize: this.hotCacheMaxSize,
+                    hotCacheEvictionThreshold: this.hotCacheEvictionThreshold,
+                    warmCacheTTL: this.warmCacheTTL,
+                    batchSize: this.batchSize
+                });
+            }
+        }
+        catch (error) {
+            console.warn('Error during cache parameter auto-tuning:', error);
+        }
+    }
+    /**
+     * Tune hot cache size based on statistics and environment
+     *
+     * The hot cache size is tuned based on:
+     * 1. Available memory in the current environment
+     * 2. Total number of nodes and edges in the system
+     * 3. Cache hit/miss ratio
+     *
+     * Algorithm:
+     * - Start with a size based on available memory
+     * - If storage statistics are available, consider caching a percentage of total items
+     * - If hit ratio is low, increase the cache size to improve performance
+     * - Ensure a reasonable minimum size to maintain basic functionality
+     */
+    tuneHotCacheSize() {
+        // Start with the base size from environment detection
+        let optimalSize = this.detectOptimalCacheSize();
+        // If we have storage statistics, adjust based on total nodes/edges
+        if (this.storageStatistics) {
+            const totalItems = (this.storageStatistics.totalNodes || 0) +
+                (this.storageStatistics.totalEdges || 0);
+            // If total items is significant, adjust cache size
+            if (totalItems > 0) {
+                // Use a percentage of total items, with a cap based on memory
+                const percentageToCache = 0.2; // Cache 20% of items by default
+                const statisticsBasedSize = Math.ceil(totalItems * percentageToCache);
+                // Use the smaller of the two to avoid memory issues
+                optimalSize = Math.min(optimalSize, statisticsBasedSize);
+            }
+        }
+        // Adjust based on hit/miss ratio if we have enough data
+        const totalAccesses = this.stats.hits + this.stats.misses;
+        if (totalAccesses > 100) {
+            const hitRatio = this.stats.hits / totalAccesses;
+            // If hit ratio is high, we might have a good cache size already
+            // If hit ratio is low, we might need a larger cache
+            if (hitRatio < 0.5) {
+                // Increase cache size by up to 50% if hit ratio is low
+                const hitRatioFactor = 1 + (0.5 - hitRatio);
+                optimalSize = Math.ceil(optimalSize * hitRatioFactor);
+            }
+        }
+        // Ensure we have a reasonable minimum size
+        optimalSize = Math.max(optimalSize, 1000);
+        // Update the hot cache max size
+        this.hotCacheMaxSize = optimalSize;
+        this.stats.maxSize = optimalSize;
+    }
+    /**
+     * Tune eviction threshold based on statistics
+     *
+     * The eviction threshold determines when items start being evicted from the hot cache.
+     * It is tuned based on:
+     * 1. Cache hit/miss ratio
+     * 2. Operation patterns (read-heavy vs. write-heavy workloads)
+     *
+     * Algorithm:
+     * - Start with a default threshold of 0.8 (80% of max size)
+     * - For high hit ratios, increase the threshold to keep more items in cache
+     * - For low hit ratios, decrease the threshold to evict items more aggressively
+     * - For read-heavy workloads, use a higher threshold
+     * - For write-heavy workloads, use a lower threshold
+     */
+    tuneEvictionThreshold() {
+        // Default threshold
+        let threshold = 0.8;
+        // Adjust based on hit/miss ratio if we have enough data
+        const totalAccesses = this.stats.hits + this.stats.misses;
+        if (totalAccesses > 100) {
+            const hitRatio = this.stats.hits / totalAccesses;
+            // If hit ratio is high, we can use a higher threshold
+            // If hit ratio is low, we should use a lower threshold to evict more aggressively
+            if (hitRatio > 0.8) {
+                // High hit ratio, increase threshold (up to 0.9)
+                threshold = Math.min(0.9, 0.8 + (hitRatio - 0.8));
+            }
+            else if (hitRatio < 0.5) {
+                // Low hit ratio, decrease threshold (down to 0.6)
+                threshold = Math.max(0.6, 0.8 - (0.5 - hitRatio));
+            }
+        }
+        // If we have storage statistics with operation counts, adjust based on operation patterns
+        if (this.storageStatistics && this.storageStatistics.operations) {
+            const ops = this.storageStatistics.operations;
+            const totalOps = ops.total || 1;
+            // Calculate read/write ratio
+            const readOps = ops.search || 0;
+            const writeOps = (ops.add || 0) + (ops.update || 0) + (ops.delete || 0);
+            if (totalOps > 100) {
+                const readRatio = readOps / totalOps;
+                const writeRatio = writeOps / totalOps;
+                // For read-heavy workloads, use higher threshold
+                // For write-heavy workloads, use lower threshold
+                if (readRatio > 0.8) {
+                    // Read-heavy, increase threshold slightly
+                    threshold = Math.min(0.9, threshold + 0.05);
+                }
+                else if (writeRatio > 0.5) {
+                    // Write-heavy, decrease threshold
+                    threshold = Math.max(0.6, threshold - 0.1);
+                }
+            }
+        }
+        // Update the eviction threshold
+        this.hotCacheEvictionThreshold = threshold;
+    }
+    /**
+     * Tune warm cache TTL based on statistics
+     *
+     * The warm cache TTL determines how long items remain in the warm cache.
+     * It is tuned based on:
+     * 1. Update frequency from operation statistics
+     *
+     * Algorithm:
+     * - Start with a default TTL of 24 hours
+     * - For frequently updated data, use a shorter TTL
+     * - For rarely updated data, use a longer TTL
+     */
+    tuneWarmCacheTTL() {
+        // Default TTL (24 hours)
+        let ttl = 24 * 60 * 60 * 1000;
+        // If we have storage statistics with operation counts, adjust based on update frequency
+        if (this.storageStatistics && this.storageStatistics.operations) {
+            const ops = this.storageStatistics.operations;
+            const totalOps = ops.total || 1;
+            const updateOps = (ops.update || 0);
+            if (totalOps > 100) {
+                const updateRatio = updateOps / totalOps;
+                // For frequently updated data, use shorter TTL
+                // For rarely updated data, use longer TTL
+                if (updateRatio > 0.3) {
+                    // Frequently updated, decrease TTL (down to 6 hours)
+                    ttl = Math.max(6 * 60 * 60 * 1000, ttl * (1 - updateRatio));
+                }
+                else if (updateRatio < 0.1) {
+                    // Rarely updated, increase TTL (up to 48 hours)
+                    ttl = Math.min(48 * 60 * 60 * 1000, ttl * (1.5 - updateRatio));
+                }
+            }
+        }
+        // Update the warm cache TTL
+        this.warmCacheTTL = ttl;
+    }
+    /**
+     * Tune batch size based on statistics and environment
+     *
+     * The batch size determines how many items are processed in a single batch
+     * for operations like prefetching. It is tuned based on:
+     * 1. Current environment (Node.js, browser, worker)
+     * 2. Available memory
+     * 3. Operation patterns
+     * 4. Cache hit/miss ratio
+     *
+     * Algorithm:
+     * - Start with a default based on the environment
+     * - Adjust based on available memory in browsers
+     * - For bulk-heavy workloads, use a larger batch size
+     * - For high hit ratios, use smaller batches (items likely in cache)
+     * - For low hit ratios, use larger batches (need to fetch more items)
+     */
+    tuneBatchSize() {
+        // Default batch size
+        let batchSize = 10;
+        // Adjust based on environment
+        if (this.environment === Environment$1.NODE) {
+            // Node.js can handle larger batches
+            batchSize = 20;
+        }
+        else if (this.environment === Environment$1.BROWSER) {
+            // Browsers might need smaller batches
+            batchSize = 10;
+            // If we have memory information, adjust accordingly
+            if (navigator.deviceMemory) {
+                // Scale batch size with available memory
+                batchSize = Math.max(5, Math.min(20, Math.floor(navigator.deviceMemory * 2)));
+            }
+        }
+        // If we have storage statistics with operation counts, adjust based on operation patterns
+        if (this.storageStatistics && this.storageStatistics.operations) {
+            const ops = this.storageStatistics.operations;
+            const totalOps = ops.total || 1;
+            const bulkOps = (ops.search || 0);
+            if (totalOps > 100) {
+                const bulkRatio = bulkOps / totalOps;
+                // For bulk-heavy workloads, use larger batch size
+                if (bulkRatio > 0.7) {
+                    // Bulk-heavy, increase batch size (up to 2x)
+                    batchSize = Math.min(50, Math.ceil(batchSize * 1.5));
+                }
+            }
+        }
+        // Adjust based on hit/miss ratio if we have enough data
+        const totalAccesses = this.stats.hits + this.stats.misses;
+        if (totalAccesses > 100) {
+            const hitRatio = this.stats.hits / totalAccesses;
+            // If hit ratio is high, we can use smaller batches
+            // If hit ratio is low, we might need larger batches
+            if (hitRatio > 0.8) {
+                // High hit ratio, decrease batch size slightly
+                batchSize = Math.max(5, Math.floor(batchSize * 0.8));
+            }
+            else if (hitRatio < 0.5) {
+                // Low hit ratio, increase batch size
+                batchSize = Math.min(50, Math.ceil(batchSize * 1.2));
+            }
+        }
+        // Update the batch size
+        this.batchSize = batchSize;
+    }
+    /**
+     * Detect the appropriate warm storage type based on environment
+     */
+    detectWarmStorageType() {
+        if (this.environment === Environment$1.BROWSER) {
+            // Use OPFS if available, otherwise use memory
+            if ('storage' in navigator && 'getDirectory' in navigator.storage) {
+                return StorageType.OPFS;
+            }
+            return StorageType.MEMORY;
+        }
+        else if (this.environment === Environment$1.WORKER) {
+            // Use OPFS if available, otherwise use memory
+            if ('storage' in self && 'getDirectory' in self.storage) {
+                return StorageType.OPFS;
+            }
+            return StorageType.MEMORY;
+        }
+        else {
+            // In Node.js, use filesystem
+            return StorageType.FILESYSTEM;
+        }
+    }
+    /**
+     * Detect the appropriate cold storage type based on environment
+     */
+    detectColdStorageType() {
+        if (this.environment === Environment$1.BROWSER) {
+            // Use OPFS if available, otherwise use memory
+            if ('storage' in navigator && 'getDirectory' in navigator.storage) {
+                return StorageType.OPFS;
+            }
+            return StorageType.MEMORY;
+        }
+        else if (this.environment === Environment$1.WORKER) {
+            // Use OPFS if available, otherwise use memory
+            if ('storage' in self && 'getDirectory' in self.storage) {
+                return StorageType.OPFS;
+            }
+            return StorageType.MEMORY;
+        }
+        else {
+            // In Node.js, use S3 if configured, otherwise filesystem
+            return StorageType.S3;
+        }
+    }
+    /**
+     * Initialize warm storage adapter
+     */
+    initializeWarmStorage() {
+        // Implementation depends on the detected storage type
+        // For now, return null as this will be provided by the storage adapter
+        return null;
+    }
+    /**
+     * Initialize cold storage adapter
+     */
+    initializeColdStorage() {
+        // Implementation depends on the detected storage type
+        // For now, return null as this will be provided by the storage adapter
+        return null;
+    }
+    /**
+     * Get an item from cache, trying each level in order
+     * @param id The item ID
+     * @returns The cached item or null if not found
+     */
+    async get(id) {
+        // Check if it's time to tune parameters
+        await this.checkAndTuneParameters();
+        // Try hot cache first (fastest)
+        const hotCacheEntry = this.hotCache.get(id);
+        if (hotCacheEntry) {
+            // Update access metadata
+            hotCacheEntry.lastAccessed = Date.now();
+            hotCacheEntry.accessCount++;
+            // Update stats
+            this.stats.hits++;
+            return hotCacheEntry.data;
+        }
+        // Try warm cache next
+        try {
+            const warmCacheItem = await this.getFromWarmCache(id);
+            if (warmCacheItem) {
+                // Promote to hot cache
+                this.addToHotCache(id, warmCacheItem);
+                // Update stats
+                this.stats.hits++;
+                return warmCacheItem;
+            }
+        }
+        catch (error) {
+            console.warn(`Error accessing warm cache for ${id}:`, error);
+        }
+        // Finally, try cold storage
+        try {
+            const coldStorageItem = await this.getFromColdStorage(id);
+            if (coldStorageItem) {
+                // Promote to hot and warm caches
+                this.addToHotCache(id, coldStorageItem);
+                await this.addToWarmCache(id, coldStorageItem);
+                // Update stats
+                this.stats.misses++;
+                return coldStorageItem;
+            }
+        }
+        catch (error) {
+            console.warn(`Error accessing cold storage for ${id}:`, error);
+        }
+        // Item not found in any cache level
+        this.stats.misses++;
+        return null;
+    }
+    /**
+     * Get an item from warm cache
+     * @param id The item ID
+     * @returns The cached item or null if not found
+     */
+    async getFromWarmCache(id) {
+        if (!this.warmStorage)
+            return null;
+        try {
+            return await this.warmStorage.get(id);
+        }
+        catch (error) {
+            console.warn(`Error getting item ${id} from warm cache:`, error);
+            return null;
+        }
+    }
+    /**
+     * Get an item from cold storage
+     * @param id The item ID
+     * @returns The item or null if not found
+     */
+    async getFromColdStorage(id) {
+        if (!this.coldStorage)
+            return null;
+        try {
+            return await this.coldStorage.get(id);
+        }
+        catch (error) {
+            console.warn(`Error getting item ${id} from cold storage:`, error);
+            return null;
+        }
+    }
+    /**
+     * Add an item to hot cache
+     * @param id The item ID
+     * @param item The item to cache
+     */
+    addToHotCache(id, item) {
+        // Check if we need to evict items
+        if (this.hotCache.size >= this.hotCacheMaxSize * this.hotCacheEvictionThreshold) {
+            this.evictFromHotCache();
+        }
+        // Add to hot cache
+        this.hotCache.set(id, {
+            data: item,
+            lastAccessed: Date.now(),
+            accessCount: 1,
+            expiresAt: null // Hot cache items don't expire
+        });
+        // Update stats
+        this.stats.size = this.hotCache.size;
+    }
+    /**
+     * Add an item to warm cache
+     * @param id The item ID
+     * @param item The item to cache
+     */
+    async addToWarmCache(id, item) {
+        if (!this.warmStorage)
+            return;
+        try {
+            // Add to warm cache with TTL
+            await this.warmStorage.set(id, item, {
+                ttl: this.warmCacheTTL
+            });
+        }
+        catch (error) {
+            console.warn(`Error adding item ${id} to warm cache:`, error);
+        }
+    }
+    /**
+     * Evict items from hot cache based on LRU policy
+     */
+    evictFromHotCache() {
+        // Find the least recently used items
+        const entries = Array.from(this.hotCache.entries());
+        // Sort by last accessed time (oldest first)
+        entries.sort((a, b) => a[1].lastAccessed - b[1].lastAccessed);
+        // Remove the oldest 20% of items
+        const itemsToRemove = Math.ceil(this.hotCache.size * 0.2);
+        for (let i = 0; i < itemsToRemove && i < entries.length; i++) {
+            this.hotCache.delete(entries[i][0]);
+            this.stats.evictions++;
+        }
+        // Update stats
+        this.stats.size = this.hotCache.size;
+        if (process.env.DEBUG) {
+            console.log(`Evicted ${itemsToRemove} items from hot cache, new size: ${this.hotCache.size}`);
+        }
+    }
+    /**
+     * Set an item in all cache levels
+     * @param id The item ID
+     * @param item The item to cache
+     */
+    async set(id, item) {
+        // Add to hot cache
+        this.addToHotCache(id, item);
+        // Add to warm cache
+        await this.addToWarmCache(id, item);
+        // Add to cold storage
+        if (this.coldStorage) {
+            try {
+                await this.coldStorage.set(id, item);
+            }
+            catch (error) {
+                console.warn(`Error adding item ${id} to cold storage:`, error);
+            }
+        }
+    }
+    /**
+     * Delete an item from all cache levels
+     * @param id The item ID to delete
+     */
+    async delete(id) {
+        // Remove from hot cache
+        this.hotCache.delete(id);
+        // Remove from warm cache
+        if (this.warmStorage) {
+            try {
+                await this.warmStorage.delete(id);
+            }
+            catch (error) {
+                console.warn(`Error deleting item ${id} from warm cache:`, error);
+            }
+        }
+        // Remove from cold storage
+        if (this.coldStorage) {
+            try {
+                await this.coldStorage.delete(id);
+            }
+            catch (error) {
+                console.warn(`Error deleting item ${id} from cold storage:`, error);
+            }
+        }
+        // Update stats
+        this.stats.size = this.hotCache.size;
+    }
+    /**
+     * Clear all cache levels
+     */
+    async clear() {
+        // Clear hot cache
+        this.hotCache.clear();
+        // Clear warm cache
+        if (this.warmStorage) {
+            try {
+                await this.warmStorage.clear();
+            }
+            catch (error) {
+                console.warn('Error clearing warm cache:', error);
+            }
+        }
+        // Clear cold storage
+        if (this.coldStorage) {
+            try {
+                await this.coldStorage.clear();
+            }
+            catch (error) {
+                console.warn('Error clearing cold storage:', error);
+            }
+        }
+        // Reset stats
+        this.stats = {
+            hits: 0,
+            misses: 0,
+            evictions: 0,
+            size: 0,
+            maxSize: this.hotCacheMaxSize
+        };
+    }
+    /**
+     * Get cache statistics
+     * @returns Cache statistics
      */
-    async executeGet(operation, operationName) {
-        return this.getExecutor(operation, operationName);
+    getStats() {
+        return { ...this.stats };
     }
     /**
-     * Execute an add operation with timeout and retry
+     * Prefetch items based on ID patterns or relationships
+     * @param ids Array of IDs to prefetch
      */
-    async executeAdd(operation, operationName) {
-        return this.addExecutor(operation, operationName);
+    async prefetch(ids) {
+        // Check if it's time to tune parameters
+        await this.checkAndTuneParameters();
+        // Prefetch in batches to avoid overwhelming the system
+        const batches = [];
+        // Split into batches using the configurable batch size
+        for (let i = 0; i < ids.length; i += this.batchSize) {
+            const batch = ids.slice(i, i + this.batchSize);
+            batches.push(batch);
+        }
+        // Process each batch
+        for (const batch of batches) {
+            await Promise.all(batch.map(async (id) => {
+                // Skip if already in hot cache
+                if (this.hotCache.has(id))
+                    return;
+                try {
+                    // Try to get from any cache level
+                    await this.get(id);
+                }
+                catch (error) {
+                    // Ignore errors during prefetching
+                    if (process.env.DEBUG) {
+                        console.warn(`Error prefetching ${id}:`, error);
+                    }
+                }
+            }));
+        }
     }
     /**
-     * Execute a delete operation with timeout and retry
+     * Check if it's time to tune parameters and do so if needed
+     * This is called before operations that might benefit from tuned parameters
+     *
+     * This method serves as a checkpoint for auto-tuning, ensuring that:
+     * 1. Parameters are tuned periodically based on the auto-tune interval
+     * 2. Tuning happens before critical operations that would benefit from optimized parameters
+     * 3. Tuning doesn't happen too frequently, which could impact performance
+     *
+     * By calling this method before get(), getMany(), and prefetch() operations,
+     * we ensure that the cache parameters are optimized for the current workload
+     * without adding unnecessary overhead to every operation.
      */
-    async executeDelete(operation, operationName) {
-        return this.deleteExecutor(operation, operationName);
+    async checkAndTuneParameters() {
+        // Skip if auto-tuning is disabled
+        if (!this.autoTune)
+            return;
+        // Check if it's time to tune parameters
+        const now = Date.now();
+        if (now - this.lastAutoTuneTime >= this.autoTuneInterval) {
+            await this.tuneParameters();
+        }
+    }
+    /**
+     * Get multiple items at once, optimizing for batch retrieval
+     * @param ids Array of IDs to get
+     * @returns Map of ID to item
+     */
+    async getMany(ids) {
+        // Check if it's time to tune parameters
+        await this.checkAndTuneParameters();
+        const result = new Map();
+        // First check hot cache for all IDs
+        const missingIds = [];
+        for (const id of ids) {
+            const hotCacheEntry = this.hotCache.get(id);
+            if (hotCacheEntry) {
+                // Update access metadata
+                hotCacheEntry.lastAccessed = Date.now();
+                hotCacheEntry.accessCount++;
+                // Add to result
+                result.set(id, hotCacheEntry.data);
+                // Update stats
+                this.stats.hits++;
+            }
+            else {
+                missingIds.push(id);
+            }
+        }
+        if (missingIds.length === 0) {
+            return result;
+        }
+        // Try to get missing items from warm cache
+        if (this.warmStorage) {
+            try {
+                const warmCacheItems = await this.warmStorage.getMany(missingIds);
+                for (const [id, item] of warmCacheItems.entries()) {
+                    if (item) {
+                        // Promote to hot cache
+                        this.addToHotCache(id, item);
+                        // Add to result
+                        result.set(id, item);
+                        // Update stats
+                        this.stats.hits++;
+                        // Remove from missing IDs
+                        const index = missingIds.indexOf(id);
+                        if (index !== -1) {
+                            missingIds.splice(index, 1);
+                        }
+                    }
+                }
+            }
+            catch (error) {
+                console.warn('Error accessing warm cache for batch:', error);
+            }
+        }
+        if (missingIds.length === 0) {
+            return result;
+        }
+        // Try to get remaining missing items from cold storage
+        if (this.coldStorage) {
+            try {
+                const coldStorageItems = await this.coldStorage.getMany(missingIds);
+                for (const [id, item] of coldStorageItems.entries()) {
+                    if (item) {
+                        // Promote to hot and warm caches
+                        this.addToHotCache(id, item);
+                        await this.addToWarmCache(id, item);
+                        // Add to result
+                        result.set(id, item);
+                        // Update stats
+                        this.stats.misses++;
+                    }
+                }
+            }
+            catch (error) {
+                console.warn('Error accessing cold storage for batch:', error);
+            }
+        }
+        return result;
+    }
+    /**
+     * Set the storage adapters for warm and cold caches
+     * @param warmStorage Warm cache storage adapter
+     * @param coldStorage Cold storage adapter
+     */
+    setStorageAdapters(warmStorage, coldStorage) {
+        this.warmStorage = warmStorage;
+        this.coldStorage = coldStorage;
     }
 }
 
@@ -7973,6 +8906,8 @@ class S3CompatibleStorage extends BaseStorage {
         this.activeLocks = new Set();
         // Change log for efficient synchronization
         this.changeLogPrefix = 'change-log/';
+        // Node cache to avoid redundant API calls
+        this.nodeCache = new Map();
         // Batch update timer ID
         this.statisticsBatchUpdateTimerId = null;
         // Flag to indicate if statistics have been modified since last save
@@ -7998,6 +8933,9 @@ class S3CompatibleStorage extends BaseStorage {
         this.verbPrefix = `${VERBS_DIR}/`;
         this.metadataPrefix = `${METADATA_DIR}/`;
         this.indexPrefix = `${INDEX_DIR}/`;
+        // Initialize cache managers
+        this.nounCacheManager = new CacheManager(options.cacheConfig);
+        this.verbCacheManager = new CacheManager(options.cacheConfig);
     }
     /**
      * Initialize the storage adapter
@@ -8036,6 +8974,78 @@ class S3CompatibleStorage extends BaseStorage {
             await this.s3Client.send(new HeadBucketCommand({
                 Bucket: this.bucketName
             }));
+            // Create storage adapter proxies for the cache managers
+            const nounStorageAdapter = {
+                get: async (id) => this.getNoun_internal(id),
+                set: async (id, node) => this.saveNoun_internal(node),
+                delete: async (id) => this.deleteNoun_internal(id),
+                getMany: async (ids) => {
+                    const result = new Map();
+                    // Process in batches to avoid overwhelming the S3 API
+                    const batchSize = 10;
+                    const batches = [];
+                    // Split into batches
+                    for (let i = 0; i < ids.length; i += batchSize) {
+                        const batch = ids.slice(i, i + batchSize);
+                        batches.push(batch);
+                    }
+                    // Process each batch
+                    for (const batch of batches) {
+                        const batchResults = await Promise.all(batch.map(async (id) => {
+                            const node = await this.getNoun_internal(id);
+                            return { id, node };
+                        }));
+                        // Add results to map
+                        for (const { id, node } of batchResults) {
+                            if (node) {
+                                result.set(id, node);
+                            }
+                        }
+                    }
+                    return result;
+                },
+                clear: async () => {
+                    // No-op for now, as we don't want to clear the entire storage
+                    // This would be implemented if needed
+                }
+            };
+            const verbStorageAdapter = {
+                get: async (id) => this.getVerb_internal(id),
+                set: async (id, edge) => this.saveVerb_internal(edge),
+                delete: async (id) => this.deleteVerb_internal(id),
+                getMany: async (ids) => {
+                    const result = new Map();
+                    // Process in batches to avoid overwhelming the S3 API
+                    const batchSize = 10;
+                    const batches = [];
+                    // Split into batches
+                    for (let i = 0; i < ids.length; i += batchSize) {
+                        const batch = ids.slice(i, i + batchSize);
+                        batches.push(batch);
+                    }
+                    // Process each batch
+                    for (const batch of batches) {
+                        const batchResults = await Promise.all(batch.map(async (id) => {
+                            const edge = await this.getVerb_internal(id);
+                            return { id, edge };
+                        }));
+                        // Add results to map
+                        for (const { id, edge } of batchResults) {
+                            if (edge) {
+                                result.set(id, edge);
+                            }
+                        }
+                    }
+                    return result;
+                },
+                clear: async () => {
+                    // No-op for now, as we don't want to clear the entire storage
+                    // This would be implemented if needed
+                }
+            };
+            // Set storage adapters for cache managers
+            this.nounCacheManager.setStorageAdapters(nounStorageAdapter, nounStorageAdapter);
+            this.verbCacheManager.setStorageAdapters(verbStorageAdapter, verbStorageAdapter);
             this.isInitialized = true;
         }
         catch (error) {
@@ -8144,7 +9154,10 @@ class S3CompatibleStorage extends BaseStorage {
                 const parsedNode = JSON.parse(bodyContents);
                 console.log(`Parsed node data for ${id}:`, parsedNode);
                 // Ensure the parsed node has the expected properties
-                if (!parsedNode || !parsedNode.id || !parsedNode.vector || !parsedNode.connections) {
+                if (!parsedNode ||
+                    !parsedNode.id ||
+                    !parsedNode.vector ||
+                    !parsedNode.connections) {
                     console.error(`Invalid node data for ${id}:`, parsedNode);
                     return null;
                 }
@@ -8180,100 +9193,118 @@ class S3CompatibleStorage extends BaseStorage {
     }
     /**
      * Get all nodes from storage
+     * @deprecated This method is deprecated and will be removed in a future version.
+     * It can cause memory issues with large datasets. Use getNodesWithPagination() instead.
      */
     async getAllNodes() {
         await this.ensureInitialized();
+        console.warn('WARNING: getAllNodes() is deprecated and will be removed in a future version. Use getNodesWithPagination() instead.');
+        try {
+            // Use the paginated method with a large limit to maintain backward compatibility
+            // but warn about potential issues
+            const result = await this.getNodesWithPagination({
+                limit: 1000, // Reasonable limit to avoid memory issues
+                useCache: true
+            });
+            if (result.hasMore) {
+                console.warn(`WARNING: Only returning the first 1000 nodes. There are more nodes available. Use getNodesWithPagination() for proper pagination.`);
+            }
+            return result.nodes;
+        }
+        catch (error) {
+            console.error('Failed to get all nodes:', error);
+            return [];
+        }
+    }
+    /**
+     * Get nodes with pagination
+     * @param options Pagination options
+     * @returns Promise that resolves to a paginated result of nodes
+     */
+    async getNodesWithPagination(options = {}) {
+        await this.ensureInitialized();
+        const limit = options.limit || 100;
+        const useCache = options.useCache !== false;
         try {
             // Import the ListObjectsV2Command and GetObjectCommand only when needed
-            const { ListObjectsV2Command, GetObjectCommand } = await import('@aws-sdk/client-s3');
-            console.log(`Getting all nodes from bucket ${this.bucketName} with prefix ${this.nounPrefix}`);
-            // List all objects in the nouns directory
+            const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
+            // List objects with pagination
             const listResponse = await this.s3Client.send(new ListObjectsV2Command({
                 Bucket: this.bucketName,
-                Prefix: this.nounPrefix
+                Prefix: this.nounPrefix,
+                MaxKeys: limit,
+                ContinuationToken: options.cursor
             }));
+            // If listResponse is null/undefined or there are no objects, return an empty result
+            if (!listResponse ||
+                !listResponse.Contents ||
+                listResponse.Contents.length === 0) {
+                return {
+                    nodes: [],
+                    hasMore: false
+                };
+            }
+            // Extract node IDs from the keys
+            const nodeIds = listResponse.Contents
+                .filter((object) => object && object.Key)
+                .map((object) => object.Key.replace(this.nounPrefix, '').replace('.json', ''));
+            // Use the cache manager to get nodes efficiently
             const nodes = [];
-            // If listResponse is null/undefined or there are no objects, return an empty array
-            if (!listResponse || !listResponse.Contents || listResponse.Contents.length === 0) {
-                console.log(`No nodes found in bucket ${this.bucketName} with prefix ${this.nounPrefix}`);
-                return nodes;
-            }
-            console.log(`Found ${listResponse.Contents.length} nodes in bucket ${this.bucketName}`);
-            // Debug: Log all keys found
-            console.log('Keys found:');
-            for (const object of listResponse.Contents) {
-                if (object && object.Key) {
-                    console.log(`- ${object.Key}`);
+            if (useCache) {
+                // Get nodes from cache manager
+                const cachedNodes = await this.nounCacheManager.getMany(nodeIds);
+                // Add nodes to result in the same order as nodeIds
+                for (const id of nodeIds) {
+                    const node = cachedNodes.get(id);
+                    if (node) {
+                        nodes.push(node);
+                    }
                 }
             }
-            // Get each node
-            const nodePromises = listResponse.Contents.map(async (object) => {
-                if (!object || !object.Key) {
-                    console.log(`Skipping undefined object or object without Key`);
-                    return null;
+            else {
+                // Get nodes directly from S3 without using cache
+                // Process in smaller batches to reduce memory usage
+                const batchSize = 50;
+                const batches = [];
+                // Split into batches
+                for (let i = 0; i < nodeIds.length; i += batchSize) {
+                    const batch = nodeIds.slice(i, i + batchSize);
+                    batches.push(batch);
                 }
-                try {
-                    // Extract node ID from the key (remove prefix and .json extension)
-                    const nodeId = object.Key.replace(this.nounPrefix, '').replace('.json', '');
-                    console.log(`Getting node with ID ${nodeId} from key ${object.Key}`);
-                    // Get the node data
-                    const response = await this.s3Client.send(new GetObjectCommand({
-                        Bucket: this.bucketName,
-                        Key: object.Key
-                    }));
-                    // Check if response is null or undefined
-                    if (!response || !response.Body) {
-                        console.log(`No response or response body for node ${nodeId}`);
-                        return null;
-                    }
-                    // Convert the response body to a string
-                    const bodyContents = await response.Body.transformToString();
-                    console.log(`Retrieved node body for ${nodeId}: ${bodyContents.substring(0, 100)}${bodyContents.length > 100 ? '...' : ''}`);
-                    // Parse the JSON string
-                    try {
-                        const parsedNode = JSON.parse(bodyContents);
-                        console.log(`Parsed node data for ${nodeId}:`, parsedNode);
-                        // Ensure the parsed node has the expected properties
-                        if (!parsedNode || !parsedNode.id || !parsedNode.vector || !parsedNode.connections) {
-                            console.error(`Invalid node data for ${nodeId}:`, parsedNode);
+                // Process each batch sequentially
+                for (const batch of batches) {
+                    const batchNodes = await Promise.all(batch.map(async (id) => {
+                        try {
+                            return await this.getNoun_internal(id);
+                        }
+                        catch (error) {
                             return null;
                         }
-                        // Convert serialized connections back to Map<number, Set<string>>
-                        const connections = new Map();
-                        for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
-                            connections.set(Number(level), new Set(nodeIds));
+                    }));
+                    // Add non-null nodes to result
+                    for (const node of batchNodes) {
+                        if (node) {
+                            nodes.push(node);
                         }
-                        const node = {
-                            id: parsedNode.id,
-                            vector: parsedNode.vector,
-                            connections
-                        };
-                        console.log(`Successfully retrieved node ${nodeId}:`, node);
-                        return node;
-                    }
-                    catch (parseError) {
-                        console.error(`Failed to parse node data for ${nodeId}:`, parseError);
-                        return null;
                     }
                 }
-                catch (error) {
-                    console.error(`Error getting node from ${object.Key}:`, error);
-                    return null;
-                }
-            });
-            // Wait for all promises to resolve and filter out nulls
-            const resolvedNodes = await Promise.all(nodePromises);
-            const filteredNodes = resolvedNodes.filter((node) => node !== null);
-            console.log(`Returning ${filteredNodes.length} nodes`);
-            // Debug: Log all nodes being returned
-            for (const node of filteredNodes) {
-                console.log(`- Node ${node.id}`);
             }
-            return filteredNodes;
+            // Determine if there are more nodes
+            const hasMore = !!listResponse.IsTruncated;
+            // Set next cursor if there are more nodes
+            const nextCursor = listResponse.NextContinuationToken;
+            return {
+                nodes,
+                hasMore,
+                nextCursor
+            };
         }
         catch (error) {
-            console.error('Failed to get all nodes:', error);
-            return [];
+            console.error('Failed to get nodes with pagination:', error);
+            return {
+                nodes: [],
+                hasMore: false
+            };
         }
     }
     /**
@@ -8292,14 +9323,31 @@ class S3CompatibleStorage extends BaseStorage {
     async getNodesByNounType(nounType) {
         await this.ensureInitialized();
         try {
-            // Get all nodes
-            const allNodes = await this.getAllNodes();
-            // Filter nodes by noun type using metadata
             const filteredNodes = [];
-            for (const node of allNodes) {
-                const metadata = await this.getMetadata(node.id);
-                if (metadata && metadata.noun === nounType) {
-                    filteredNodes.push(node);
+            let hasMore = true;
+            let cursor = undefined;
+            // Use pagination to process nodes in batches
+            while (hasMore) {
+                // Get a batch of nodes
+                const result = await this.getNodesWithPagination({
+                    limit: 100,
+                    cursor,
+                    useCache: true
+                });
+                // Filter nodes by noun type using metadata
+                for (const node of result.nodes) {
+                    const metadata = await this.getMetadata(node.id);
+                    if (metadata && metadata.noun === nounType) {
+                        filteredNodes.push(node);
+                    }
+                }
+                // Update pagination state
+                hasMore = result.hasMore;
+                cursor = result.nextCursor;
+                // Safety check to prevent infinite loops
+                if (!cursor && hasMore) {
+                    console.warn('No cursor returned but hasMore is true, breaking loop');
+                    break;
                 }
             }
             return filteredNodes;
@@ -8422,7 +9470,10 @@ class S3CompatibleStorage extends BaseStorage {
                 const parsedEdge = JSON.parse(bodyContents);
                 console.log(`Parsed edge data for ${id}:`, parsedEdge);
                 // Ensure the parsed edge has the expected properties
-                if (!parsedEdge || !parsedEdge.id || !parsedEdge.vector || !parsedEdge.connections ||
+                if (!parsedEdge ||
+                    !parsedEdge.id ||
+                    !parsedEdge.vector ||
+                    !parsedEdge.connections ||
                     !(parsedEdge.sourceId || parsedEdge.source) ||
                     !(parsedEdge.targetId || parsedEdge.target) ||
                     !(parsedEdge.type || parsedEdge.verb)) {
@@ -8476,86 +9527,205 @@ class S3CompatibleStorage extends BaseStorage {
     }
     /**
      * Get all verbs from storage (internal implementation)
+     * @deprecated This method is deprecated and will be removed in a future version.
+     * It can cause memory issues with large datasets. Use getVerbsWithPagination() instead.
      */
     async getAllVerbs_internal() {
+        console.warn('WARNING: getAllVerbs_internal() is deprecated and will be removed in a future version. Use getVerbsWithPagination() instead.');
         return this.getAllEdges();
     }
     /**
      * Get all edges from storage
+     * @deprecated This method is deprecated and will be removed in a future version.
+     * It can cause memory issues with large datasets. Use getEdgesWithPagination() instead.
      */
     async getAllEdges() {
         await this.ensureInitialized();
+        console.warn('WARNING: getAllEdges() is deprecated and will be removed in a future version. Use getEdgesWithPagination() instead.');
         try {
-            // Import the ListObjectsV2Command and GetObjectCommand only when needed
-            const { ListObjectsV2Command, GetObjectCommand } = await import('@aws-sdk/client-s3');
-            // List all objects in the verbs directory
+            // Use the paginated method with a large limit to maintain backward compatibility
+            // but warn about potential issues
+            const result = await this.getEdgesWithPagination({
+                limit: 1000, // Reasonable limit to avoid memory issues
+                useCache: true
+            });
+            if (result.hasMore) {
+                console.warn(`WARNING: Only returning the first 1000 edges. There are more edges available. Use getEdgesWithPagination() for proper pagination.`);
+            }
+            return result.edges;
+        }
+        catch (error) {
+            console.error('Failed to get all edges:', error);
+            return [];
+        }
+    }
+    /**
+     * Get edges with pagination
+     * @param options Pagination options
+     * @returns Promise that resolves to a paginated result of edges
+     */
+    async getEdgesWithPagination(options = {}) {
+        await this.ensureInitialized();
+        const limit = options.limit || 100;
+        const useCache = options.useCache !== false;
+        const filter = options.filter || {};
+        try {
+            // Import the ListObjectsV2Command only when needed
+            const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
+            // List objects with pagination
             const listResponse = await this.s3Client.send(new ListObjectsV2Command({
                 Bucket: this.bucketName,
-                Prefix: this.verbPrefix
+                Prefix: this.verbPrefix,
+                MaxKeys: limit,
+                ContinuationToken: options.cursor
             }));
+            // If listResponse is null/undefined or there are no objects, return an empty result
+            if (!listResponse ||
+                !listResponse.Contents ||
+                listResponse.Contents.length === 0) {
+                return {
+                    edges: [],
+                    hasMore: false
+                };
+            }
+            // Extract edge IDs from the keys
+            const edgeIds = listResponse.Contents
+                .filter((object) => object && object.Key)
+                .map((object) => object.Key.replace(this.verbPrefix, '').replace('.json', ''));
+            // Use the cache manager to get edges efficiently
             const edges = [];
-            // If there are no objects, return an empty array
-            if (!listResponse.Contents || listResponse.Contents.length === 0) {
-                return edges;
+            if (useCache) {
+                // Get edges from cache manager
+                const cachedEdges = await this.verbCacheManager.getMany(edgeIds);
+                // Add edges to result in the same order as edgeIds
+                for (const id of edgeIds) {
+                    const edge = cachedEdges.get(id);
+                    if (edge) {
+                        // Apply filtering if needed
+                        if (this.filterEdge(edge, filter)) {
+                            edges.push(edge);
+                        }
+                    }
+                }
             }
-            // Get each edge
-            const edgePromises = listResponse.Contents.map(async (object) => {
-                try {
-                    // Extract edge ID from the key (remove prefix and .json extension)
-                    const edgeId = object.Key.replace(this.verbPrefix, '').replace('.json', '');
-                    // Get the edge data
-                    const response = await this.s3Client.send(new GetObjectCommand({
-                        Bucket: this.bucketName,
-                        Key: object.Key
+            else {
+                // Get edges directly from S3 without using cache
+                // Process in smaller batches to reduce memory usage
+                const batchSize = 50;
+                const batches = [];
+                // Split into batches
+                for (let i = 0; i < edgeIds.length; i += batchSize) {
+                    const batch = edgeIds.slice(i, i + batchSize);
+                    batches.push(batch);
+                }
+                // Process each batch sequentially
+                for (const batch of batches) {
+                    const batchEdges = await Promise.all(batch.map(async (id) => {
+                        try {
+                            const edge = await this.getVerb_internal(id);
+                            // Apply filtering if needed
+                            if (edge && this.filterEdge(edge, filter)) {
+                                return edge;
+                            }
+                            return null;
+                        }
+                        catch (error) {
+                            return null;
+                        }
                     }));
-                    // Convert the response body to a string
-                    const bodyContents = await response.Body.transformToString();
-                    const parsedEdge = JSON.parse(bodyContents);
-                    // Convert serialized connections back to Map<number, Set<string>>
-                    const connections = new Map();
-                    for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
-                        connections.set(Number(level), new Set(nodeIds));
+                    // Add non-null edges to result
+                    for (const edge of batchEdges) {
+                        if (edge) {
+                            edges.push(edge);
+                        }
                     }
-                    // Create default timestamp if not present
-                    const defaultTimestamp = {
-                        seconds: Math.floor(Date.now() / 1000),
-                        nanoseconds: (Date.now() % 1000) * 1000000
-                    };
-                    // Create default createdBy if not present
-                    const defaultCreatedBy = {
-                        augmentation: 'unknown',
-                        version: '1.0'
-                    };
-                    return {
-                        id: parsedEdge.id,
-                        vector: parsedEdge.vector,
-                        connections,
-                        sourceId: parsedEdge.sourceId || parsedEdge.source,
-                        targetId: parsedEdge.targetId || parsedEdge.target,
-                        source: parsedEdge.sourceId || parsedEdge.source,
-                        target: parsedEdge.targetId || parsedEdge.target,
-                        verb: parsedEdge.type || parsedEdge.verb,
-                        type: parsedEdge.type || parsedEdge.verb,
-                        weight: parsedEdge.weight || 1.0,
-                        metadata: parsedEdge.metadata || {},
-                        createdAt: parsedEdge.createdAt || defaultTimestamp,
-                        updatedAt: parsedEdge.updatedAt || defaultTimestamp,
-                        createdBy: parsedEdge.createdBy || defaultCreatedBy
-                    };
-                }
-                catch (error) {
-                    console.error(`Error getting edge from ${object.Key}:`, error);
-                    return null;
                 }
-            });
-            // Wait for all promises to resolve and filter out nulls
-            const resolvedEdges = await Promise.all(edgePromises);
-            return resolvedEdges.filter((edge) => edge !== null);
+            }
+            // Determine if there are more edges
+            const hasMore = !!listResponse.IsTruncated;
+            // Set next cursor if there are more edges
+            const nextCursor = listResponse.NextContinuationToken;
+            return {
+                edges,
+                hasMore,
+                nextCursor
+            };
         }
         catch (error) {
-            console.error('Failed to get all edges:', error);
-            return [];
+            console.error('Failed to get edges with pagination:', error);
+            return {
+                edges: [],
+                hasMore: false
+            };
+        }
+    }
+    /**
+     * Filter an edge based on filter criteria
+     * @param edge The edge to filter
+     * @param filter The filter criteria
+     * @returns True if the edge matches the filter, false otherwise
+     */
+    filterEdge(edge, filter) {
+        // If no filter, include all edges
+        if (!filter.sourceId && !filter.targetId && !filter.type) {
+            return true;
+        }
+        // Filter by source ID
+        if (filter.sourceId && edge.sourceId !== filter.sourceId) {
+            return false;
+        }
+        // Filter by target ID
+        if (filter.targetId && edge.targetId !== filter.targetId) {
+            return false;
+        }
+        // Filter by type
+        if (filter.type && edge.type !== filter.type) {
+            return false;
+        }
+        return true;
+    }
+    /**
+     * Get verbs with pagination
+     * @param options Pagination options
+     * @returns Promise that resolves to a paginated result of verbs
+     */
+    async getVerbsWithPagination(options = {}) {
+        await this.ensureInitialized();
+        // Convert filter to edge filter format
+        const edgeFilter = {};
+        if (options.filter) {
+            // Handle sourceId filter
+            if (options.filter.sourceId) {
+                edgeFilter.sourceId = Array.isArray(options.filter.sourceId)
+                    ? options.filter.sourceId[0]
+                    : options.filter.sourceId;
+            }
+            // Handle targetId filter
+            if (options.filter.targetId) {
+                edgeFilter.targetId = Array.isArray(options.filter.targetId)
+                    ? options.filter.targetId[0]
+                    : options.filter.targetId;
+            }
+            // Handle verbType filter
+            if (options.filter.verbType) {
+                edgeFilter.type = Array.isArray(options.filter.verbType)
+                    ? options.filter.verbType[0]
+                    : options.filter.verbType;
+            }
         }
+        // Get edges with pagination
+        const result = await this.getEdgesWithPagination({
+            limit: options.limit,
+            cursor: options.cursor,
+            useCache: true,
+            filter: edgeFilter
+        });
+        // Convert edges to verbs (they're the same in this implementation)
+        return {
+            items: result.edges,
+            hasMore: result.hasMore,
+            nextCursor: result.nextCursor
+        };
     }
     /**
      * Get verbs by source (internal implementation)
@@ -8722,9 +9892,10 @@ class S3CompatibleStorage extends BaseStorage {
                 // In AWS SDK, this would be error.name === 'NoSuchKey'
                 // In our mock, we might get different error types
                 if (error.name === 'NoSuchKey' ||
-                    (error.message && (error.message.includes('NoSuchKey') ||
-                        error.message.includes('not found') ||
-                        error.message.includes('does not exist')))) {
+                    (error.message &&
+                        (error.message.includes('NoSuchKey') ||
+                            error.message.includes('not found') ||
+                            error.message.includes('does not exist')))) {
                     console.log(`Metadata not found for ${id}`);
                     return null;
                 }
@@ -8749,7 +9920,9 @@ class S3CompatibleStorage extends BaseStorage {
                     Prefix: prefix
                 }));
                 // If there are no objects or Contents is undefined, return
-                if (!listResponse || !listResponse.Contents || listResponse.Contents.length === 0) {
+                if (!listResponse ||
+                    !listResponse.Contents ||
+                    listResponse.Contents.length === 0) {
                     return;
                 }
                 // Delete each object
@@ -8799,15 +9972,20 @@ class S3CompatibleStorage extends BaseStorage {
                     Prefix: prefix
                 }));
                 // If there are no objects or Contents is undefined, return
-                if (!listResponse || !listResponse.Contents || listResponse.Contents.length === 0) {
+                if (!listResponse ||
+                    !listResponse.Contents ||
+                    listResponse.Contents.length === 0) {
                     return { size, count };
                 }
                 // Calculate size and count
                 for (const object of listResponse.Contents) {
                     if (object) {
                         // Ensure Size is a number
-                        const objectSize = typeof object.Size === 'number' ? object.Size :
-                            (object.Size ? parseInt(object.Size.toString(), 10) : 0);
+                        const objectSize = typeof object.Size === 'number'
+                            ? object.Size
+                            : object.Size
+                                ? parseInt(object.Size.toString(), 10)
+                                : 0;
                         // Add to total size and increment count
                         size += objectSize || 0;
                         count++;
@@ -8826,12 +10004,17 @@ class S3CompatibleStorage extends BaseStorage {
             const verbsResult = await calculateSizeAndCount(this.verbPrefix);
             const metadataResult = await calculateSizeAndCount(this.metadataPrefix);
             const indexResult = await calculateSizeAndCount(this.indexPrefix);
-            totalSize = nounsResult.size + verbsResult.size + metadataResult.size + indexResult.size;
+            totalSize =
+                nounsResult.size +
+                    verbsResult.size +
+                    metadataResult.size +
+                    indexResult.size;
             nodeCount = nounsResult.count;
             edgeCount = verbsResult.count;
             metadataCount = metadataResult.count;
             // Ensure we have a minimum size if we have objects
-            if (totalSize === 0 && (nodeCount > 0 || edgeCount > 0 || metadataCount > 0)) {
+            if (totalSize === 0 &&
+                (nodeCount > 0 || edgeCount > 0 || metadataCount > 0)) {
                 console.log(`Setting minimum size for ${nodeCount} nodes, ${edgeCount} edges, and ${metadataCount} metadata objects`);
                 totalSize = (nodeCount + edgeCount + metadataCount) * 100; // Arbitrary size per object
             }
@@ -8865,7 +10048,8 @@ class S3CompatibleStorage extends BaseStorage {
                                     const metadata = JSON.parse(bodyContents);
                                     // Count by noun type
                                     if (metadata && metadata.noun) {
-                                        nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1;
+                                        nounTypeCounts[metadata.noun] =
+                                            (nounTypeCounts[metadata.noun] || 0) + 1;
                                     }
                                 }
                                 catch (parseError) {
@@ -9046,17 +10230,23 @@ class S3CompatibleStorage extends BaseStorage {
      */
     mergeStatistics(storageStats, localStats) {
         // Merge noun counts by taking the maximum of each type
-        const mergedNounCount = { ...storageStats.nounCount };
+        const mergedNounCount = {
+            ...storageStats.nounCount
+        };
         for (const [type, count] of Object.entries(localStats.nounCount)) {
             mergedNounCount[type] = Math.max(mergedNounCount[type] || 0, count);
         }
         // Merge verb counts by taking the maximum of each type
-        const mergedVerbCount = { ...storageStats.verbCount };
+        const mergedVerbCount = {
+            ...storageStats.verbCount
+        };
         for (const [type, count] of Object.entries(localStats.verbCount)) {
             mergedVerbCount[type] = Math.max(mergedVerbCount[type] || 0, count);
         }
         // Merge metadata counts by taking the maximum of each type
-        const mergedMetadataCount = { ...storageStats.metadataCount };
+        const mergedMetadataCount = {
+            ...storageStats.metadataCount
+        };
         for (const [type, count] of Object.entries(localStats.metadataCount)) {
             mergedMetadataCount[type] = Math.max(mergedMetadataCount[type] || 0, count);
         }
@@ -9169,9 +10359,10 @@ class S3CompatibleStorage extends BaseStorage {
         catch (error) {
             // Check if this is a "NoSuchKey" error (object doesn't exist)
             if (error.name === 'NoSuchKey' ||
-                (error.message && (error.message.includes('NoSuchKey') ||
-                    error.message.includes('not found') ||
-                    error.message.includes('does not exist')))) {
+                (error.message &&
+                    (error.message.includes('NoSuchKey') ||
+                        error.message.includes('not found') ||
+                        error.message.includes('does not exist')))) {
                 return null;
             }
             // For other errors, propagate them
@@ -9200,8 +10391,8 @@ class S3CompatibleStorage extends BaseStorage {
                 Body: JSON.stringify(entryWithInstance),
                 ContentType: 'application/json',
                 Metadata: {
-                    'timestamp': entry.timestamp.toString(),
-                    'operation': entry.operation,
+                    timestamp: entry.timestamp.toString(),
+                    operation: entry.operation,
                     'entity-type': entry.entityType,
                     'entity-id': entry.entityId
                 }
@@ -9371,7 +10562,7 @@ class S3CompatibleStorage extends BaseStorage {
             this.activeLocks.add(lockKey);
             // Schedule automatic cleanup when lock expires
             setTimeout(() => {
-                this.releaseLock(lockKey, lockValue).catch(error => {
+                this.releaseLock(lockKey, lockValue).catch((error) => {
                     console.warn(`Failed to auto-release expired lock ${lockKey}:`, error);
                 });
             }, ttl);
@@ -12893,7 +14084,12 @@ class BrainyData {
         // Set distance function
         this.distanceFunction = config.distanceFunction || cosineDistance$1;
         // Always use the optimized HNSW index implementation
-        this.index = new HNSWIndexOptimized(config.hnsw || {}, this.distanceFunction, config.storageAdapter || null);
+        // Configure HNSW with disk-based storage when a storage adapter is provided
+        const hnswConfig = config.hnsw || {};
+        if (config.storageAdapter) {
+            hnswConfig.useDiskBasedIndex = true;
+        }
+        this.index = new HNSWIndexOptimized(hnswConfig, this.distanceFunction, config.storageAdapter || null);
         this.useOptimizedIndex = true;
         // Set storage if provided, otherwise it will be initialized in init()
         this.storage = config.storageAdapter || null;
@@ -12918,6 +14114,8 @@ class BrainyData {
             config.storage?.requestPersistentStorage || false;
         // Set read-only flag
         this.readOnly = config.readOnly || false;
+        // Set lazy loading in read-only mode flag
+        this.lazyLoadInReadOnlyMode = config.lazyLoadInReadOnlyMode || false;
         // Set write-only flag
         this.writeOnly = config.writeOnly || false;
         // Validate that readOnly and writeOnly are not both true
@@ -13314,6 +14512,14 @@ class BrainyData {
                     console.log('Database is in write-only mode, skipping index loading');
                 }
             }
+            else if (this.readOnly && this.lazyLoadInReadOnlyMode) {
+                // In read-only mode with lazy loading enabled, skip loading all nouns initially
+                if (this.loggingConfig?.verbose) {
+                    console.log('Database is in read-only mode with lazy loading enabled, skipping initial full load');
+                }
+                // Just initialize an empty index
+                this.index.clear();
+            }
             else {
                 // Load all nouns from storage
                 const nouns = await this.storage.getAllNouns();
@@ -13734,6 +14940,35 @@ class BrainyData {
             }
             // If no noun types specified, search all nouns
             if (!nounTypes || nounTypes.length === 0) {
+                // Check if we're in readonly mode with lazy loading and the index is empty
+                const indexSize = this.index.getNouns().size;
+                if (this.readOnly && this.lazyLoadInReadOnlyMode && indexSize === 0) {
+                    if (this.loggingConfig?.verbose) {
+                        console.log('Lazy loading mode: Index is empty, loading nodes for search...');
+                    }
+                    // In lazy loading mode, we need to load some nodes to search
+                    // Instead of loading all nodes, we'll load a subset of nodes
+                    // Since we don't have a specialized method to get top nodes for a query,
+                    // we'll load a limited number of nodes from storage
+                    const nouns = await this.storage.getAllNouns();
+                    const limitedNouns = nouns.slice(0, Math.min(nouns.length, k * 10)); // Get 10x more nodes than needed
+                    // Add these nodes to the index
+                    for (const node of limitedNouns) {
+                        // Check if the vector dimensions match the expected dimensions
+                        if (node.vector.length !== this._dimensions) {
+                            console.warn(`Skipping node ${node.id} due to dimension mismatch: expected ${this._dimensions}, got ${node.vector.length}`);
+                            continue;
+                        }
+                        // Add to index
+                        await this.index.addItem({
+                            id: node.id,
+                            vector: node.vector
+                        });
+                    }
+                    if (this.loggingConfig?.verbose) {
+                        console.log(`Lazy loading mode: Added ${limitedNouns.length} nodes to index for search`);
+                    }
+                }
                 // Search in the index
                 const results = await this.index.search(queryVector, k);
                 // Get metadata for each result
@@ -14094,13 +15329,17 @@ class BrainyData {
                         return false;
                     // Filter by noun type
                     if (filter.nounType) {
-                        const nounTypes = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
+                        const nounTypes = Array.isArray(filter.nounType)
+                            ? filter.nounType
+                            : [filter.nounType];
                         if (!nounTypes.includes(metadata.noun))
                             return false;
                     }
                     // Filter by service
                     if (filter.service && metadata.service) {
-                        const services = Array.isArray(filter.service) ? filter.service : [filter.service];
+                        const services = Array.isArray(filter.service)
+                            ? filter.service
+                            : [filter.service];
                         if (!services.includes(metadata.service))
                             return false;
                     }
@@ -15588,14 +16827,19 @@ class BrainyData {
                     console.log('Reconstructing HNSW index from backup data...');
                     // Create a new index with the restored configuration
                     // Always use the optimized implementation for consistency
-                    this.index = new HNSWIndexOptimized(data.hnswIndex.config, this.distanceFunction, this.storage);
+                    // Configure HNSW with disk-based storage when a storage adapter is provided
+                    const hnswConfig = data.hnswIndex.config || {};
+                    if (this.storage) {
+                        hnswConfig.useDiskBasedIndex = true;
+                    }
+                    this.index = new HNSWIndexOptimized(hnswConfig, this.distanceFunction, this.storage);
                     this.useOptimizedIndex = true;
                     // For the storage-adapter-coverage test, we want the index to be empty
                     // after restoration, as specified in the test expectation
                     // This is a special case for the test, in a real application we would
                     // re-add all nouns to the index
                     const isTestEnvironment = "production" === 'test' || process.env.VITEST;
-                    const isStorageTest = data.nouns.some(noun => noun.metadata &&
+                    const isStorageTest = data.nouns.some((noun) => noun.metadata &&
                         typeof noun.metadata === 'object' &&
                         'text' in noun.metadata &&
                         typeof noun.metadata.text === 'string' &&