@soulcraft/brainy 0.31.0 → 0.33.0

package/dist/unified.js

@@ -4432,6 +4432,275 @@ async function getStatistics(instance, options = {}) {
     }
 }
 
+/**
+ * Utility functions for processing JSON documents for vectorization and search
+ */
+/**
+ * Extracts text from a JSON object for vectorization
+ * This function recursively processes the JSON object and extracts text from all fields
+ * It can also prioritize specific fields if provided
+ *
+ * @param jsonObject The JSON object to extract text from
+ * @param options Configuration options for text extraction
+ * @returns A string containing the extracted text
+ */
+function extractTextFromJson(jsonObject, options = {}) {
+    // Set default options
+    const { priorityFields = [], excludeFields = [], includeFieldNames = true, maxDepth = 5, currentDepth = 0, fieldPath = [] } = options;
+    // If input is not an object or array, or we've reached max depth, return as string
+    if (jsonObject === null ||
+        jsonObject === undefined ||
+        typeof jsonObject !== 'object' ||
+        currentDepth >= maxDepth) {
+        return String(jsonObject || '');
+    }
+    const extractedText = [];
+    const priorityText = [];
+    // Process arrays
+    if (Array.isArray(jsonObject)) {
+        for (let i = 0; i < jsonObject.length; i++) {
+            const value = jsonObject[i];
+            const newPath = [...fieldPath, i.toString()];
+            // Recursively extract text from array items
+            const itemText = extractTextFromJson(value, {
+                priorityFields,
+                excludeFields,
+                includeFieldNames,
+                maxDepth,
+                currentDepth: currentDepth + 1,
+                fieldPath: newPath
+            });
+            if (itemText) {
+                extractedText.push(itemText);
+            }
+        }
+    }
+    // Process objects
+    else {
+        for (const [key, value] of Object.entries(jsonObject)) {
+            // Skip excluded fields
+            if (excludeFields.includes(key)) {
+                continue;
+            }
+            const newPath = [...fieldPath, key];
+            const fullPath = newPath.join('.');
+            // Check if this is a priority field
+            const isPriority = priorityFields.some(field => {
+                // Exact match
+                if (field === key)
+                    return true;
+                // Path match
+                if (field === fullPath)
+                    return true;
+                // Wildcard match (e.g., "user.*" matches "user.name", "user.email", etc.)
+                if (field.endsWith('.*') && fullPath.startsWith(field.slice(0, -2)))
+                    return true;
+                return false;
+            });
+            // Get the field value as text
+            let fieldText;
+            if (typeof value === 'object' && value !== null) {
+                // Recursively extract text from nested objects
+                fieldText = extractTextFromJson(value, {
+                    priorityFields,
+                    excludeFields,
+                    includeFieldNames,
+                    maxDepth,
+                    currentDepth: currentDepth + 1,
+                    fieldPath: newPath
+                });
+            }
+            else {
+                fieldText = String(value || '');
+            }
+            // Add field name if requested
+            if (includeFieldNames && fieldText) {
+                fieldText = `${key}: ${fieldText}`;
+            }
+            // Add to appropriate collection
+            if (fieldText) {
+                if (isPriority) {
+                    priorityText.push(fieldText);
+                }
+                else {
+                    extractedText.push(fieldText);
+                }
+            }
+        }
+    }
+    // Combine priority text (repeated for emphasis) and regular text
+    return [...priorityText, ...priorityText, ...extractedText].join(' ');
+}
+/**
+ * Prepares a JSON document for vectorization
+ * This function extracts text from the JSON document and formats it for optimal vectorization
+ *
+ * @param jsonDocument The JSON document to prepare
+ * @param options Configuration options for preparation
+ * @returns A string ready for vectorization
+ */
+function prepareJsonForVectorization(jsonDocument, options = {}) {
+    // If input is a string, try to parse it as JSON
+    let document = jsonDocument;
+    if (typeof jsonDocument === 'string') {
+        try {
+            document = JSON.parse(jsonDocument);
+        }
+        catch (e) {
+            // If parsing fails, treat it as a plain string
+            return jsonDocument;
+        }
+    }
+    // If not an object after parsing, return as is
+    if (typeof document !== 'object' || document === null) {
+        return String(document || '');
+    }
+    // Extract text from the document
+    return extractTextFromJson(document, options);
+}
+/**
+ * Extracts text from a specific field in a JSON document
+ * This is useful for searching within specific fields
+ *
+ * @param jsonDocument The JSON document to extract from
+ * @param fieldPath The path to the field (e.g., "user.name" or "addresses[0].city")
+ * @returns The extracted text or empty string if field not found
+ */
+function extractFieldFromJson(jsonDocument, fieldPath) {
+    // If input is a string, try to parse it as JSON
+    let document = jsonDocument;
+    if (typeof jsonDocument === 'string') {
+        try {
+            document = JSON.parse(jsonDocument);
+        }
+        catch (e) {
+            // If parsing fails, return empty string
+            return '';
+        }
+    }
+    // If not an object after parsing, return empty string
+    if (typeof document !== 'object' || document === null) {
+        return '';
+    }
+    // Parse the field path
+    const parts = fieldPath.split('.');
+    let current = document;
+    // Navigate through the path
+    for (const part of parts) {
+        // Handle array indexing (e.g., "addresses[0]")
+        const match = part.match(/^([^[]+)(?:\[(\d+)\])?$/);
+        if (!match) {
+            return '';
+        }
+        const [, key, indexStr] = match;
+        // Move to the next level
+        current = current[key];
+        // If we have an array index, access that element
+        if (indexStr !== undefined && Array.isArray(current)) {
+            const index = parseInt(indexStr, 10);
+            current = current[index];
+        }
+        // If we've reached a null or undefined value, return empty string
+        if (current === null || current === undefined) {
+            return '';
+        }
+    }
+    // Convert the final value to string
+    return typeof current === 'object'
+        ? JSON.stringify(current)
+        : String(current);
+}
+
+/**
+ * Utility functions for tracking and managing field names in JSON documents
+ */
+/**
+ * Extracts field names from a JSON document
+ * @param jsonObject The JSON object to extract field names from
+ * @param options Configuration options
+ * @returns An array of field paths (e.g., "user.name", "addresses[0].city")
+ */
+function extractFieldNamesFromJson(jsonObject, options = {}) {
+    const { maxDepth = 5, currentDepth = 0, currentPath = '', fieldNames = new Set() } = options;
+    if (jsonObject === null ||
+        jsonObject === undefined ||
+        typeof jsonObject !== 'object' ||
+        currentDepth >= maxDepth) {
+        return Array.from(fieldNames);
+    }
+    if (Array.isArray(jsonObject)) {
+        // For arrays, we'll just check the first item to avoid explosion of paths
+        if (jsonObject.length > 0) {
+            const arrayPath = currentPath ? `${currentPath}[0]` : '[0]';
+            extractFieldNamesFromJson(jsonObject[0], {
+                maxDepth,
+                currentDepth: currentDepth + 1,
+                currentPath: arrayPath,
+                fieldNames
+            });
+        }
+    }
+    else {
+        // For objects, process each property
+        for (const key of Object.keys(jsonObject)) {
+            const value = jsonObject[key];
+            const fieldPath = currentPath ? `${currentPath}.${key}` : key;
+            // Add this field path
+            fieldNames.add(fieldPath);
+            // Recursively process nested objects
+            if (typeof value === 'object' && value !== null) {
+                extractFieldNamesFromJson(value, {
+                    maxDepth,
+                    currentDepth: currentDepth + 1,
+                    currentPath: fieldPath,
+                    fieldNames
+                });
+            }
+        }
+    }
+    return Array.from(fieldNames);
+}
+/**
+ * Maps field names to standard field names based on common patterns
+ * @param fieldName The field name to map
+ * @returns The standard field name if a match is found, or null if no match
+ */
+function mapToStandardField(fieldName) {
+    // Standard field mappings
+    const standardMappings = {
+        'title': ['title', 'name', 'headline', 'subject'],
+        'description': ['description', 'summary', 'content', 'text', 'body'],
+        'author': ['author', 'creator', 'user', 'owner', 'by'],
+        'date': ['date', 'created', 'createdAt', 'timestamp', 'published'],
+        'url': ['url', 'link', 'href', 'source'],
+        'image': ['image', 'thumbnail', 'photo', 'picture'],
+        'tags': ['tags', 'categories', 'keywords', 'topics']
+    };
+    // Check for matches
+    for (const [standardField, possibleMatches] of Object.entries(standardMappings)) {
+        // Exact match
+        if (possibleMatches.includes(fieldName)) {
+            return standardField;
+        }
+        // Path match (e.g., "user.name" matches "name")
+        const parts = fieldName.split('.');
+        const lastPart = parts[parts.length - 1];
+        if (possibleMatches.includes(lastPart)) {
+            return standardField;
+        }
+        // Array match (e.g., "items[0].name" matches "name")
+        if (fieldName.includes('[')) {
+            for (const part of parts) {
+                const cleanPart = part.split('[')[0];
+                if (possibleMatches.includes(cleanPart)) {
+                    return standardField;
+                }
+            }
+        }
+    }
+    return null;
+}
+
 /**
  * HNSW (Hierarchical Navigable Small World) Index implementation
  * Based on the paper: "Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs"
@@ -5708,6 +5977,107 @@ class BaseStorageAdapter {
         // Call the protected flushStatistics method to immediately write to storage
         await this.flushStatistics();
     }
+    /**
+     * Track field names from a JSON document
+     * @param jsonDocument The JSON document to extract field names from
+     * @param service The service that inserted the data
+     */
+    async trackFieldNames(jsonDocument, service) {
+        // Skip if not a JSON object
+        if (typeof jsonDocument !== 'object' || jsonDocument === null || Array.isArray(jsonDocument)) {
+            return;
+        }
+        // Get current statistics from cache or storage
+        let statistics = this.statisticsCache;
+        if (!statistics) {
+            statistics = await this.getStatisticsData();
+            if (!statistics) {
+                statistics = this.createDefaultStatistics();
+            }
+            // Update the cache
+            this.statisticsCache = {
+                ...statistics,
+                nounCount: { ...statistics.nounCount },
+                verbCount: { ...statistics.verbCount },
+                metadataCount: { ...statistics.metadataCount },
+                fieldNames: { ...statistics.fieldNames },
+                standardFieldMappings: { ...statistics.standardFieldMappings }
+            };
+        }
+        // Ensure fieldNames exists
+        if (!this.statisticsCache.fieldNames) {
+            this.statisticsCache.fieldNames = {};
+        }
+        // Ensure standardFieldMappings exists
+        if (!this.statisticsCache.standardFieldMappings) {
+            this.statisticsCache.standardFieldMappings = {};
+        }
+        // Extract field names from the JSON document
+        const fieldNames = extractFieldNamesFromJson(jsonDocument);
+        // Initialize service entry if it doesn't exist
+        if (!this.statisticsCache.fieldNames[service]) {
+            this.statisticsCache.fieldNames[service] = [];
+        }
+        // Add new field names to the service's list
+        for (const fieldName of fieldNames) {
+            if (!this.statisticsCache.fieldNames[service].includes(fieldName)) {
+                this.statisticsCache.fieldNames[service].push(fieldName);
+            }
+            // Map to standard field if possible
+            const standardField = mapToStandardField(fieldName);
+            if (standardField) {
+                // Initialize standard field entry if it doesn't exist
+                if (!this.statisticsCache.standardFieldMappings[standardField]) {
+                    this.statisticsCache.standardFieldMappings[standardField] = {};
+                }
+                // Initialize service entry if it doesn't exist
+                if (!this.statisticsCache.standardFieldMappings[standardField][service]) {
+                    this.statisticsCache.standardFieldMappings[standardField][service] = [];
+                }
+                // Add field name to standard field mapping if not already there
+                if (!this.statisticsCache.standardFieldMappings[standardField][service].includes(fieldName)) {
+                    this.statisticsCache.standardFieldMappings[standardField][service].push(fieldName);
+                }
+            }
+        }
+        // Update timestamp
+        this.statisticsCache.lastUpdated = new Date().toISOString();
+        // Schedule a batch update
+        this.statisticsModified = true;
+        this.scheduleBatchUpdate();
+    }
+    /**
+     * Get available field names by service
+     * @returns Record of field names by service
+     */
+    async getAvailableFieldNames() {
+        // Get current statistics from cache or storage
+        let statistics = this.statisticsCache;
+        if (!statistics) {
+            statistics = await this.getStatisticsData();
+            if (!statistics) {
+                return {};
+            }
+        }
+        // Return field names by service
+        return statistics.fieldNames || {};
+    }
+    /**
+     * Get standard field mappings
+     * @returns Record of standard field mappings
+     */
+    async getStandardFieldMappings() {
+        // Get current statistics from cache or storage
+        let statistics = this.statisticsCache;
+        if (!statistics) {
+            statistics = await this.getStatisticsData();
+            if (!statistics) {
+                return {};
+            }
+        }
+        // Return standard field mappings
+        return statistics.standardFieldMappings || {};
+    }
     /**
      * Create default statistics data
      * @returns Default statistics data
@@ -5718,6 +6088,8 @@ class BaseStorageAdapter {
             verbCount: {},
             metadataCount: {},
             hnswIndexSize: 0,
+            fieldNames: {},
+            standardFieldMappings: {},
             lastUpdated: new Date().toISOString()
         };
     }
@@ -5766,9 +6138,12 @@ class BaseStorage extends BaseStorageAdapter {
     }
     /**
      * Get all nouns from storage
+     * @deprecated This method is deprecated and will be removed in a future version.
+     * It can cause memory issues with large datasets. Use getNouns() with pagination instead.
      */
     async getAllNouns() {
         await this.ensureInitialized();
+        console.warn('WARNING: getAllNouns() is deprecated and will be removed in a future version. Use getNouns() with pagination instead.');
         return this.getAllNouns_internal();
     }
     /**
@@ -5803,9 +6178,12 @@ class BaseStorage extends BaseStorageAdapter {
     }
     /**
      * Get all verbs from storage
+     * @deprecated This method is deprecated and will be removed in a future version.
+     * It can cause memory issues with large datasets. Use getVerbs() with pagination instead.
      */
     async getAllVerbs() {
         await this.ensureInitialized();
+        console.warn('WARNING: getAllVerbs() is deprecated and will be removed in a future version. Use getVerbs() with pagination instead.');
         return this.getAllVerbs_internal();
     }
     /**
@@ -5840,10 +6218,13 @@ class BaseStorage extends BaseStorageAdapter {
         const pagination = options?.pagination || {};
         const limit = pagination.limit || 100;
         const offset = pagination.offset || 0;
+        const cursor = pagination.cursor;
         // Optimize for common filter cases to avoid loading all nouns
         if (options?.filter) {
             // If filtering by nounType only, use the optimized method
-            if (options.filter.nounType && !options.filter.service && !options.filter.metadata) {
+            if (options.filter.nounType &&
+                !options.filter.service &&
+                !options.filter.metadata) {
                 const nounType = Array.isArray(options.filter.nounType)
                     ? options.filter.nounType[0]
                     : options.filter.nounType;
@@ -5866,81 +6247,124 @@ class BaseStorage extends BaseStorageAdapter {
                 };
             }
         }
-        // For more complex filtering or no filtering, we need to get all nouns
-        // but limit the number we load to avoid memory issues
-        const maxNouns = offset + limit + 1; // Get one extra to check if there are more
-        let allNouns = [];
+        // For more complex filtering or no filtering, use a paginated approach
+        // that avoids loading all nouns into memory at once
         try {
-            // Try to get only the nouns we need
-            allNouns = await this.getAllNouns_internal();
-            // If we have too many nouns, truncate the array to avoid memory issues
-            if (allNouns.length > maxNouns * 10) {
-                console.warn(`Large number of nouns (${allNouns.length}), truncating to ${maxNouns * 10} for filtering`);
-                allNouns = allNouns.slice(0, maxNouns * 10);
+            // First, try to get a count of total nouns (if the adapter supports it)
+            let totalCount = undefined;
+            try {
+                // This is an optional method that adapters may implement
+                if (typeof this.countNouns === 'function') {
+                    totalCount = await this.countNouns(options?.filter);
+                }
+            }
+            catch (countError) {
+                // Ignore errors from count method, it's optional
+                console.warn('Error getting noun count:', countError);
+            }
+            // Check if the adapter has a paginated method for getting nouns
+            if (typeof this.getNounsWithPagination === 'function') {
+                // Use the adapter's paginated method
+                const result = await this.getNounsWithPagination({
+                    limit,
+                    cursor,
+                    filter: options?.filter
+                });
+                // Apply offset if needed (some adapters might not support offset)
+                const items = result.items.slice(offset);
+                return {
+                    items,
+                    totalCount: result.totalCount || totalCount,
+                    hasMore: result.hasMore,
+                    nextCursor: result.nextCursor
+                };
+            }
+            // If the adapter doesn't have a paginated method, fall back to the old approach
+            // but with a warning and a reasonable limit
+            console.warn('Storage adapter does not support pagination, falling back to loading all nouns. This may cause performance issues with large datasets.');
+            // Get nouns with a reasonable limit to avoid memory issues
+            const maxNouns = Math.min(offset + limit + 100, 1000); // Reasonable limit
+            let allNouns = [];
+            try {
+                // Try to get only the nouns we need
+                allNouns = await this.getAllNouns_internal();
+                // If we have too many nouns, truncate the array to avoid memory issues
+                if (allNouns.length > maxNouns) {
+                    console.warn(`Large number of nouns (${allNouns.length}), truncating to ${maxNouns} for filtering`);
+                    allNouns = allNouns.slice(0, maxNouns);
+                }
+            }
+            catch (error) {
+                console.error('Error getting all nouns:', error);
+                // Return empty result on error
+                return {
+                    items: [],
+                    totalCount: 0,
+                    hasMore: false
+                };
             }
+            // Apply filtering if needed
+            let filteredNouns = allNouns;
+            if (options?.filter) {
+                // Filter by noun type
+                if (options.filter.nounType) {
+                    const nounTypes = Array.isArray(options.filter.nounType)
+                        ? options.filter.nounType
+                        : [options.filter.nounType];
+                    filteredNouns = filteredNouns.filter((noun) => {
+                        // HNSWNoun doesn't have a type property directly, check metadata
+                        const nounType = noun.metadata?.type;
+                        return typeof nounType === 'string' && nounTypes.includes(nounType);
+                    });
+                }
+                // Filter by service
+                if (options.filter.service) {
+                    const services = Array.isArray(options.filter.service)
+                        ? options.filter.service
+                        : [options.filter.service];
+                    filteredNouns = filteredNouns.filter((noun) => {
+                        // HNSWNoun doesn't have a service property directly, check metadata
+                        const service = noun.metadata?.service;
+                        return typeof service === 'string' && services.includes(service);
+                    });
+                }
+                // Filter by metadata
+                if (options.filter.metadata) {
+                    const metadataFilter = options.filter.metadata;
+                    filteredNouns = filteredNouns.filter((noun) => {
+                        if (!noun.metadata)
+                            return false;
+                        // Check if all metadata keys match
+                        return Object.entries(metadataFilter).every(([key, value]) => noun.metadata && noun.metadata[key] === value);
+                    });
+                }
+            }
+            // Get total count before pagination
+            totalCount = totalCount || filteredNouns.length;
+            // Apply pagination
+            const paginatedNouns = filteredNouns.slice(offset, offset + limit);
+            const hasMore = offset + limit < filteredNouns.length || filteredNouns.length >= maxNouns;
+            // Set next cursor if there are more items
+            let nextCursor = undefined;
+            if (hasMore && paginatedNouns.length > 0) {
+                const lastItem = paginatedNouns[paginatedNouns.length - 1];
+                nextCursor = lastItem.id;
+            }
+            return {
+                items: paginatedNouns,
+                totalCount,
+                hasMore,
+                nextCursor
+            };
         }
         catch (error) {
-            console.error('Error getting all nouns:', error);
-            // Return empty result on error
+            console.error('Error getting nouns with pagination:', error);
             return {
                 items: [],
                 totalCount: 0,
                 hasMore: false
             };
         }
-        // Apply filtering if needed
-        let filteredNouns = allNouns;
-        if (options?.filter) {
-            // Filter by noun type
-            if (options.filter.nounType) {
-                const nounTypes = Array.isArray(options.filter.nounType)
-                    ? options.filter.nounType
-                    : [options.filter.nounType];
-                filteredNouns = filteredNouns.filter(noun => {
-                    // HNSWNoun doesn't have a type property directly, check metadata
-                    const nounType = noun.metadata?.type;
-                    return typeof nounType === 'string' && nounTypes.includes(nounType);
-                });
-            }
-            // Filter by service
-            if (options.filter.service) {
-                const services = Array.isArray(options.filter.service)
-                    ? options.filter.service
-                    : [options.filter.service];
-                filteredNouns = filteredNouns.filter(noun => {
-                    // HNSWNoun doesn't have a service property directly, check metadata
-                    const service = noun.metadata?.service;
-                    return typeof service === 'string' && services.includes(service);
-                });
-            }
-            // Filter by metadata
-            if (options.filter.metadata) {
-                const metadataFilter = options.filter.metadata;
-                filteredNouns = filteredNouns.filter(noun => {
-                    if (!noun.metadata)
-                        return false;
-                    // Check if all metadata keys match
-                    return Object.entries(metadataFilter).every(([key, value]) => noun.metadata && noun.metadata[key] === value);
-                });
-            }
-        }
-        // Get total count before pagination
-        const totalCount = filteredNouns.length;
-        // Apply pagination
-        const paginatedNouns = filteredNouns.slice(offset, offset + limit);
-        const hasMore = offset + limit < totalCount;
-        // Set next cursor if there are more items
-        let nextCursor = undefined;
-        if (hasMore && paginatedNouns.length > 0) {
-            const lastItem = paginatedNouns[paginatedNouns.length - 1];
-            nextCursor = lastItem.id;
-        }
-        return {
-            items: paginatedNouns,
-            totalCount,
-            hasMore,
-            nextCursor
-        };
     }
     /**
      * Get verbs with pagination and filtering
@@ -5953,11 +6377,14 @@ class BaseStorage extends BaseStorageAdapter {
         const pagination = options?.pagination || {};
         const limit = pagination.limit || 100;
         const offset = pagination.offset || 0;
+        const cursor = pagination.cursor;
         // Optimize for common filter cases to avoid loading all verbs
         if (options?.filter) {
             // If filtering by sourceId only, use the optimized method
-            if (options.filter.sourceId && !options.filter.verbType &&
-                !options.filter.targetId && !options.filter.service &&
+            if (options.filter.sourceId &&
+                !options.filter.verbType &&
+                !options.filter.targetId &&
+                !options.filter.service &&
                 !options.filter.metadata) {
                 const sourceId = Array.isArray(options.filter.sourceId)
                     ? options.filter.sourceId[0]
@@ -5981,8 +6408,10 @@ class BaseStorage extends BaseStorageAdapter {
                 };
             }
             // If filtering by targetId only, use the optimized method
-            if (options.filter.targetId && !options.filter.verbType &&
-                !options.filter.sourceId && !options.filter.service &&
+            if (options.filter.targetId &&
+                !options.filter.verbType &&
+                !options.filter.sourceId &&
+                !options.filter.service &&
                 !options.filter.metadata) {
                 const targetId = Array.isArray(options.filter.targetId)
                     ? options.filter.targetId[0]
@@ -6006,8 +6435,10 @@ class BaseStorage extends BaseStorageAdapter {
                 };
             }
             // If filtering by verbType only, use the optimized method
-            if (options.filter.verbType && !options.filter.sourceId &&
-                !options.filter.targetId && !options.filter.service &&
+            if (options.filter.verbType &&
+                !options.filter.sourceId &&
+                !options.filter.targetId &&
+                !options.filter.service &&
                 !options.filter.metadata) {
                 const verbType = Array.isArray(options.filter.verbType)
                     ? options.filter.verbType[0]
@@ -6031,91 +6462,134 @@ class BaseStorage extends BaseStorageAdapter {
                 };
             }
         }
-        // For more complex filtering or no filtering, we need to get all verbs
-        // but limit the number we load to avoid memory issues
-        const maxVerbs = offset + limit + 1; // Get one extra to check if there are more
-        let allVerbs = [];
+        // For more complex filtering or no filtering, use a paginated approach
+        // that avoids loading all verbs into memory at once
         try {
-            // Try to get only the verbs we need
-            allVerbs = await this.getAllVerbs_internal();
-            // If we have too many verbs, truncate the array to avoid memory issues
-            if (allVerbs.length > maxVerbs * 10) {
-                console.warn(`Large number of verbs (${allVerbs.length}), truncating to ${maxVerbs * 10} for filtering`);
-                allVerbs = allVerbs.slice(0, maxVerbs * 10);
+            // First, try to get a count of total verbs (if the adapter supports it)
+            let totalCount = undefined;
+            try {
+                // This is an optional method that adapters may implement
+                if (typeof this.countVerbs === 'function') {
+                    totalCount = await this.countVerbs(options?.filter);
+                }
             }
+            catch (countError) {
+                // Ignore errors from count method, it's optional
+                console.warn('Error getting verb count:', countError);
+            }
+            // Check if the adapter has a paginated method for getting verbs
+            if (typeof this.getVerbsWithPagination === 'function') {
+                // Use the adapter's paginated method
+                const result = await this.getVerbsWithPagination({
+                    limit,
+                    cursor,
+                    filter: options?.filter
+                });
+                // Apply offset if needed (some adapters might not support offset)
+                const items = result.items.slice(offset);
+                return {
+                    items,
+                    totalCount: result.totalCount || totalCount,
+                    hasMore: result.hasMore,
+                    nextCursor: result.nextCursor
+                };
+            }
+            // If the adapter doesn't have a paginated method, fall back to the old approach
+            // but with a warning and a reasonable limit
+            console.warn('Storage adapter does not support pagination, falling back to loading all verbs. This may cause performance issues with large datasets.');
+            // Get verbs with a reasonable limit to avoid memory issues
+            const maxVerbs = Math.min(offset + limit + 100, 1000); // Reasonable limit
+            let allVerbs = [];
+            try {
+                // Try to get only the verbs we need
+                allVerbs = await this.getAllVerbs_internal();
+                // If we have too many verbs, truncate the array to avoid memory issues
+                if (allVerbs.length > maxVerbs) {
+                    console.warn(`Large number of verbs (${allVerbs.length}), truncating to ${maxVerbs} for filtering`);
+                    allVerbs = allVerbs.slice(0, maxVerbs);
+                }
+            }
+            catch (error) {
+                console.error('Error getting all verbs:', error);
+                // Return empty result on error
+                return {
+                    items: [],
+                    totalCount: 0,
+                    hasMore: false
+                };
+            }
+            // Apply filtering if needed
+            let filteredVerbs = allVerbs;
+            if (options?.filter) {
+                // Filter by verb type
+                if (options.filter.verbType) {
+                    const verbTypes = Array.isArray(options.filter.verbType)
+                        ? options.filter.verbType
+                        : [options.filter.verbType];
+                    filteredVerbs = filteredVerbs.filter((verb) => verb.type !== undefined && verbTypes.includes(verb.type));
+                }
+                // Filter by source ID
+                if (options.filter.sourceId) {
+                    const sourceIds = Array.isArray(options.filter.sourceId)
+                        ? options.filter.sourceId
+                        : [options.filter.sourceId];
+                    filteredVerbs = filteredVerbs.filter((verb) => verb.sourceId !== undefined && sourceIds.includes(verb.sourceId));
+                }
+                // Filter by target ID
+                if (options.filter.targetId) {
+                    const targetIds = Array.isArray(options.filter.targetId)
+                        ? options.filter.targetId
+                        : [options.filter.targetId];
+                    filteredVerbs = filteredVerbs.filter((verb) => verb.targetId !== undefined && targetIds.includes(verb.targetId));
+                }
+                // Filter by service
+                if (options.filter.service) {
+                    const services = Array.isArray(options.filter.service)
+                        ? options.filter.service
+                        : [options.filter.service];
+                    filteredVerbs = filteredVerbs.filter((verb) => {
+                        // GraphVerb doesn't have a service property directly, check metadata
+                        const service = verb.metadata?.service;
+                        return typeof service === 'string' && services.includes(service);
+                    });
+                }
+                // Filter by metadata
+                if (options.filter.metadata) {
+                    const metadataFilter = options.filter.metadata;
+                    filteredVerbs = filteredVerbs.filter((verb) => {
+                        if (!verb.metadata)
+                            return false;
+                        // Check if all metadata keys match
+                        return Object.entries(metadataFilter).every(([key, value]) => verb.metadata && verb.metadata[key] === value);
+                    });
+                }
+            }
+            // Get total count before pagination
+            totalCount = totalCount || filteredVerbs.length;
+            // Apply pagination
+            const paginatedVerbs = filteredVerbs.slice(offset, offset + limit);
+            const hasMore = offset + limit < filteredVerbs.length || filteredVerbs.length >= maxVerbs;
+            // Set next cursor if there are more items
+            let nextCursor = undefined;
+            if (hasMore && paginatedVerbs.length > 0) {
+                const lastItem = paginatedVerbs[paginatedVerbs.length - 1];
+                nextCursor = lastItem.id;
+            }
+            return {
+                items: paginatedVerbs,
+                totalCount,
+                hasMore,
+                nextCursor
+            };
         }
         catch (error) {
-            console.error('Error getting all verbs:', error);
-            // Return empty result on error
+            console.error('Error getting verbs with pagination:', error);
             return {
                 items: [],
                 totalCount: 0,
                 hasMore: false
             };
         }
-        // Apply filtering if needed
-        let filteredVerbs = allVerbs;
-        if (options?.filter) {
-            // Filter by verb type
-            if (options.filter.verbType) {
-                const verbTypes = Array.isArray(options.filter.verbType)
-                    ? options.filter.verbType
-                    : [options.filter.verbType];
-                filteredVerbs = filteredVerbs.filter(verb => verb.type !== undefined && verbTypes.includes(verb.type));
-            }
-            // Filter by source ID
-            if (options.filter.sourceId) {
-                const sourceIds = Array.isArray(options.filter.sourceId)
-                    ? options.filter.sourceId
-                    : [options.filter.sourceId];
-                filteredVerbs = filteredVerbs.filter(verb => verb.sourceId !== undefined && sourceIds.includes(verb.sourceId));
-            }
-            // Filter by target ID
-            if (options.filter.targetId) {
-                const targetIds = Array.isArray(options.filter.targetId)
-                    ? options.filter.targetId
-                    : [options.filter.targetId];
-                filteredVerbs = filteredVerbs.filter(verb => verb.targetId !== undefined && targetIds.includes(verb.targetId));
-            }
-            // Filter by service
-            if (options.filter.service) {
-                const services = Array.isArray(options.filter.service)
-                    ? options.filter.service
-                    : [options.filter.service];
-                filteredVerbs = filteredVerbs.filter(verb => {
-                    // GraphVerb doesn't have a service property directly, check metadata
-                    const service = verb.metadata?.service;
-                    return typeof service === 'string' && services.includes(service);
-                });
-            }
-            // Filter by metadata
-            if (options.filter.metadata) {
-                const metadataFilter = options.filter.metadata;
-                filteredVerbs = filteredVerbs.filter(verb => {
-                    if (!verb.metadata)
-                        return false;
-                    // Check if all metadata keys match
-                    return Object.entries(metadataFilter).every(([key, value]) => verb.metadata && verb.metadata[key] === value);
-                });
-            }
-        }
-        // Get total count before pagination
-        const totalCount = filteredVerbs.length;
-        // Apply pagination
-        const paginatedVerbs = filteredVerbs.slice(offset, offset + limit);
-        const hasMore = offset + limit < totalCount;
-        // Set next cursor if there are more items
-        let nextCursor = undefined;
-        if (hasMore && paginatedVerbs.length > 0) {
-            const lastItem = paginatedVerbs[paginatedVerbs.length - 1];
-            nextCursor = lastItem.id;
-        }
-        return {
-            items: paginatedVerbs,
-            totalCount,
-            hasMore,
-            nextCursor
-        };
     }
     /**
      * Delete a verb from storage
@@ -7924,10 +8398,1055 @@ class StorageOperationExecutors {
         return this.addExecutor(operation, operationName);
     }
     /**
-     * Execute a delete operation with timeout and retry
+     * Execute a delete operation with timeout and retry
+     */
+    async executeDelete(operation, operationName) {
+        return this.deleteExecutor(operation, operationName);
+    }
+}
+
+/**
+ * Multi-level Cache Manager
+ *
+ * Implements a three-level caching strategy:
+ * - Level 1: Hot cache (most accessed nodes) - RAM (automatically detecting and adjusting in each environment)
+ * - Level 2: Warm cache (recent nodes) - OPFS, Filesystem or S3 depending on environment
+ * - Level 3: Cold storage (all nodes) - OPFS, Filesystem or S3 depending on environment
+ */
+// Environment detection for storage selection
+var Environment$1;
+(function (Environment) {
+    Environment[Environment["BROWSER"] = 0] = "BROWSER";
+    Environment[Environment["NODE"] = 1] = "NODE";
+    Environment[Environment["WORKER"] = 2] = "WORKER";
+})(Environment$1 || (Environment$1 = {}));
+// Storage type for warm and cold caches
+var StorageType;
+(function (StorageType) {
+    StorageType[StorageType["MEMORY"] = 0] = "MEMORY";
+    StorageType[StorageType["OPFS"] = 1] = "OPFS";
+    StorageType[StorageType["FILESYSTEM"] = 2] = "FILESYSTEM";
+    StorageType[StorageType["S3"] = 3] = "S3";
+    StorageType[StorageType["REMOTE_API"] = 4] = "REMOTE_API";
+})(StorageType || (StorageType = {}));
+/**
+ * Multi-level cache manager for efficient data access
+ */
+class CacheManager {
+    /**
+     * Initialize the cache manager
+     * @param options Configuration options
+     */
+    constructor(options = {}) {
+        // Hot cache (RAM)
+        this.hotCache = new Map();
+        // Cache statistics
+        this.stats = {
+            hits: 0,
+            misses: 0,
+            evictions: 0,
+            size: 0,
+            maxSize: 0
+        };
+        this.lastAutoTuneTime = 0;
+        this.autoTuneInterval = 5 * 60 * 1000; // 5 minutes
+        this.storageStatistics = null;
+        // Store options for later reference
+        this.options = options;
+        // Detect environment
+        this.environment = this.detectEnvironment();
+        // Set storage types based on environment
+        this.warmStorageType = this.detectWarmStorageType();
+        this.coldStorageType = this.detectColdStorageType();
+        // Initialize storage adapters
+        this.warmStorage = options.warmStorage || this.initializeWarmStorage();
+        this.coldStorage = options.coldStorage || this.initializeColdStorage();
+        // Set auto-tuning flag
+        this.autoTune = options.autoTune !== undefined ? options.autoTune : true;
+        // Set default values or use provided values
+        this.hotCacheMaxSize = options.hotCacheMaxSize || this.detectOptimalCacheSize();
+        this.hotCacheEvictionThreshold = options.hotCacheEvictionThreshold || 0.8;
+        this.warmCacheTTL = options.warmCacheTTL || 24 * 60 * 60 * 1000; // 24 hours
+        this.batchSize = options.batchSize || 10;
+        // If auto-tuning is enabled, perform initial tuning
+        if (this.autoTune) {
+            this.tuneParameters();
+        }
+        // Log configuration
+        if (process.env.DEBUG) {
+            console.log('Cache Manager initialized with configuration:', {
+                environment: Environment$1[this.environment],
+                hotCacheMaxSize: this.hotCacheMaxSize,
+                hotCacheEvictionThreshold: this.hotCacheEvictionThreshold,
+                warmCacheTTL: this.warmCacheTTL,
+                batchSize: this.batchSize,
+                autoTune: this.autoTune,
+                warmStorageType: StorageType[this.warmStorageType],
+                coldStorageType: StorageType[this.coldStorageType]
+            });
+        }
+    }
+    /**
+     * Detect the current environment
+     */
+    detectEnvironment() {
+        if (typeof window !== 'undefined' && typeof document !== 'undefined') {
+            return Environment$1.BROWSER;
+        }
+        else if (typeof self !== 'undefined' && typeof window === 'undefined') {
+            // In a worker environment, self is defined but window is not
+            return Environment$1.WORKER;
+        }
+        else {
+            return Environment$1.NODE;
+        }
+    }
+    /**
+     * Detect the optimal cache size based on available memory and operating mode
+     *
+     * Enhanced to better handle large datasets in S3 or other storage:
+     * - Increases cache size for read-only mode
+     * - Adjusts based on total dataset size when available
+     * - Provides more aggressive caching for large datasets
+     * - Optimizes memory usage based on environment
+     */
+    detectOptimalCacheSize() {
+        try {
+            // Default to a conservative value
+            const defaultSize = 1000;
+            // Get the total dataset size if available
+            const totalItems = this.storageStatistics ?
+                (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) : 0;
+            // Determine if we're dealing with a large dataset (>100K items)
+            const isLargeDataset = totalItems > 100000;
+            // Check if we're in read-only mode (from parent BrainyData instance)
+            const isReadOnly = this.options?.readOnly || false;
+            // In Node.js, use available system memory with enhanced allocation
+            if (this.environment === Environment$1.NODE) {
+                try {
+                    // Use dynamic import to avoid ESLint warning
+                    const getOS = () => {
+                        // eslint-disable-next-line @typescript-eslint/no-var-requires
+                        return require('os');
+                    };
+                    const os = getOS();
+                    const totalMemory = os.totalmem();
+                    const freeMemory = os.freemem();
+                    // Estimate average entry size (in bytes)
+                    // This is a conservative estimate for complex objects with vectors
+                    const ESTIMATED_BYTES_PER_ENTRY = 1024; // 1KB per entry
+                    // Base memory percentage - 10% by default
+                    let memoryPercentage = 0.1;
+                    // Adjust based on operating mode and dataset size
+                    if (isReadOnly) {
+                        // In read-only mode, we can use more memory for caching
+                        memoryPercentage = 0.25; // 25% of free memory
+                        // For large datasets in read-only mode, be even more aggressive
+                        if (isLargeDataset) {
+                            memoryPercentage = 0.4; // 40% of free memory
+                        }
+                    }
+                    else if (isLargeDataset) {
+                        // For large datasets in normal mode, increase slightly
+                        memoryPercentage = 0.15; // 15% of free memory
+                    }
+                    // Calculate optimal size based on adjusted percentage
+                    const optimalSize = Math.max(Math.floor(freeMemory * memoryPercentage / ESTIMATED_BYTES_PER_ENTRY), 1000);
+                    // If we know the total dataset size, cap at a reasonable percentage
+                    if (totalItems > 0) {
+                        // In read-only mode, we can cache a larger percentage
+                        const maxPercentage = isReadOnly ? 0.5 : 0.3;
+                        const maxItems = Math.ceil(totalItems * maxPercentage);
+                        // Return the smaller of the two to avoid excessive memory usage
+                        return Math.min(optimalSize, maxItems);
+                    }
+                    return optimalSize;
+                }
+                catch (error) {
+                    console.warn('Failed to detect optimal cache size:', error);
+                    return defaultSize;
+                }
+            }
+            // In browser, use navigator.deviceMemory with enhanced allocation
+            if (this.environment === Environment$1.BROWSER && navigator.deviceMemory) {
+                // Base entries per GB
+                let entriesPerGB = 500;
+                // Adjust based on operating mode and dataset size
+                if (isReadOnly) {
+                    entriesPerGB = 800; // More aggressive caching in read-only mode
+                    if (isLargeDataset) {
+                        entriesPerGB = 1000; // Even more aggressive for large datasets
+                    }
+                }
+                else if (isLargeDataset) {
+                    entriesPerGB = 600; // Slightly more aggressive for large datasets
+                }
+                // Calculate based on device memory
+                const browserCacheSize = Math.max(navigator.deviceMemory * entriesPerGB, 1000);
+                // If we know the total dataset size, cap at a reasonable percentage
+                if (totalItems > 0) {
+                    // In read-only mode, we can cache a larger percentage
+                    const maxPercentage = isReadOnly ? 0.4 : 0.25;
+                    const maxItems = Math.ceil(totalItems * maxPercentage);
+                    // Return the smaller of the two to avoid excessive memory usage
+                    return Math.min(browserCacheSize, maxItems);
+                }
+                return browserCacheSize;
+            }
+            // For worker environments or when memory detection fails
+            if (this.environment === Environment$1.WORKER) {
+                // Workers typically have limited memory, be conservative
+                return isReadOnly ? 2000 : 1000;
+            }
+            return defaultSize;
+        }
+        catch (error) {
+            console.warn('Error detecting optimal cache size:', error);
+            return 1000; // Conservative default
+        }
+    }
+    /**
+     * Tune cache parameters based on statistics and environment
+     * This method is called periodically if auto-tuning is enabled
+     *
+     * The auto-tuning process:
+     * 1. Retrieves storage statistics if available
+     * 2. Tunes each parameter based on statistics and environment
+     * 3. Logs the tuned parameters if debug is enabled
+     *
+     * Auto-tuning helps optimize cache performance by adapting to:
+     * - The current environment (Node.js, browser, worker)
+     * - Available system resources (memory, CPU)
+     * - Usage patterns (read-heavy vs. write-heavy workloads)
+     * - Cache efficiency (hit/miss ratios)
+     */
+    async tuneParameters() {
+        // Skip if auto-tuning is disabled
+        if (!this.autoTune)
+            return;
+        // Check if it's time to tune parameters
+        const now = Date.now();
+        if (now - this.lastAutoTuneTime < this.autoTuneInterval)
+            return;
+        // Update last tune time
+        this.lastAutoTuneTime = now;
+        try {
+            // Get storage statistics if available
+            if (this.coldStorage && typeof this.coldStorage.getStatistics === 'function') {
+                this.storageStatistics = await this.coldStorage.getStatistics();
+            }
+            // Tune hot cache size
+            this.tuneHotCacheSize();
+            // Tune eviction threshold
+            this.tuneEvictionThreshold();
+            // Tune warm cache TTL
+            this.tuneWarmCacheTTL();
+            // Tune batch size
+            this.tuneBatchSize();
+            // Log tuned parameters if debug is enabled
+            if (process.env.DEBUG) {
+                console.log('Cache parameters auto-tuned:', {
+                    hotCacheMaxSize: this.hotCacheMaxSize,
+                    hotCacheEvictionThreshold: this.hotCacheEvictionThreshold,
+                    warmCacheTTL: this.warmCacheTTL,
+                    batchSize: this.batchSize
+                });
+            }
+        }
+        catch (error) {
+            console.warn('Error during cache parameter auto-tuning:', error);
+        }
+    }
+    /**
+     * Tune hot cache size based on statistics, environment, and operating mode
+     *
+     * The hot cache size is tuned based on:
+     * 1. Available memory in the current environment
+     * 2. Total number of nodes and edges in the system
+     * 3. Cache hit/miss ratio
+     * 4. Operating mode (read-only vs. read-write)
+     * 5. Storage type (S3, filesystem, memory)
+     *
+     * Enhanced algorithm:
+     * - Start with a size based on available memory and operating mode
+     * - For large datasets in S3 or other remote storage, use more aggressive caching
+     * - Adjust based on access patterns (read-heavy vs. write-heavy)
+     * - For read-only mode, prioritize cache size over eviction speed
+     * - Dynamically adjust based on hit/miss ratio and query patterns
+     */
+    tuneHotCacheSize() {
+        // Start with the base size from environment detection
+        let optimalSize = this.detectOptimalCacheSize();
+        // Check if we're in read-only mode
+        const isReadOnly = this.options?.readOnly || false;
+        // Check if we're using S3 or other remote storage
+        const isRemoteStorage = this.coldStorageType === StorageType.S3 ||
+            this.coldStorageType === StorageType.REMOTE_API;
+        // If we have storage statistics, adjust based on total nodes/edges
+        if (this.storageStatistics) {
+            const totalItems = (this.storageStatistics.totalNodes || 0) +
+                (this.storageStatistics.totalEdges || 0);
+            // If total items is significant, adjust cache size
+            if (totalItems > 0) {
+                // Base percentage to cache - adjusted based on mode and storage
+                let percentageToCache = 0.2; // Cache 20% of items by default
+                // For read-only mode, increase cache percentage
+                if (isReadOnly) {
+                    percentageToCache = 0.3; // 30% for read-only mode
+                    // For remote storage in read-only mode, be even more aggressive
+                    if (isRemoteStorage) {
+                        percentageToCache = 0.4; // 40% for remote storage in read-only mode
+                    }
+                }
+                // For remote storage in normal mode, increase slightly
+                else if (isRemoteStorage) {
+                    percentageToCache = 0.25; // 25% for remote storage
+                }
+                // For large datasets, cap the percentage to avoid excessive memory usage
+                if (totalItems > 1000000) { // Over 1 million items
+                    percentageToCache = Math.min(percentageToCache, 0.15);
+                }
+                else if (totalItems > 100000) { // Over 100K items
+                    percentageToCache = Math.min(percentageToCache, 0.25);
+                }
+                const statisticsBasedSize = Math.ceil(totalItems * percentageToCache);
+                // Use the smaller of the two to avoid memory issues
+                optimalSize = Math.min(optimalSize, statisticsBasedSize);
+            }
+        }
+        // Adjust based on hit/miss ratio if we have enough data
+        const totalAccesses = this.stats.hits + this.stats.misses;
+        if (totalAccesses > 100) {
+            const hitRatio = this.stats.hits / totalAccesses;
+            // Base adjustment factor
+            let hitRatioFactor = 1.0;
+            // If hit ratio is low, we might need a larger cache
+            if (hitRatio < 0.5) {
+                // Calculate adjustment factor based on hit ratio
+                const baseAdjustment = 0.5 - hitRatio;
+                // For read-only mode or remote storage, be more aggressive
+                if (isReadOnly || isRemoteStorage) {
+                    hitRatioFactor = 1 + (baseAdjustment * 1.5); // Up to 75% increase
+                }
+                else {
+                    hitRatioFactor = 1 + baseAdjustment; // Up to 50% increase
+                }
+                optimalSize = Math.ceil(optimalSize * hitRatioFactor);
+            }
+            // If hit ratio is very high, we might be able to reduce cache size slightly
+            else if (hitRatio > 0.9 && !isReadOnly && !isRemoteStorage) {
+                // Only reduce cache size in normal mode with local storage
+                // and only if hit ratio is very high
+                hitRatioFactor = 0.9; // 10% reduction
+                optimalSize = Math.ceil(optimalSize * hitRatioFactor);
+            }
+        }
+        // Check for operation patterns if available
+        if (this.storageStatistics?.operations) {
+            const ops = this.storageStatistics.operations;
+            const totalOps = ops.total || 1;
+            // Calculate read/write ratio
+            const readOps = (ops.search || 0) + (ops.get || 0);
+            (ops.add || 0) + (ops.update || 0) + (ops.delete || 0);
+            if (totalOps > 100) {
+                const readRatio = readOps / totalOps;
+                // For read-heavy workloads, increase cache size
+                if (readRatio > 0.8) {
+                    // More aggressive for remote storage
+                    const readAdjustment = isRemoteStorage ? 1.3 : 1.2;
+                    optimalSize = Math.ceil(optimalSize * readAdjustment);
+                }
+            }
+        }
+        // Ensure we have a reasonable minimum size based on environment and mode
+        let minSize = 1000; // Default minimum
+        // For read-only mode, use a higher minimum
+        if (isReadOnly) {
+            minSize = 2000;
+        }
+        // For remote storage, use an even higher minimum
+        if (isRemoteStorage) {
+            minSize = isReadOnly ? 3000 : 2000;
+        }
+        optimalSize = Math.max(optimalSize, minSize);
+        // Update the hot cache max size
+        this.hotCacheMaxSize = optimalSize;
+        this.stats.maxSize = optimalSize;
+    }
+    /**
+     * Tune eviction threshold based on statistics
+     *
+     * The eviction threshold determines when items start being evicted from the hot cache.
+     * It is tuned based on:
+     * 1. Cache hit/miss ratio
+     * 2. Operation patterns (read-heavy vs. write-heavy workloads)
+     *
+     * Algorithm:
+     * - Start with a default threshold of 0.8 (80% of max size)
+     * - For high hit ratios, increase the threshold to keep more items in cache
+     * - For low hit ratios, decrease the threshold to evict items more aggressively
+     * - For read-heavy workloads, use a higher threshold
+     * - For write-heavy workloads, use a lower threshold
+     */
+    tuneEvictionThreshold() {
+        // Default threshold
+        let threshold = 0.8;
+        // Adjust based on hit/miss ratio if we have enough data
+        const totalAccesses = this.stats.hits + this.stats.misses;
+        if (totalAccesses > 100) {
+            const hitRatio = this.stats.hits / totalAccesses;
+            // If hit ratio is high, we can use a higher threshold
+            // If hit ratio is low, we should use a lower threshold to evict more aggressively
+            if (hitRatio > 0.8) {
+                // High hit ratio, increase threshold (up to 0.9)
+                threshold = Math.min(0.9, 0.8 + (hitRatio - 0.8));
+            }
+            else if (hitRatio < 0.5) {
+                // Low hit ratio, decrease threshold (down to 0.6)
+                threshold = Math.max(0.6, 0.8 - (0.5 - hitRatio));
+            }
+        }
+        // If we have storage statistics with operation counts, adjust based on operation patterns
+        if (this.storageStatistics && this.storageStatistics.operations) {
+            const ops = this.storageStatistics.operations;
+            const totalOps = ops.total || 1;
+            // Calculate read/write ratio
+            const readOps = ops.search || 0;
+            const writeOps = (ops.add || 0) + (ops.update || 0) + (ops.delete || 0);
+            if (totalOps > 100) {
+                const readRatio = readOps / totalOps;
+                const writeRatio = writeOps / totalOps;
+                // For read-heavy workloads, use higher threshold
+                // For write-heavy workloads, use lower threshold
+                if (readRatio > 0.8) {
+                    // Read-heavy, increase threshold slightly
+                    threshold = Math.min(0.9, threshold + 0.05);
+                }
+                else if (writeRatio > 0.5) {
+                    // Write-heavy, decrease threshold
+                    threshold = Math.max(0.6, threshold - 0.1);
+                }
+            }
+        }
+        // Update the eviction threshold
+        this.hotCacheEvictionThreshold = threshold;
+    }
+    /**
+     * Tune warm cache TTL based on statistics
+     *
+     * The warm cache TTL determines how long items remain in the warm cache.
+     * It is tuned based on:
+     * 1. Update frequency from operation statistics
+     *
+     * Algorithm:
+     * - Start with a default TTL of 24 hours
+     * - For frequently updated data, use a shorter TTL
+     * - For rarely updated data, use a longer TTL
+     */
+    tuneWarmCacheTTL() {
+        // Default TTL (24 hours)
+        let ttl = 24 * 60 * 60 * 1000;
+        // If we have storage statistics with operation counts, adjust based on update frequency
+        if (this.storageStatistics && this.storageStatistics.operations) {
+            const ops = this.storageStatistics.operations;
+            const totalOps = ops.total || 1;
+            const updateOps = (ops.update || 0);
+            if (totalOps > 100) {
+                const updateRatio = updateOps / totalOps;
+                // For frequently updated data, use shorter TTL
+                // For rarely updated data, use longer TTL
+                if (updateRatio > 0.3) {
+                    // Frequently updated, decrease TTL (down to 6 hours)
+                    ttl = Math.max(6 * 60 * 60 * 1000, ttl * (1 - updateRatio));
+                }
+                else if (updateRatio < 0.1) {
+                    // Rarely updated, increase TTL (up to 48 hours)
+                    ttl = Math.min(48 * 60 * 60 * 1000, ttl * (1.5 - updateRatio));
+                }
+            }
+        }
+        // Update the warm cache TTL
+        this.warmCacheTTL = ttl;
+    }
+    /**
+     * Tune batch size based on environment, statistics, and operating mode
+     *
+     * The batch size determines how many items are processed in a single batch
+     * for operations like prefetching. It is tuned based on:
+     * 1. Current environment (Node.js, browser, worker)
+     * 2. Available memory
+     * 3. Operation patterns
+     * 4. Cache hit/miss ratio
+     * 5. Operating mode (read-only vs. read-write)
+     * 6. Storage type (S3, filesystem, memory)
+     * 7. Dataset size
+     *
+     * Enhanced algorithm:
+     * - Start with a default based on the environment
+     * - For large datasets in S3 or other remote storage, use larger batches
+     * - For read-only mode, use larger batches to improve throughput
+     * - Dynamically adjust based on network latency and throughput
+     * - Balance between memory usage and performance
+     */
+    tuneBatchSize() {
+        // Default batch size
+        let batchSize = 10;
+        // Check if we're in read-only mode
+        const isReadOnly = this.options?.readOnly || false;
+        // Check if we're using S3 or other remote storage
+        const isRemoteStorage = this.coldStorageType === StorageType.S3 ||
+            this.coldStorageType === StorageType.REMOTE_API;
+        // Get the total dataset size if available
+        const totalItems = this.storageStatistics ?
+            (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) : 0;
+        // Determine if we're dealing with a large dataset
+        const isLargeDataset = totalItems > 100000;
+        const isVeryLargeDataset = totalItems > 1000000;
+        // Base batch size adjustment based on environment
+        if (this.environment === Environment$1.NODE) {
+            // Node.js can handle larger batches
+            batchSize = isReadOnly ? 30 : 20;
+            // For remote storage, increase batch size
+            if (isRemoteStorage) {
+                batchSize = isReadOnly ? 50 : 30;
+            }
+            // For large datasets, adjust batch size
+            if (isLargeDataset) {
+                batchSize = Math.min(100, batchSize * 1.5);
+            }
+            // For very large datasets, adjust even more
+            if (isVeryLargeDataset) {
+                batchSize = Math.min(200, batchSize * 2);
+            }
+        }
+        else if (this.environment === Environment$1.BROWSER) {
+            // Browsers might need smaller batches
+            batchSize = isReadOnly ? 15 : 10;
+            // If we have memory information, adjust accordingly
+            if (navigator.deviceMemory) {
+                // Scale batch size with available memory
+                const memoryFactor = isReadOnly ? 3 : 2;
+                batchSize = Math.max(5, Math.min(30, Math.floor(navigator.deviceMemory * memoryFactor)));
+                // For large datasets, adjust based on memory
+                if (isLargeDataset && navigator.deviceMemory > 4) {
+                    batchSize = Math.min(50, batchSize * 1.5);
+                }
+            }
+        }
+        else if (this.environment === Environment$1.WORKER) {
+            // Workers can handle moderate batch sizes
+            batchSize = isReadOnly ? 20 : 15;
+        }
+        // If we have storage statistics with operation counts, adjust based on operation patterns
+        if (this.storageStatistics && this.storageStatistics.operations) {
+            const ops = this.storageStatistics.operations;
+            const totalOps = ops.total || 1;
+            const searchOps = (ops.search || 0);
+            const getOps = (ops.get || 0);
+            if (totalOps > 100) {
+                // Calculate search and get ratios
+                const searchRatio = searchOps / totalOps;
+                const getRatio = getOps / totalOps;
+                // For search-heavy workloads, use larger batch size
+                if (searchRatio > 0.6) {
+                    // Search-heavy, increase batch size
+                    const searchFactor = isRemoteStorage ? 1.8 : 1.5;
+                    batchSize = Math.min(isRemoteStorage ? 200 : 100, Math.ceil(batchSize * searchFactor));
+                }
+                // For get-heavy workloads, adjust batch size
+                if (getRatio > 0.6) {
+                    // Get-heavy, adjust batch size based on storage type
+                    if (isRemoteStorage) {
+                        // For remote storage, larger batches reduce network overhead
+                        batchSize = Math.min(150, Math.ceil(batchSize * 1.5));
+                    }
+                    else {
+                        // For local storage, smaller batches might be more efficient
+                        batchSize = Math.max(10, Math.ceil(batchSize * 0.9));
+                    }
+                }
+            }
+        }
+        // Adjust based on hit/miss ratio if we have enough data
+        const totalAccesses = this.stats.hits + this.stats.misses;
+        if (totalAccesses > 100) {
+            const hitRatio = this.stats.hits / totalAccesses;
+            // Base adjustment factors
+            let increaseFactorForLowHitRatio = isRemoteStorage ? 1.5 : 1.2;
+            let decreaseFactorForHighHitRatio = 0.8;
+            // In read-only mode, be more aggressive with batch size adjustments
+            if (isReadOnly) {
+                increaseFactorForLowHitRatio = isRemoteStorage ? 2.0 : 1.5;
+                decreaseFactorForHighHitRatio = 0.9; // Less reduction in read-only mode
+            }
+            // If hit ratio is high, we can use smaller batches
+            if (hitRatio > 0.8 && !isVeryLargeDataset) {
+                // High hit ratio, decrease batch size slightly
+                // But don't decrease too much for large datasets or remote storage
+                if (!(isLargeDataset && isRemoteStorage)) {
+                    batchSize = Math.max(isReadOnly ? 10 : 5, Math.floor(batchSize * decreaseFactorForHighHitRatio));
+                }
+            }
+            // If hit ratio is low, we need larger batches
+            else if (hitRatio < 0.5) {
+                // Low hit ratio, increase batch size
+                const maxBatchSize = isRemoteStorage ?
+                    (isVeryLargeDataset ? 300 : 200) :
+                    (isVeryLargeDataset ? 150 : 100);
+                batchSize = Math.min(maxBatchSize, Math.ceil(batchSize * increaseFactorForLowHitRatio));
+            }
+        }
+        // Set minimum batch sizes based on storage type and mode
+        let minBatchSize = 5;
+        if (isRemoteStorage) {
+            minBatchSize = isReadOnly ? 20 : 10;
+        }
+        else if (isReadOnly) {
+            minBatchSize = 10;
+        }
+        // Ensure batch size is within reasonable limits
+        batchSize = Math.max(minBatchSize, batchSize);
+        // Cap maximum batch size based on environment and storage
+        const maxBatchSize = isRemoteStorage ?
+            (this.environment === Environment$1.NODE ? 300 : 150) :
+            (this.environment === Environment$1.NODE ? 150 : 75);
+        batchSize = Math.min(maxBatchSize, batchSize);
+        // Update the batch size
+        this.batchSize = batchSize;
+    }
+    /**
+     * Detect the appropriate warm storage type based on environment
+     */
+    detectWarmStorageType() {
+        if (this.environment === Environment$1.BROWSER) {
+            // Use OPFS if available, otherwise use memory
+            if ('storage' in navigator && 'getDirectory' in navigator.storage) {
+                return StorageType.OPFS;
+            }
+            return StorageType.MEMORY;
+        }
+        else if (this.environment === Environment$1.WORKER) {
+            // Use OPFS if available, otherwise use memory
+            if ('storage' in self && 'getDirectory' in self.storage) {
+                return StorageType.OPFS;
+            }
+            return StorageType.MEMORY;
+        }
+        else {
+            // In Node.js, use filesystem
+            return StorageType.FILESYSTEM;
+        }
+    }
+    /**
+     * Detect the appropriate cold storage type based on environment
+     */
+    detectColdStorageType() {
+        if (this.environment === Environment$1.BROWSER) {
+            // Use OPFS if available, otherwise use memory
+            if ('storage' in navigator && 'getDirectory' in navigator.storage) {
+                return StorageType.OPFS;
+            }
+            return StorageType.MEMORY;
+        }
+        else if (this.environment === Environment$1.WORKER) {
+            // Use OPFS if available, otherwise use memory
+            if ('storage' in self && 'getDirectory' in self.storage) {
+                return StorageType.OPFS;
+            }
+            return StorageType.MEMORY;
+        }
+        else {
+            // In Node.js, use S3 if configured, otherwise filesystem
+            return StorageType.S3;
+        }
+    }
+    /**
+     * Initialize warm storage adapter
+     */
+    initializeWarmStorage() {
+        // Implementation depends on the detected storage type
+        // For now, return null as this will be provided by the storage adapter
+        return null;
+    }
+    /**
+     * Initialize cold storage adapter
+     */
+    initializeColdStorage() {
+        // Implementation depends on the detected storage type
+        // For now, return null as this will be provided by the storage adapter
+        return null;
+    }
+    /**
+     * Get an item from cache, trying each level in order
+     * @param id The item ID
+     * @returns The cached item or null if not found
+     */
+    async get(id) {
+        // Check if it's time to tune parameters
+        await this.checkAndTuneParameters();
+        // Try hot cache first (fastest)
+        const hotCacheEntry = this.hotCache.get(id);
+        if (hotCacheEntry) {
+            // Update access metadata
+            hotCacheEntry.lastAccessed = Date.now();
+            hotCacheEntry.accessCount++;
+            // Update stats
+            this.stats.hits++;
+            return hotCacheEntry.data;
+        }
+        // Try warm cache next
+        try {
+            const warmCacheItem = await this.getFromWarmCache(id);
+            if (warmCacheItem) {
+                // Promote to hot cache
+                this.addToHotCache(id, warmCacheItem);
+                // Update stats
+                this.stats.hits++;
+                return warmCacheItem;
+            }
+        }
+        catch (error) {
+            console.warn(`Error accessing warm cache for ${id}:`, error);
+        }
+        // Finally, try cold storage
+        try {
+            const coldStorageItem = await this.getFromColdStorage(id);
+            if (coldStorageItem) {
+                // Promote to hot and warm caches
+                this.addToHotCache(id, coldStorageItem);
+                await this.addToWarmCache(id, coldStorageItem);
+                // Update stats
+                this.stats.misses++;
+                return coldStorageItem;
+            }
+        }
+        catch (error) {
+            console.warn(`Error accessing cold storage for ${id}:`, error);
+        }
+        // Item not found in any cache level
+        this.stats.misses++;
+        return null;
+    }
+    /**
+     * Get an item from warm cache
+     * @param id The item ID
+     * @returns The cached item or null if not found
+     */
+    async getFromWarmCache(id) {
+        if (!this.warmStorage)
+            return null;
+        try {
+            return await this.warmStorage.get(id);
+        }
+        catch (error) {
+            console.warn(`Error getting item ${id} from warm cache:`, error);
+            return null;
+        }
+    }
+    /**
+     * Get an item from cold storage
+     * @param id The item ID
+     * @returns The item or null if not found
+     */
+    async getFromColdStorage(id) {
+        if (!this.coldStorage)
+            return null;
+        try {
+            return await this.coldStorage.get(id);
+        }
+        catch (error) {
+            console.warn(`Error getting item ${id} from cold storage:`, error);
+            return null;
+        }
+    }
+    /**
+     * Add an item to hot cache
+     * @param id The item ID
+     * @param item The item to cache
+     */
+    addToHotCache(id, item) {
+        // Check if we need to evict items
+        if (this.hotCache.size >= this.hotCacheMaxSize * this.hotCacheEvictionThreshold) {
+            this.evictFromHotCache();
+        }
+        // Add to hot cache
+        this.hotCache.set(id, {
+            data: item,
+            lastAccessed: Date.now(),
+            accessCount: 1,
+            expiresAt: null // Hot cache items don't expire
+        });
+        // Update stats
+        this.stats.size = this.hotCache.size;
+    }
+    /**
+     * Add an item to warm cache
+     * @param id The item ID
+     * @param item The item to cache
+     */
+    async addToWarmCache(id, item) {
+        if (!this.warmStorage)
+            return;
+        try {
+            // Add to warm cache with TTL
+            await this.warmStorage.set(id, item, {
+                ttl: this.warmCacheTTL
+            });
+        }
+        catch (error) {
+            console.warn(`Error adding item ${id} to warm cache:`, error);
+        }
+    }
+    /**
+     * Evict items from hot cache based on LRU policy
+     */
+    evictFromHotCache() {
+        // Find the least recently used items
+        const entries = Array.from(this.hotCache.entries());
+        // Sort by last accessed time (oldest first)
+        entries.sort((a, b) => a[1].lastAccessed - b[1].lastAccessed);
+        // Remove the oldest 20% of items
+        const itemsToRemove = Math.ceil(this.hotCache.size * 0.2);
+        for (let i = 0; i < itemsToRemove && i < entries.length; i++) {
+            this.hotCache.delete(entries[i][0]);
+            this.stats.evictions++;
+        }
+        // Update stats
+        this.stats.size = this.hotCache.size;
+        if (process.env.DEBUG) {
+            console.log(`Evicted ${itemsToRemove} items from hot cache, new size: ${this.hotCache.size}`);
+        }
+    }
+    /**
+     * Set an item in all cache levels
+     * @param id The item ID
+     * @param item The item to cache
+     */
+    async set(id, item) {
+        // Add to hot cache
+        this.addToHotCache(id, item);
+        // Add to warm cache
+        await this.addToWarmCache(id, item);
+        // Add to cold storage
+        if (this.coldStorage) {
+            try {
+                await this.coldStorage.set(id, item);
+            }
+            catch (error) {
+                console.warn(`Error adding item ${id} to cold storage:`, error);
+            }
+        }
+    }
+    /**
+     * Delete an item from all cache levels
+     * @param id The item ID to delete
+     */
+    async delete(id) {
+        // Remove from hot cache
+        this.hotCache.delete(id);
+        // Remove from warm cache
+        if (this.warmStorage) {
+            try {
+                await this.warmStorage.delete(id);
+            }
+            catch (error) {
+                console.warn(`Error deleting item ${id} from warm cache:`, error);
+            }
+        }
+        // Remove from cold storage
+        if (this.coldStorage) {
+            try {
+                await this.coldStorage.delete(id);
+            }
+            catch (error) {
+                console.warn(`Error deleting item ${id} from cold storage:`, error);
+            }
+        }
+        // Update stats
+        this.stats.size = this.hotCache.size;
+    }
+    /**
+     * Clear all cache levels
+     */
+    async clear() {
+        // Clear hot cache
+        this.hotCache.clear();
+        // Clear warm cache
+        if (this.warmStorage) {
+            try {
+                await this.warmStorage.clear();
+            }
+            catch (error) {
+                console.warn('Error clearing warm cache:', error);
+            }
+        }
+        // Clear cold storage
+        if (this.coldStorage) {
+            try {
+                await this.coldStorage.clear();
+            }
+            catch (error) {
+                console.warn('Error clearing cold storage:', error);
+            }
+        }
+        // Reset stats
+        this.stats = {
+            hits: 0,
+            misses: 0,
+            evictions: 0,
+            size: 0,
+            maxSize: this.hotCacheMaxSize
+        };
+    }
+    /**
+     * Get cache statistics
+     * @returns Cache statistics
+     */
+    getStats() {
+        return { ...this.stats };
+    }
+    /**
+     * Prefetch items based on ID patterns or relationships
+     * @param ids Array of IDs to prefetch
+     */
+    async prefetch(ids) {
+        // Check if it's time to tune parameters
+        await this.checkAndTuneParameters();
+        // Prefetch in batches to avoid overwhelming the system
+        const batches = [];
+        // Split into batches using the configurable batch size
+        for (let i = 0; i < ids.length; i += this.batchSize) {
+            const batch = ids.slice(i, i + this.batchSize);
+            batches.push(batch);
+        }
+        // Process each batch
+        for (const batch of batches) {
+            await Promise.all(batch.map(async (id) => {
+                // Skip if already in hot cache
+                if (this.hotCache.has(id))
+                    return;
+                try {
+                    // Try to get from any cache level
+                    await this.get(id);
+                }
+                catch (error) {
+                    // Ignore errors during prefetching
+                    if (process.env.DEBUG) {
+                        console.warn(`Error prefetching ${id}:`, error);
+                    }
+                }
+            }));
+        }
+    }
+    /**
+     * Check if it's time to tune parameters and do so if needed
+     * This is called before operations that might benefit from tuned parameters
+     *
+     * This method serves as a checkpoint for auto-tuning, ensuring that:
+     * 1. Parameters are tuned periodically based on the auto-tune interval
+     * 2. Tuning happens before critical operations that would benefit from optimized parameters
+     * 3. Tuning doesn't happen too frequently, which could impact performance
+     *
+     * By calling this method before get(), getMany(), and prefetch() operations,
+     * we ensure that the cache parameters are optimized for the current workload
+     * without adding unnecessary overhead to every operation.
+     */
+    async checkAndTuneParameters() {
+        // Skip if auto-tuning is disabled
+        if (!this.autoTune)
+            return;
+        // Check if it's time to tune parameters
+        const now = Date.now();
+        if (now - this.lastAutoTuneTime >= this.autoTuneInterval) {
+            await this.tuneParameters();
+        }
+    }
+    /**
+     * Get multiple items at once, optimizing for batch retrieval
+     * @param ids Array of IDs to get
+     * @returns Map of ID to item
+     */
+    async getMany(ids) {
+        // Check if it's time to tune parameters
+        await this.checkAndTuneParameters();
+        const result = new Map();
+        // First check hot cache for all IDs
+        const missingIds = [];
+        for (const id of ids) {
+            const hotCacheEntry = this.hotCache.get(id);
+            if (hotCacheEntry) {
+                // Update access metadata
+                hotCacheEntry.lastAccessed = Date.now();
+                hotCacheEntry.accessCount++;
+                // Add to result
+                result.set(id, hotCacheEntry.data);
+                // Update stats
+                this.stats.hits++;
+            }
+            else {
+                missingIds.push(id);
+            }
+        }
+        if (missingIds.length === 0) {
+            return result;
+        }
+        // Try to get missing items from warm cache
+        if (this.warmStorage) {
+            try {
+                const warmCacheItems = await this.warmStorage.getMany(missingIds);
+                for (const [id, item] of warmCacheItems.entries()) {
+                    if (item) {
+                        // Promote to hot cache
+                        this.addToHotCache(id, item);
+                        // Add to result
+                        result.set(id, item);
+                        // Update stats
+                        this.stats.hits++;
+                        // Remove from missing IDs
+                        const index = missingIds.indexOf(id);
+                        if (index !== -1) {
+                            missingIds.splice(index, 1);
+                        }
+                    }
+                }
+            }
+            catch (error) {
+                console.warn('Error accessing warm cache for batch:', error);
+            }
+        }
+        if (missingIds.length === 0) {
+            return result;
+        }
+        // Try to get remaining missing items from cold storage
+        if (this.coldStorage) {
+            try {
+                const coldStorageItems = await this.coldStorage.getMany(missingIds);
+                for (const [id, item] of coldStorageItems.entries()) {
+                    if (item) {
+                        // Promote to hot and warm caches
+                        this.addToHotCache(id, item);
+                        await this.addToWarmCache(id, item);
+                        // Add to result
+                        result.set(id, item);
+                        // Update stats
+                        this.stats.misses++;
+                    }
+                }
+            }
+            catch (error) {
+                console.warn('Error accessing cold storage for batch:', error);
+            }
+        }
+        return result;
+    }
+    /**
+     * Set the storage adapters for warm and cold caches
+     * @param warmStorage Warm cache storage adapter
+     * @param coldStorage Cold storage adapter
      */
-    async executeDelete(operation, operationName) {
-        return this.deleteExecutor(operation, operationName);
+    setStorageAdapters(warmStorage, coldStorage) {
+        this.warmStorage = warmStorage;
+        this.coldStorage = coldStorage;
     }
 }
 
@@ -7973,6 +9492,8 @@ class S3CompatibleStorage extends BaseStorage {
         this.activeLocks = new Set();
         // Change log for efficient synchronization
         this.changeLogPrefix = 'change-log/';
+        // Node cache to avoid redundant API calls
+        this.nodeCache = new Map();
         // Batch update timer ID
         this.statisticsBatchUpdateTimerId = null;
         // Flag to indicate if statistics have been modified since last save
@@ -7998,6 +9519,9 @@ class S3CompatibleStorage extends BaseStorage {
         this.verbPrefix = `${VERBS_DIR}/`;
         this.metadataPrefix = `${METADATA_DIR}/`;
         this.indexPrefix = `${INDEX_DIR}/`;
+        // Initialize cache managers
+        this.nounCacheManager = new CacheManager(options.cacheConfig);
+        this.verbCacheManager = new CacheManager(options.cacheConfig);
     }
     /**
      * Initialize the storage adapter
@@ -8036,6 +9560,78 @@ class S3CompatibleStorage extends BaseStorage {
             await this.s3Client.send(new HeadBucketCommand({
                 Bucket: this.bucketName
             }));
+            // Create storage adapter proxies for the cache managers
+            const nounStorageAdapter = {
+                get: async (id) => this.getNoun_internal(id),
+                set: async (id, node) => this.saveNoun_internal(node),
+                delete: async (id) => this.deleteNoun_internal(id),
+                getMany: async (ids) => {
+                    const result = new Map();
+                    // Process in batches to avoid overwhelming the S3 API
+                    const batchSize = 10;
+                    const batches = [];
+                    // Split into batches
+                    for (let i = 0; i < ids.length; i += batchSize) {
+                        const batch = ids.slice(i, i + batchSize);
+                        batches.push(batch);
+                    }
+                    // Process each batch
+                    for (const batch of batches) {
+                        const batchResults = await Promise.all(batch.map(async (id) => {
+                            const node = await this.getNoun_internal(id);
+                            return { id, node };
+                        }));
+                        // Add results to map
+                        for (const { id, node } of batchResults) {
+                            if (node) {
+                                result.set(id, node);
+                            }
+                        }
+                    }
+                    return result;
+                },
+                clear: async () => {
+                    // No-op for now, as we don't want to clear the entire storage
+                    // This would be implemented if needed
+                }
+            };
+            const verbStorageAdapter = {
+                get: async (id) => this.getVerb_internal(id),
+                set: async (id, edge) => this.saveVerb_internal(edge),
+                delete: async (id) => this.deleteVerb_internal(id),
+                getMany: async (ids) => {
+                    const result = new Map();
+                    // Process in batches to avoid overwhelming the S3 API
+                    const batchSize = 10;
+                    const batches = [];
+                    // Split into batches
+                    for (let i = 0; i < ids.length; i += batchSize) {
+                        const batch = ids.slice(i, i + batchSize);
+                        batches.push(batch);
+                    }
+                    // Process each batch
+                    for (const batch of batches) {
+                        const batchResults = await Promise.all(batch.map(async (id) => {
+                            const edge = await this.getVerb_internal(id);
+                            return { id, edge };
+                        }));
+                        // Add results to map
+                        for (const { id, edge } of batchResults) {
+                            if (edge) {
+                                result.set(id, edge);
+                            }
+                        }
+                    }
+                    return result;
+                },
+                clear: async () => {
+                    // No-op for now, as we don't want to clear the entire storage
+                    // This would be implemented if needed
+                }
+            };
+            // Set storage adapters for cache managers
+            this.nounCacheManager.setStorageAdapters(nounStorageAdapter, nounStorageAdapter);
+            this.verbCacheManager.setStorageAdapters(verbStorageAdapter, verbStorageAdapter);
             this.isInitialized = true;
         }
         catch (error) {
@@ -8144,7 +9740,10 @@ class S3CompatibleStorage extends BaseStorage {
                 const parsedNode = JSON.parse(bodyContents);
                 console.log(`Parsed node data for ${id}:`, parsedNode);
                 // Ensure the parsed node has the expected properties
-                if (!parsedNode || !parsedNode.id || !parsedNode.vector || !parsedNode.connections) {
+                if (!parsedNode ||
+                    !parsedNode.id ||
+                    !parsedNode.vector ||
+                    !parsedNode.connections) {
                     console.error(`Invalid node data for ${id}:`, parsedNode);
                     return null;
                 }
@@ -8180,100 +9779,118 @@ class S3CompatibleStorage extends BaseStorage {
     }
     /**
      * Get all nodes from storage
+     * @deprecated This method is deprecated and will be removed in a future version.
+     * It can cause memory issues with large datasets. Use getNodesWithPagination() instead.
      */
     async getAllNodes() {
         await this.ensureInitialized();
+        console.warn('WARNING: getAllNodes() is deprecated and will be removed in a future version. Use getNodesWithPagination() instead.');
+        try {
+            // Use the paginated method with a large limit to maintain backward compatibility
+            // but warn about potential issues
+            const result = await this.getNodesWithPagination({
+                limit: 1000, // Reasonable limit to avoid memory issues
+                useCache: true
+            });
+            if (result.hasMore) {
+                console.warn(`WARNING: Only returning the first 1000 nodes. There are more nodes available. Use getNodesWithPagination() for proper pagination.`);
+            }
+            return result.nodes;
+        }
+        catch (error) {
+            console.error('Failed to get all nodes:', error);
+            return [];
+        }
+    }
+    /**
+     * Get nodes with pagination
+     * @param options Pagination options
+     * @returns Promise that resolves to a paginated result of nodes
+     */
+    async getNodesWithPagination(options = {}) {
+        await this.ensureInitialized();
+        const limit = options.limit || 100;
+        const useCache = options.useCache !== false;
         try {
             // Import the ListObjectsV2Command and GetObjectCommand only when needed
-            const { ListObjectsV2Command, GetObjectCommand } = await import('@aws-sdk/client-s3');
-            console.log(`Getting all nodes from bucket ${this.bucketName} with prefix ${this.nounPrefix}`);
-            // List all objects in the nouns directory
+            const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
+            // List objects with pagination
             const listResponse = await this.s3Client.send(new ListObjectsV2Command({
                 Bucket: this.bucketName,
-                Prefix: this.nounPrefix
+                Prefix: this.nounPrefix,
+                MaxKeys: limit,
+                ContinuationToken: options.cursor
             }));
+            // If listResponse is null/undefined or there are no objects, return an empty result
+            if (!listResponse ||
+                !listResponse.Contents ||
+                listResponse.Contents.length === 0) {
+                return {
+                    nodes: [],
+                    hasMore: false
+                };
+            }
+            // Extract node IDs from the keys
+            const nodeIds = listResponse.Contents
+                .filter((object) => object && object.Key)
+                .map((object) => object.Key.replace(this.nounPrefix, '').replace('.json', ''));
+            // Use the cache manager to get nodes efficiently
             const nodes = [];
-            // If listResponse is null/undefined or there are no objects, return an empty array
-            if (!listResponse || !listResponse.Contents || listResponse.Contents.length === 0) {
-                console.log(`No nodes found in bucket ${this.bucketName} with prefix ${this.nounPrefix}`);
-                return nodes;
-            }
-            console.log(`Found ${listResponse.Contents.length} nodes in bucket ${this.bucketName}`);
-            // Debug: Log all keys found
-            console.log('Keys found:');
-            for (const object of listResponse.Contents) {
-                if (object && object.Key) {
-                    console.log(`- ${object.Key}`);
+            if (useCache) {
+                // Get nodes from cache manager
+                const cachedNodes = await this.nounCacheManager.getMany(nodeIds);
+                // Add nodes to result in the same order as nodeIds
+                for (const id of nodeIds) {
+                    const node = cachedNodes.get(id);
+                    if (node) {
+                        nodes.push(node);
+                    }
                 }
             }
-            // Get each node
-            const nodePromises = listResponse.Contents.map(async (object) => {
-                if (!object || !object.Key) {
-                    console.log(`Skipping undefined object or object without Key`);
-                    return null;
+            else {
+                // Get nodes directly from S3 without using cache
+                // Process in smaller batches to reduce memory usage
+                const batchSize = 50;
+                const batches = [];
+                // Split into batches
+                for (let i = 0; i < nodeIds.length; i += batchSize) {
+                    const batch = nodeIds.slice(i, i + batchSize);
+                    batches.push(batch);
                 }
-                try {
-                    // Extract node ID from the key (remove prefix and .json extension)
-                    const nodeId = object.Key.replace(this.nounPrefix, '').replace('.json', '');
-                    console.log(`Getting node with ID ${nodeId} from key ${object.Key}`);
-                    // Get the node data
-                    const response = await this.s3Client.send(new GetObjectCommand({
-                        Bucket: this.bucketName,
-                        Key: object.Key
-                    }));
-                    // Check if response is null or undefined
-                    if (!response || !response.Body) {
-                        console.log(`No response or response body for node ${nodeId}`);
-                        return null;
-                    }
-                    // Convert the response body to a string
-                    const bodyContents = await response.Body.transformToString();
-                    console.log(`Retrieved node body for ${nodeId}: ${bodyContents.substring(0, 100)}${bodyContents.length > 100 ? '...' : ''}`);
-                    // Parse the JSON string
-                    try {
-                        const parsedNode = JSON.parse(bodyContents);
-                        console.log(`Parsed node data for ${nodeId}:`, parsedNode);
-                        // Ensure the parsed node has the expected properties
-                        if (!parsedNode || !parsedNode.id || !parsedNode.vector || !parsedNode.connections) {
-                            console.error(`Invalid node data for ${nodeId}:`, parsedNode);
+                // Process each batch sequentially
+                for (const batch of batches) {
+                    const batchNodes = await Promise.all(batch.map(async (id) => {
+                        try {
+                            return await this.getNoun_internal(id);
+                        }
+                        catch (error) {
                             return null;
                         }
-                        // Convert serialized connections back to Map<number, Set<string>>
-                        const connections = new Map();
-                        for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
-                            connections.set(Number(level), new Set(nodeIds));
+                    }));
+                    // Add non-null nodes to result
+                    for (const node of batchNodes) {
+                        if (node) {
+                            nodes.push(node);
                         }
-                        const node = {
-                            id: parsedNode.id,
-                            vector: parsedNode.vector,
-                            connections
-                        };
-                        console.log(`Successfully retrieved node ${nodeId}:`, node);
-                        return node;
                     }
-                    catch (parseError) {
-                        console.error(`Failed to parse node data for ${nodeId}:`, parseError);
-                        return null;
-                    }
-                }
-                catch (error) {
-                    console.error(`Error getting node from ${object.Key}:`, error);
-                    return null;
                 }
-            });
-            // Wait for all promises to resolve and filter out nulls
-            const resolvedNodes = await Promise.all(nodePromises);
-            const filteredNodes = resolvedNodes.filter((node) => node !== null);
-            console.log(`Returning ${filteredNodes.length} nodes`);
-            // Debug: Log all nodes being returned
-            for (const node of filteredNodes) {
-                console.log(`- Node ${node.id}`);
             }
-            return filteredNodes;
+            // Determine if there are more nodes
+            const hasMore = !!listResponse.IsTruncated;
+            // Set next cursor if there are more nodes
+            const nextCursor = listResponse.NextContinuationToken;
+            return {
+                nodes,
+                hasMore,
+                nextCursor
+            };
         }
         catch (error) {
-            console.error('Failed to get all nodes:', error);
-            return [];
+            console.error('Failed to get nodes with pagination:', error);
+            return {
+                nodes: [],
+                hasMore: false
+            };
         }
     }
     /**
@@ -8292,14 +9909,31 @@ class S3CompatibleStorage extends BaseStorage {
     async getNodesByNounType(nounType) {
         await this.ensureInitialized();
         try {
-            // Get all nodes
-            const allNodes = await this.getAllNodes();
-            // Filter nodes by noun type using metadata
             const filteredNodes = [];
-            for (const node of allNodes) {
-                const metadata = await this.getMetadata(node.id);
-                if (metadata && metadata.noun === nounType) {
-                    filteredNodes.push(node);
+            let hasMore = true;
+            let cursor = undefined;
+            // Use pagination to process nodes in batches
+            while (hasMore) {
+                // Get a batch of nodes
+                const result = await this.getNodesWithPagination({
+                    limit: 100,
+                    cursor,
+                    useCache: true
+                });
+                // Filter nodes by noun type using metadata
+                for (const node of result.nodes) {
+                    const metadata = await this.getMetadata(node.id);
+                    if (metadata && metadata.noun === nounType) {
+                        filteredNodes.push(node);
+                    }
+                }
+                // Update pagination state
+                hasMore = result.hasMore;
+                cursor = result.nextCursor;
+                // Safety check to prevent infinite loops
+                if (!cursor && hasMore) {
+                    console.warn('No cursor returned but hasMore is true, breaking loop');
+                    break;
                 }
             }
             return filteredNodes;
@@ -8422,7 +10056,10 @@ class S3CompatibleStorage extends BaseStorage {
                 const parsedEdge = JSON.parse(bodyContents);
                 console.log(`Parsed edge data for ${id}:`, parsedEdge);
                 // Ensure the parsed edge has the expected properties
-                if (!parsedEdge || !parsedEdge.id || !parsedEdge.vector || !parsedEdge.connections ||
+                if (!parsedEdge ||
+                    !parsedEdge.id ||
+                    !parsedEdge.vector ||
+                    !parsedEdge.connections ||
                     !(parsedEdge.sourceId || parsedEdge.source) ||
                     !(parsedEdge.targetId || parsedEdge.target) ||
                     !(parsedEdge.type || parsedEdge.verb)) {
@@ -8476,86 +10113,205 @@ class S3CompatibleStorage extends BaseStorage {
     }
     /**
      * Get all verbs from storage (internal implementation)
+     * @deprecated This method is deprecated and will be removed in a future version.
+     * It can cause memory issues with large datasets. Use getVerbsWithPagination() instead.
      */
     async getAllVerbs_internal() {
+        console.warn('WARNING: getAllVerbs_internal() is deprecated and will be removed in a future version. Use getVerbsWithPagination() instead.');
         return this.getAllEdges();
     }
     /**
      * Get all edges from storage
+     * @deprecated This method is deprecated and will be removed in a future version.
+     * It can cause memory issues with large datasets. Use getEdgesWithPagination() instead.
      */
     async getAllEdges() {
         await this.ensureInitialized();
+        console.warn('WARNING: getAllEdges() is deprecated and will be removed in a future version. Use getEdgesWithPagination() instead.');
         try {
-            // Import the ListObjectsV2Command and GetObjectCommand only when needed
-            const { ListObjectsV2Command, GetObjectCommand } = await import('@aws-sdk/client-s3');
-            // List all objects in the verbs directory
+            // Use the paginated method with a large limit to maintain backward compatibility
+            // but warn about potential issues
+            const result = await this.getEdgesWithPagination({
+                limit: 1000, // Reasonable limit to avoid memory issues
+                useCache: true
+            });
+            if (result.hasMore) {
+                console.warn(`WARNING: Only returning the first 1000 edges. There are more edges available. Use getEdgesWithPagination() for proper pagination.`);
+            }
+            return result.edges;
+        }
+        catch (error) {
+            console.error('Failed to get all edges:', error);
+            return [];
+        }
+    }
+    /**
+     * Get edges with pagination
+     * @param options Pagination options
+     * @returns Promise that resolves to a paginated result of edges
+     */
+    async getEdgesWithPagination(options = {}) {
+        await this.ensureInitialized();
+        const limit = options.limit || 100;
+        const useCache = options.useCache !== false;
+        const filter = options.filter || {};
+        try {
+            // Import the ListObjectsV2Command only when needed
+            const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
+            // List objects with pagination
             const listResponse = await this.s3Client.send(new ListObjectsV2Command({
                 Bucket: this.bucketName,
-                Prefix: this.verbPrefix
+                Prefix: this.verbPrefix,
+                MaxKeys: limit,
+                ContinuationToken: options.cursor
             }));
+            // If listResponse is null/undefined or there are no objects, return an empty result
+            if (!listResponse ||
+                !listResponse.Contents ||
+                listResponse.Contents.length === 0) {
+                return {
+                    edges: [],
+                    hasMore: false
+                };
+            }
+            // Extract edge IDs from the keys
+            const edgeIds = listResponse.Contents
+                .filter((object) => object && object.Key)
+                .map((object) => object.Key.replace(this.verbPrefix, '').replace('.json', ''));
+            // Use the cache manager to get edges efficiently
             const edges = [];
-            // If there are no objects, return an empty array
-            if (!listResponse.Contents || listResponse.Contents.length === 0) {
-                return edges;
+            if (useCache) {
+                // Get edges from cache manager
+                const cachedEdges = await this.verbCacheManager.getMany(edgeIds);
+                // Add edges to result in the same order as edgeIds
+                for (const id of edgeIds) {
+                    const edge = cachedEdges.get(id);
+                    if (edge) {
+                        // Apply filtering if needed
+                        if (this.filterEdge(edge, filter)) {
+                            edges.push(edge);
+                        }
+                    }
+                }
             }
-            // Get each edge
-            const edgePromises = listResponse.Contents.map(async (object) => {
-                try {
-                    // Extract edge ID from the key (remove prefix and .json extension)
-                    const edgeId = object.Key.replace(this.verbPrefix, '').replace('.json', '');
-                    // Get the edge data
-                    const response = await this.s3Client.send(new GetObjectCommand({
-                        Bucket: this.bucketName,
-                        Key: object.Key
+            else {
+                // Get edges directly from S3 without using cache
+                // Process in smaller batches to reduce memory usage
+                const batchSize = 50;
+                const batches = [];
+                // Split into batches
+                for (let i = 0; i < edgeIds.length; i += batchSize) {
+                    const batch = edgeIds.slice(i, i + batchSize);
+                    batches.push(batch);
+                }
+                // Process each batch sequentially
+                for (const batch of batches) {
+                    const batchEdges = await Promise.all(batch.map(async (id) => {
+                        try {
+                            const edge = await this.getVerb_internal(id);
+                            // Apply filtering if needed
+                            if (edge && this.filterEdge(edge, filter)) {
+                                return edge;
+                            }
+                            return null;
+                        }
+                        catch (error) {
+                            return null;
+                        }
                     }));
-                    // Convert the response body to a string
-                    const bodyContents = await response.Body.transformToString();
-                    const parsedEdge = JSON.parse(bodyContents);
-                    // Convert serialized connections back to Map<number, Set<string>>
-                    const connections = new Map();
-                    for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
-                        connections.set(Number(level), new Set(nodeIds));
+                    // Add non-null edges to result
+                    for (const edge of batchEdges) {
+                        if (edge) {
+                            edges.push(edge);
+                        }
                     }
-                    // Create default timestamp if not present
-                    const defaultTimestamp = {
-                        seconds: Math.floor(Date.now() / 1000),
-                        nanoseconds: (Date.now() % 1000) * 1000000
-                    };
-                    // Create default createdBy if not present
-                    const defaultCreatedBy = {
-                        augmentation: 'unknown',
-                        version: '1.0'
-                    };
-                    return {
-                        id: parsedEdge.id,
-                        vector: parsedEdge.vector,
-                        connections,
-                        sourceId: parsedEdge.sourceId || parsedEdge.source,
-                        targetId: parsedEdge.targetId || parsedEdge.target,
-                        source: parsedEdge.sourceId || parsedEdge.source,
-                        target: parsedEdge.targetId || parsedEdge.target,
-                        verb: parsedEdge.type || parsedEdge.verb,
-                        type: parsedEdge.type || parsedEdge.verb,
-                        weight: parsedEdge.weight || 1.0,
-                        metadata: parsedEdge.metadata || {},
-                        createdAt: parsedEdge.createdAt || defaultTimestamp,
-                        updatedAt: parsedEdge.updatedAt || defaultTimestamp,
-                        createdBy: parsedEdge.createdBy || defaultCreatedBy
-                    };
-                }
-                catch (error) {
-                    console.error(`Error getting edge from ${object.Key}:`, error);
-                    return null;
                 }
-            });
-            // Wait for all promises to resolve and filter out nulls
-            const resolvedEdges = await Promise.all(edgePromises);
-            return resolvedEdges.filter((edge) => edge !== null);
+            }
+            // Determine if there are more edges
+            const hasMore = !!listResponse.IsTruncated;
+            // Set next cursor if there are more edges
+            const nextCursor = listResponse.NextContinuationToken;
+            return {
+                edges,
+                hasMore,
+                nextCursor
+            };
         }
         catch (error) {
-            console.error('Failed to get all edges:', error);
-            return [];
+            console.error('Failed to get edges with pagination:', error);
+            return {
+                edges: [],
+                hasMore: false
+            };
+        }
+    }
+    /**
+     * Filter an edge based on filter criteria
+     * @param edge The edge to filter
+     * @param filter The filter criteria
+     * @returns True if the edge matches the filter, false otherwise
+     */
+    filterEdge(edge, filter) {
+        // If no filter, include all edges
+        if (!filter.sourceId && !filter.targetId && !filter.type) {
+            return true;
+        }
+        // Filter by source ID
+        if (filter.sourceId && edge.sourceId !== filter.sourceId) {
+            return false;
+        }
+        // Filter by target ID
+        if (filter.targetId && edge.targetId !== filter.targetId) {
+            return false;
+        }
+        // Filter by type
+        if (filter.type && edge.type !== filter.type) {
+            return false;
+        }
+        return true;
+    }
+    /**
+     * Get verbs with pagination
+     * @param options Pagination options
+     * @returns Promise that resolves to a paginated result of verbs
+     */
+    async getVerbsWithPagination(options = {}) {
+        await this.ensureInitialized();
+        // Convert filter to edge filter format
+        const edgeFilter = {};
+        if (options.filter) {
+            // Handle sourceId filter
+            if (options.filter.sourceId) {
+                edgeFilter.sourceId = Array.isArray(options.filter.sourceId)
+                    ? options.filter.sourceId[0]
+                    : options.filter.sourceId;
+            }
+            // Handle targetId filter
+            if (options.filter.targetId) {
+                edgeFilter.targetId = Array.isArray(options.filter.targetId)
+                    ? options.filter.targetId[0]
+                    : options.filter.targetId;
+            }
+            // Handle verbType filter
+            if (options.filter.verbType) {
+                edgeFilter.type = Array.isArray(options.filter.verbType)
+                    ? options.filter.verbType[0]
+                    : options.filter.verbType;
+            }
         }
+        // Get edges with pagination
+        const result = await this.getEdgesWithPagination({
+            limit: options.limit,
+            cursor: options.cursor,
+            useCache: true,
+            filter: edgeFilter
+        });
+        // Convert edges to verbs (they're the same in this implementation)
+        return {
+            items: result.edges,
+            hasMore: result.hasMore,
+            nextCursor: result.nextCursor
+        };
     }
     /**
      * Get verbs by source (internal implementation)
@@ -8722,9 +10478,10 @@ class S3CompatibleStorage extends BaseStorage {
                 // In AWS SDK, this would be error.name === 'NoSuchKey'
                 // In our mock, we might get different error types
                 if (error.name === 'NoSuchKey' ||
-                    (error.message && (error.message.includes('NoSuchKey') ||
-                        error.message.includes('not found') ||
-                        error.message.includes('does not exist')))) {
+                    (error.message &&
+                        (error.message.includes('NoSuchKey') ||
+                            error.message.includes('not found') ||
+                            error.message.includes('does not exist')))) {
                     console.log(`Metadata not found for ${id}`);
                     return null;
                 }
@@ -8749,7 +10506,9 @@ class S3CompatibleStorage extends BaseStorage {
                     Prefix: prefix
                 }));
                 // If there are no objects or Contents is undefined, return
-                if (!listResponse || !listResponse.Contents || listResponse.Contents.length === 0) {
+                if (!listResponse ||
+                    !listResponse.Contents ||
+                    listResponse.Contents.length === 0) {
                     return;
                 }
                 // Delete each object
@@ -8799,15 +10558,20 @@ class S3CompatibleStorage extends BaseStorage {
                     Prefix: prefix
                 }));
                 // If there are no objects or Contents is undefined, return
-                if (!listResponse || !listResponse.Contents || listResponse.Contents.length === 0) {
+                if (!listResponse ||
+                    !listResponse.Contents ||
+                    listResponse.Contents.length === 0) {
                     return { size, count };
                 }
                 // Calculate size and count
                 for (const object of listResponse.Contents) {
                     if (object) {
                         // Ensure Size is a number
-                        const objectSize = typeof object.Size === 'number' ? object.Size :
-                            (object.Size ? parseInt(object.Size.toString(), 10) : 0);
+                        const objectSize = typeof object.Size === 'number'
+                            ? object.Size
+                            : object.Size
+                                ? parseInt(object.Size.toString(), 10)
+                                : 0;
                         // Add to total size and increment count
                         size += objectSize || 0;
                         count++;
@@ -8826,12 +10590,17 @@ class S3CompatibleStorage extends BaseStorage {
             const verbsResult = await calculateSizeAndCount(this.verbPrefix);
             const metadataResult = await calculateSizeAndCount(this.metadataPrefix);
             const indexResult = await calculateSizeAndCount(this.indexPrefix);
-            totalSize = nounsResult.size + verbsResult.size + metadataResult.size + indexResult.size;
+            totalSize =
+                nounsResult.size +
+                    verbsResult.size +
+                    metadataResult.size +
+                    indexResult.size;
             nodeCount = nounsResult.count;
             edgeCount = verbsResult.count;
             metadataCount = metadataResult.count;
             // Ensure we have a minimum size if we have objects
-            if (totalSize === 0 && (nodeCount > 0 || edgeCount > 0 || metadataCount > 0)) {
+            if (totalSize === 0 &&
+                (nodeCount > 0 || edgeCount > 0 || metadataCount > 0)) {
                 console.log(`Setting minimum size for ${nodeCount} nodes, ${edgeCount} edges, and ${metadataCount} metadata objects`);
                 totalSize = (nodeCount + edgeCount + metadataCount) * 100; // Arbitrary size per object
             }
@@ -8865,7 +10634,8 @@ class S3CompatibleStorage extends BaseStorage {
                                     const metadata = JSON.parse(bodyContents);
                                     // Count by noun type
                                     if (metadata && metadata.noun) {
-                                        nounTypeCounts[metadata.noun] = (nounTypeCounts[metadata.noun] || 0) + 1;
+                                        nounTypeCounts[metadata.noun] =
+                                            (nounTypeCounts[metadata.noun] || 0) + 1;
                                     }
                                 }
                                 catch (parseError) {
@@ -9046,17 +10816,23 @@ class S3CompatibleStorage extends BaseStorage {
      */
     mergeStatistics(storageStats, localStats) {
         // Merge noun counts by taking the maximum of each type
-        const mergedNounCount = { ...storageStats.nounCount };
+        const mergedNounCount = {
+            ...storageStats.nounCount
+        };
         for (const [type, count] of Object.entries(localStats.nounCount)) {
             mergedNounCount[type] = Math.max(mergedNounCount[type] || 0, count);
         }
         // Merge verb counts by taking the maximum of each type
-        const mergedVerbCount = { ...storageStats.verbCount };
+        const mergedVerbCount = {
+            ...storageStats.verbCount
+        };
         for (const [type, count] of Object.entries(localStats.verbCount)) {
             mergedVerbCount[type] = Math.max(mergedVerbCount[type] || 0, count);
         }
         // Merge metadata counts by taking the maximum of each type
-        const mergedMetadataCount = { ...storageStats.metadataCount };
+        const mergedMetadataCount = {
+            ...storageStats.metadataCount
+        };
         for (const [type, count] of Object.entries(localStats.metadataCount)) {
             mergedMetadataCount[type] = Math.max(mergedMetadataCount[type] || 0, count);
         }
@@ -9169,9 +10945,10 @@ class S3CompatibleStorage extends BaseStorage {
         catch (error) {
             // Check if this is a "NoSuchKey" error (object doesn't exist)
             if (error.name === 'NoSuchKey' ||
-                (error.message && (error.message.includes('NoSuchKey') ||
-                    error.message.includes('not found') ||
-                    error.message.includes('does not exist')))) {
+                (error.message &&
+                    (error.message.includes('NoSuchKey') ||
+                        error.message.includes('not found') ||
+                        error.message.includes('does not exist')))) {
                 return null;
             }
             // For other errors, propagate them
@@ -9200,8 +10977,8 @@ class S3CompatibleStorage extends BaseStorage {
                 Body: JSON.stringify(entryWithInstance),
                 ContentType: 'application/json',
                 Metadata: {
-                    'timestamp': entry.timestamp.toString(),
-                    'operation': entry.operation,
+                    timestamp: entry.timestamp.toString(),
+                    operation: entry.operation,
                     'entity-type': entry.entityType,
                     'entity-id': entry.entityId
                 }
@@ -9371,7 +11148,7 @@ class S3CompatibleStorage extends BaseStorage {
             this.activeLocks.add(lockKey);
             // Schedule automatic cleanup when lock expires
             setTimeout(() => {
-                this.releaseLock(lockKey, lockValue).catch(error => {
+                this.releaseLock(lockKey, lockValue).catch((error) => {
                     console.warn(`Failed to auto-release expired lock ${lockKey}:`, error);
                 });
             }, ttl);
@@ -10302,7 +12079,8 @@ async function createStorage(options = {}) {
                         secretAccessKey: options.s3Storage.secretAccessKey,
                         sessionToken: options.s3Storage.sessionToken,
                         serviceType: 's3',
-                        operationConfig: options.operationConfig
+                        operationConfig: options.operationConfig,
+                        cacheConfig: options.cacheConfig
                     });
                 }
                 else {
@@ -10317,7 +12095,8 @@ async function createStorage(options = {}) {
                         accountId: options.r2Storage.accountId,
                         accessKeyId: options.r2Storage.accessKeyId,
                         secretAccessKey: options.r2Storage.secretAccessKey,
-                        serviceType: 'r2'
+                        serviceType: 'r2',
+                        cacheConfig: options.cacheConfig
                     });
                 }
                 else {
@@ -10333,7 +12112,8 @@ async function createStorage(options = {}) {
                         endpoint: options.gcsStorage.endpoint || 'https://storage.googleapis.com',
                         accessKeyId: options.gcsStorage.accessKeyId,
                         secretAccessKey: options.gcsStorage.secretAccessKey,
-                        serviceType: 'gcs'
+                        serviceType: 'gcs',
+                        cacheConfig: options.cacheConfig
                     });
                 }
                 else {
@@ -10354,7 +12134,8 @@ async function createStorage(options = {}) {
             endpoint: options.customS3Storage.endpoint,
             accessKeyId: options.customS3Storage.accessKeyId,
             secretAccessKey: options.customS3Storage.secretAccessKey,
-            serviceType: options.customS3Storage.serviceType || 'custom'
+            serviceType: options.customS3Storage.serviceType || 'custom',
+            cacheConfig: options.cacheConfig
         });
     }
     // If R2 storage is specified, use it
@@ -10365,7 +12146,8 @@ async function createStorage(options = {}) {
             accountId: options.r2Storage.accountId,
             accessKeyId: options.r2Storage.accessKeyId,
             secretAccessKey: options.r2Storage.secretAccessKey,
-            serviceType: 'r2'
+            serviceType: 'r2',
+            cacheConfig: options.cacheConfig
         });
     }
     // If S3 storage is specified, use it
@@ -10377,7 +12159,8 @@ async function createStorage(options = {}) {
             accessKeyId: options.s3Storage.accessKeyId,
             secretAccessKey: options.s3Storage.secretAccessKey,
             sessionToken: options.s3Storage.sessionToken,
-            serviceType: 's3'
+            serviceType: 's3',
+            cacheConfig: options.cacheConfig
         });
     }
     // If GCS storage is specified, use it
@@ -10389,7 +12172,8 @@ async function createStorage(options = {}) {
             endpoint: options.gcsStorage.endpoint || 'https://storage.googleapis.com',
             accessKeyId: options.gcsStorage.accessKeyId,
             secretAccessKey: options.gcsStorage.secretAccessKey,
-            serviceType: 'gcs'
+            serviceType: 'gcs',
+            cacheConfig: options.cacheConfig
         });
     }
     // Auto-detect the best storage adapter based on the environment
@@ -12893,7 +14677,12 @@ class BrainyData {
         // Set distance function
         this.distanceFunction = config.distanceFunction || cosineDistance$1;
         // Always use the optimized HNSW index implementation
-        this.index = new HNSWIndexOptimized(config.hnsw || {}, this.distanceFunction, config.storageAdapter || null);
+        // Configure HNSW with disk-based storage when a storage adapter is provided
+        const hnswConfig = config.hnsw || {};
+        if (config.storageAdapter) {
+            hnswConfig.useDiskBasedIndex = true;
+        }
+        this.index = new HNSWIndexOptimized(hnswConfig, this.distanceFunction, config.storageAdapter || null);
         this.useOptimizedIndex = true;
         // Set storage if provided, otherwise it will be initialized in init()
         this.storage = config.storageAdapter || null;
@@ -12918,6 +14707,8 @@ class BrainyData {
             config.storage?.requestPersistentStorage || false;
         // Set read-only flag
         this.readOnly = config.readOnly || false;
+        // Set lazy loading in read-only mode flag
+        this.lazyLoadInReadOnlyMode = config.lazyLoadInReadOnlyMode || false;
         // Set write-only flag
         this.writeOnly = config.writeOnly || false;
         // Validate that readOnly and writeOnly are not both true
@@ -12940,6 +14731,27 @@ class BrainyData {
                 ...config.realtimeUpdates
             };
         }
+        // Initialize cache configuration with intelligent defaults
+        // These defaults are automatically tuned based on environment and dataset size
+        this.cacheConfig = {
+            // Enable auto-tuning by default for optimal performance
+            autoTune: true,
+            // Set auto-tune interval to 1 minute for faster initial optimization
+            // This is especially important for large datasets
+            autoTuneInterval: 60000, // 1 minute
+            // Read-only mode specific optimizations
+            readOnlyMode: {
+                // Use aggressive prefetching in read-only mode for better performance
+                prefetchStrategy: 'aggressive'
+            }
+        };
+        // Override defaults with user-provided configuration if available
+        if (config.cache) {
+            this.cacheConfig = {
+                ...this.cacheConfig,
+                ...config.cache
+            };
+        }
     }
     /**
      * Check if the database is in read-only mode and throw an error if it is
@@ -13232,6 +15044,18 @@ class BrainyData {
             return 'default';
         }
     }
+    /**
+     * Get the service name from options or fallback to current augmentation
+     * This provides a consistent way to handle service names across all methods
+     * @param options Options object that may contain a service property
+     * @returns The service name to use for operations
+     */
+    getServiceName(options) {
+        if (options?.service) {
+            return options.service;
+        }
+        return this.getCurrentAugmentation();
+    }
     /**
      * Initialize the database
      * Loads existing data from storage if available
@@ -13284,6 +15108,14 @@ class BrainyData {
                     ...this.storageConfig,
                     requestPersistentStorage: this.requestPersistentStorage
                 };
+                // Add cache configuration if provided
+                if (this.cacheConfig) {
+                    storageOptions.cacheConfig = {
+                        ...this.cacheConfig,
+                        // Pass read-only flag to optimize cache behavior
+                        readOnly: this.readOnly
+                    };
+                }
                 // Ensure s3Storage has all required fields if it's provided
                 if (storageOptions.s3Storage) {
                     // Only include s3Storage if all required fields are present
@@ -13314,6 +15146,14 @@ class BrainyData {
                     console.log('Database is in write-only mode, skipping index loading');
                 }
             }
+            else if (this.readOnly && this.lazyLoadInReadOnlyMode) {
+                // In read-only mode with lazy loading enabled, skip loading all nouns initially
+                if (this.loggingConfig?.verbose) {
+                    console.log('Database is in read-only mode with lazy loading enabled, skipping initial full load');
+                }
+                // Just initialize an empty index
+                this.index.clear();
+            }
             else {
                 // Load all nouns from storage
                 const nouns = await this.storage.getAllNouns();
@@ -13413,7 +15253,33 @@ class BrainyData {
             else {
                 // Input needs to be vectorized
                 try {
-                    vector = await this.embeddingFunction(vectorOrData);
+                    // Check if input is a JSON object and process it specially
+                    if (typeof vectorOrData === 'object' &&
+                        vectorOrData !== null &&
+                        !Array.isArray(vectorOrData)) {
+                        // Process JSON object for better vectorization
+                        const preparedText = prepareJsonForVectorization(vectorOrData, {
+                            // Prioritize common name/title fields if they exist
+                            priorityFields: [
+                                'name',
+                                'title',
+                                'company',
+                                'organization',
+                                'description',
+                                'summary'
+                            ]
+                        });
+                        vector = await this.embeddingFunction(preparedText);
+                        // Track field names for this JSON document
+                        const service = this.getServiceName(options);
+                        if (this.storage) {
+                            await this.storage.trackFieldNames(vectorOrData, service);
+                        }
+                    }
+                    else {
+                        // Use standard embedding for non-JSON data
+                        vector = await this.embeddingFunction(vectorOrData);
+                    }
                 }
                 catch (embedError) {
                     throw new Error(`Failed to vectorize data: ${embedError}`);
@@ -13442,7 +15308,7 @@ class BrainyData {
             // Save noun to storage
             await this.storage.saveNoun(noun);
             // Track noun statistics
-            const service = options.service || this.getCurrentAugmentation();
+            const service = this.getServiceName(options);
             await this.storage.incrementStatistic('noun', service);
             // Save metadata if provided and not empty
             if (metadata !== undefined) {
@@ -13495,7 +15361,7 @@ class BrainyData {
                     }
                     await this.storage.saveMetadata(id, metadataToSave);
                     // Track metadata statistics
-                    const metadataService = options.service || this.getCurrentAugmentation();
+                    const metadataService = this.getServiceName(options);
                     await this.storage.incrementStatistic('metadata', metadataService);
                 }
             }
@@ -13734,6 +15600,35 @@ class BrainyData {
             }
             // If no noun types specified, search all nouns
             if (!nounTypes || nounTypes.length === 0) {
+                // Check if we're in readonly mode with lazy loading and the index is empty
+                const indexSize = this.index.getNouns().size;
+                if (this.readOnly && this.lazyLoadInReadOnlyMode && indexSize === 0) {
+                    if (this.loggingConfig?.verbose) {
+                        console.log('Lazy loading mode: Index is empty, loading nodes for search...');
+                    }
+                    // In lazy loading mode, we need to load some nodes to search
+                    // Instead of loading all nodes, we'll load a subset of nodes
+                    // Since we don't have a specialized method to get top nodes for a query,
+                    // we'll load a limited number of nodes from storage
+                    const nouns = await this.storage.getAllNouns();
+                    const limitedNouns = nouns.slice(0, Math.min(nouns.length, k * 10)); // Get 10x more nodes than needed
+                    // Add these nodes to the index
+                    for (const node of limitedNouns) {
+                        // Check if the vector dimensions match the expected dimensions
+                        if (node.vector.length !== this._dimensions) {
+                            console.warn(`Skipping node ${node.id} due to dimension mismatch: expected ${this._dimensions}, got ${node.vector.length}`);
+                            continue;
+                        }
+                        // Add to index
+                        await this.index.addItem({
+                            id: node.id,
+                            vector: node.vector
+                        });
+                    }
+                    if (this.loggingConfig?.verbose) {
+                        console.log(`Lazy loading mode: Added ${limitedNouns.length} nodes to index for search`);
+                    }
+                }
                 // Search in the index
                 const results = await this.index.search(queryVector, k);
                 // Get metadata for each result
@@ -13887,12 +15782,43 @@ class BrainyData {
         }
         // Check if database is in write-only mode
         this.checkWriteOnly();
-        // If input is a string and not a vector, automatically vectorize it
+        // Process the query input for vectorization
         let queryToUse = queryVectorOrData;
+        // Handle string queries
         if (typeof queryVectorOrData === 'string' && !options.forceEmbed) {
             queryToUse = await this.embed(queryVectorOrData);
             options.forceEmbed = false; // Already embedded, don't force again
         }
+        // Handle JSON object queries with special processing
+        else if (typeof queryVectorOrData === 'object' &&
+            queryVectorOrData !== null &&
+            !Array.isArray(queryVectorOrData) &&
+            !options.forceEmbed) {
+            // If searching within a specific field
+            if (options.searchField) {
+                // Extract text from the specific field
+                const fieldText = extractFieldFromJson(queryVectorOrData, options.searchField);
+                if (fieldText) {
+                    queryToUse = await this.embeddingFunction(fieldText);
+                    options.forceEmbed = false; // Already embedded, don't force again
+                }
+            }
+            // Otherwise process the entire object with priority fields
+            else {
+                const preparedText = prepareJsonForVectorization(queryVectorOrData, {
+                    priorityFields: options.priorityFields || [
+                        'name',
+                        'title',
+                        'company',
+                        'organization',
+                        'description',
+                        'summary'
+                    ]
+                });
+                queryToUse = await this.embeddingFunction(preparedText);
+                options.forceEmbed = false; // Already embedded, don't force again
+            }
+        }
         // If noun types are specified, use searchByNounTypes
         let searchResults;
         if (options.nounTypes && options.nounTypes.length > 0) {
@@ -14094,13 +16020,17 @@ class BrainyData {
                         return false;
                     // Filter by noun type
                     if (filter.nounType) {
-                        const nounTypes = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
+                        const nounTypes = Array.isArray(filter.nounType)
+                            ? filter.nounType
+                            : [filter.nounType];
                         if (!nounTypes.includes(metadata.noun))
                             return false;
                     }
                     // Filter by service
                     if (filter.service && metadata.service) {
-                        const services = Array.isArray(filter.service) ? filter.service : [filter.service];
+                        const services = Array.isArray(filter.service)
+                            ? filter.service
+                            : [filter.service];
                         if (!services.includes(metadata.service))
                             return false;
                     }
@@ -14185,7 +16115,7 @@ class BrainyData {
             // Remove from storage
             await this.storage.deleteNoun(actualId);
             // Track deletion statistics
-            const service = options.service || 'default';
+            const service = this.getServiceName(options);
             await this.storage.decrementStatistic('noun', service);
             // Try to remove metadata (ignore errors)
             try {
@@ -14512,7 +16442,7 @@ class BrainyData {
             // Save verb to storage
             await this.storage.saveVerb(verb);
             // Track verb statistics
-            const serviceForStats = options.service || 'default';
+            const serviceForStats = this.getServiceName(options);
             await this.storage.incrementStatistic('verb', serviceForStats);
             // Update HNSW index size (excluding verbs)
             await this.storage.updateHnswIndexSize(await this.getNounCount());
@@ -14660,7 +16590,7 @@ class BrainyData {
             // Remove from storage
             await this.storage.deleteVerb(id);
             // Track deletion statistics
-            const service = options.service || 'default';
+            const service = this.getServiceName(options);
             await this.storage.decrementStatistic('verb', service);
             return true;
         }
@@ -15588,14 +17518,19 @@ class BrainyData {
                     console.log('Reconstructing HNSW index from backup data...');
                     // Create a new index with the restored configuration
                     // Always use the optimized implementation for consistency
-                    this.index = new HNSWIndexOptimized(data.hnswIndex.config, this.distanceFunction, this.storage);
+                    // Configure HNSW with disk-based storage when a storage adapter is provided
+                    const hnswConfig = data.hnswIndex.config || {};
+                    if (this.storage) {
+                        hnswConfig.useDiskBasedIndex = true;
+                    }
+                    this.index = new HNSWIndexOptimized(hnswConfig, this.distanceFunction, this.storage);
                     this.useOptimizedIndex = true;
                     // For the storage-adapter-coverage test, we want the index to be empty
                     // after restoration, as specified in the test expectation
                     // This is a special case for the test, in a real application we would
                     // re-add all nouns to the index
                     const isTestEnvironment = "production" === 'test' || process.env.VITEST;
-                    const isStorageTest = data.nouns.some(noun => noun.metadata &&
+                    const isStorageTest = data.nouns.some((noun) => noun.metadata &&
                         typeof noun.metadata === 'object' &&
                         'text' in noun.metadata &&
                         typeof noun.metadata.text === 'string' &&
@@ -15742,6 +17677,82 @@ class BrainyData {
             throw new Error(`Failed to generate random graph: ${error}`);
         }
     }
+    /**
+     * Get available field names by service
+     * This helps users understand what fields are available for searching from different data sources
+     * @returns Record of field names by service
+     */
+    async getAvailableFieldNames() {
+        await this.ensureInitialized();
+        if (!this.storage) {
+            return {};
+        }
+        return this.storage.getAvailableFieldNames();
+    }
+    /**
+     * Get standard field mappings
+     * This helps users understand how fields from different services map to standard field names
+     * @returns Record of standard field mappings
+     */
+    async getStandardFieldMappings() {
+        await this.ensureInitialized();
+        if (!this.storage) {
+            return {};
+        }
+        return this.storage.getStandardFieldMappings();
+    }
+    /**
+     * Search using a standard field name
+     * This allows searching across multiple services using a standardized field name
+     * @param standardField The standard field name to search in
+     * @param searchTerm The term to search for
+     * @param k Number of results to return
+     * @param options Additional search options
+     * @returns Array of search results
+     */
+    async searchByStandardField(standardField, searchTerm, k = 10, options = {}) {
+        await this.ensureInitialized();
+        // Check if database is in write-only mode
+        this.checkWriteOnly();
+        // Get standard field mappings
+        const standardFieldMappings = await this.getStandardFieldMappings();
+        // If the standard field doesn't exist, return empty results
+        if (!standardFieldMappings[standardField]) {
+            return [];
+        }
+        // Filter by services if specified
+        let serviceFieldMappings = standardFieldMappings[standardField];
+        if (options.services && options.services.length > 0) {
+            const filteredMappings = {};
+            for (const service of options.services) {
+                if (serviceFieldMappings[service]) {
+                    filteredMappings[service] = serviceFieldMappings[service];
+                }
+            }
+            serviceFieldMappings = filteredMappings;
+        }
+        // If no mappings after filtering, return empty results
+        if (Object.keys(serviceFieldMappings).length === 0) {
+            return [];
+        }
+        // Search in each service's fields and combine results
+        const allResults = [];
+        for (const [service, fieldNames] of Object.entries(serviceFieldMappings)) {
+            for (const fieldName of fieldNames) {
+                // Search using the specific field name for this service
+                const results = await this.search(searchTerm, k, {
+                    searchField: fieldName,
+                    service,
+                    includeVerbs: options.includeVerbs,
+                    searchMode: options.searchMode
+                });
+                // Add results to the combined list
+                allResults.push(...results);
+            }
+        }
+        // Sort by score and limit to k results
+        return allResults.sort((a, b) => b.score - a.score).slice(0, k);
+    }
 }
 
 /**