@soulcraft/brainy 0.39.0 → 0.41.0

package/dist/unified.js

@@ -4939,7 +4939,8 @@ class HNSWIndex {
         const noun = {
             id,
             vector,
-            connections: new Map()
+            connections: new Map(),
+            level: nounLevel
         };
         // Initialize empty connection sets for each level
         for (let level = 0; level <= nounLevel; level++) {
@@ -5264,6 +5265,29 @@ class HNSWIndex {
     getConfig() {
         return { ...this.config };
     }
+    /**
+     * Get index health metrics
+     */
+    getIndexHealth() {
+        let totalConnections = 0;
+        const layerCounts = new Array(this.maxLevel + 1).fill(0);
+        // Count connections and layer distribution
+        this.nouns.forEach(noun => {
+            // Count connections at each layer
+            for (let level = 0; level <= noun.level; level++) {
+                totalConnections += noun.connections.get(level)?.size || 0;
+                layerCounts[level]++;
+            }
+        });
+        const totalNodes = this.nouns.size;
+        const averageConnections = totalNodes > 0 ? totalConnections / totalNodes : 0;
+        return {
+            averageConnections,
+            layerDistribution: layerCounts,
+            maxLayer: this.maxLevel,
+            totalNodes
+        };
+    }
     /**
      * Search within a specific layer
      * Returns a map of noun IDs to distances, sorted by distance
@@ -5721,7 +5745,8 @@ class HNSWIndexOptimized extends HNSWIndex {
             const noun = {
                 id,
                 vector,
-                connections: new Map()
+                connections: new Map(),
+                level: 0
             };
             // Store the noun
             this.storage.saveNoun(noun).catch((error) => {
@@ -6253,6 +6278,7 @@ class BaseStorage extends BaseStorageAdapter {
     constructor() {
         super(...arguments);
         this.isInitialized = false;
+        this.readOnly = false;
     }
     /**
      * Ensure the storage adapter is initialized
@@ -6869,7 +6895,8 @@ class MemoryStorage extends BaseStorage {
         const nounCopy = {
             id: noun.id,
             vector: [...noun.vector],
-            connections: new Map()
+            connections: new Map(),
+            level: noun.level || 0
         };
         // Copy connections
         for (const [level, connections] of noun.connections.entries()) {
@@ -6892,7 +6919,8 @@ class MemoryStorage extends BaseStorage {
         const nounCopy = {
             id: noun.id,
             vector: [...noun.vector],
-            connections: new Map()
+            connections: new Map(),
+            level: noun.level || 0
         };
         // Copy connections
         for (const [level, connections] of noun.connections.entries()) {
@@ -6911,7 +6939,8 @@ class MemoryStorage extends BaseStorage {
             const nounCopy = {
                 id: noun.id,
                 vector: [...noun.vector],
-                connections: new Map()
+                connections: new Map(),
+                level: noun.level || 0
             };
             // Copy connections
             for (const [level, connections] of noun.connections.entries()) {
@@ -6986,7 +7015,8 @@ class MemoryStorage extends BaseStorage {
             const nounCopy = {
                 id: noun.id,
                 vector: [...noun.vector],
-                connections: new Map()
+                connections: new Map(),
+                level: noun.level || 0
             };
             // Copy connections
             for (const [level, connections] of noun.connections.entries()) {
@@ -7294,6 +7324,9 @@ class MemoryStorage extends BaseStorage {
         this.nounMetadata.clear();
         this.verbMetadata.clear();
         this.statistics = null;
+        // Clear the statistics cache
+        this.statisticsCache = null;
+        this.statisticsModified = false;
     }
     /**
      * Get information about storage usage and capacity
@@ -7527,7 +7560,8 @@ class OPFSStorage extends BaseStorage {
             return {
                 id: data.id,
                 vector: data.vector,
-                connections
+                connections,
+                level: data.level || 0
             };
         }
         catch (error) {
@@ -7558,7 +7592,8 @@ class OPFSStorage extends BaseStorage {
                         allNouns.push({
                             id: data.id,
                             vector: data.vector,
-                            connections
+                            connections,
+                            level: data.level || 0
                         });
                     }
                     catch (error) {
@@ -7609,7 +7644,8 @@ class OPFSStorage extends BaseStorage {
                             nodes.push({
                                 id: data.id,
                                 vector: data.vector,
-                                connections
+                                connections,
+                                level: data.level || 0
                             });
                         }
                     }
@@ -7979,6 +8015,9 @@ class OPFSStorage extends BaseStorage {
             await removeDirectoryContents(this.verbMetadataDir);
             // Remove all files in the index directory
             await removeDirectoryContents(this.indexDir);
+            // Clear the statistics cache
+            this.statisticsCache = null;
+            this.statisticsModified = false;
         }
         catch (error) {
             console.error('Error clearing storage:', error);
@@ -10001,6 +10040,135 @@ class CacheManager {
     }
 }
 
+/**
+ * Centralized logging utility for Brainy
+ * Provides configurable log levels and consistent logging across the codebase
+ */
+var LogLevel;
+(function (LogLevel) {
+    LogLevel[LogLevel["ERROR"] = 0] = "ERROR";
+    LogLevel[LogLevel["WARN"] = 1] = "WARN";
+    LogLevel[LogLevel["INFO"] = 2] = "INFO";
+    LogLevel[LogLevel["DEBUG"] = 3] = "DEBUG";
+    LogLevel[LogLevel["TRACE"] = 4] = "TRACE";
+})(LogLevel || (LogLevel = {}));
+let Logger$1 = class Logger {
+    constructor() {
+        this.config = {
+            level: LogLevel.WARN, // Default to WARN - only critical messages
+            timestamps: true,
+            includeModule: true
+        };
+        // Set log level from environment variable if available
+        const envLogLevel = process.env.BRAINY_LOG_LEVEL;
+        if (envLogLevel) {
+            const level = LogLevel[envLogLevel.toUpperCase()];
+            if (level !== undefined) {
+                this.config.level = level;
+            }
+        }
+        // Parse module-specific log levels
+        const moduleLogLevels = process.env.BRAINY_MODULE_LOG_LEVELS;
+        if (moduleLogLevels) {
+            try {
+                this.config.modules = JSON.parse(moduleLogLevels);
+            }
+            catch (e) {
+                // Ignore parsing errors
+            }
+        }
+    }
+    static getInstance() {
+        if (!Logger.instance) {
+            Logger.instance = new Logger();
+        }
+        return Logger.instance;
+    }
+    configure(config) {
+        this.config = { ...this.config, ...config };
+    }
+    shouldLog(level, module) {
+        // Check module-specific level first
+        if (this.config.modules && this.config.modules[module] !== undefined) {
+            return level <= this.config.modules[module];
+        }
+        // Otherwise use global level
+        return level <= this.config.level;
+    }
+    formatMessage(level, module, message) {
+        const parts = [];
+        if (this.config.timestamps) {
+            parts.push(`[${new Date().toISOString()}]`);
+        }
+        parts.push(`[${LogLevel[level]}]`);
+        if (this.config.includeModule) {
+            parts.push(`[${module}]`);
+        }
+        parts.push(message);
+        return parts.join(' ');
+    }
+    log(level, module, message, ...args) {
+        if (!this.shouldLog(level, module)) {
+            return;
+        }
+        if (this.config.handler) {
+            this.config.handler(level, module, message, ...args);
+            return;
+        }
+        const formattedMessage = this.formatMessage(level, module, message);
+        switch (level) {
+            case LogLevel.ERROR:
+                console.error(formattedMessage, ...args);
+                break;
+            case LogLevel.WARN:
+                console.warn(formattedMessage, ...args);
+                break;
+            case LogLevel.INFO:
+                console.info(formattedMessage, ...args);
+                break;
+            case LogLevel.DEBUG:
+            case LogLevel.TRACE:
+                console.log(formattedMessage, ...args);
+                break;
+        }
+    }
+    error(module, message, ...args) {
+        this.log(LogLevel.ERROR, module, message, ...args);
+    }
+    warn(module, message, ...args) {
+        this.log(LogLevel.WARN, module, message, ...args);
+    }
+    info(module, message, ...args) {
+        this.log(LogLevel.INFO, module, message, ...args);
+    }
+    debug(module, message, ...args) {
+        this.log(LogLevel.DEBUG, module, message, ...args);
+    }
+    trace(module, message, ...args) {
+        this.log(LogLevel.TRACE, module, message, ...args);
+    }
+    // Create a module-specific logger
+    createModuleLogger(module) {
+        return {
+            error: (message, ...args) => this.error(module, message, ...args),
+            warn: (message, ...args) => this.warn(module, message, ...args),
+            info: (message, ...args) => this.info(module, message, ...args),
+            debug: (message, ...args) => this.debug(module, message, ...args),
+            trace: (message, ...args) => this.trace(module, message, ...args)
+        };
+    }
+};
+// Export singleton instance
+const logger = Logger$1.getInstance();
+// Export convenience function for creating module loggers
+function createModuleLogger(module) {
+    return logger.createModuleLogger(module);
+}
+// Export function to configure logger
+function configureLogger(config) {
+    logger.configure(config);
+}
+
 /**
  * S3-Compatible Storage Adapter
  * Uses the AWS S3 client to interact with S3-compatible storage services
@@ -10043,6 +10211,8 @@ class S3CompatibleStorage extends BaseStorage {
         this.activeLocks = new Set();
         // Change log for efficient synchronization
         this.changeLogPrefix = 'change-log/';
+        // Module logger
+        this.logger = createModuleLogger('S3Storage');
         // Node cache to avoid redundant API calls
         this.nodeCache = new Map();
         // Batch update timer ID
@@ -10063,6 +10233,7 @@ class S3CompatibleStorage extends BaseStorage {
         this.secretAccessKey = options.secretAccessKey;
         this.sessionToken = options.sessionToken;
         this.serviceType = options.serviceType || 's3';
+        this.readOnly = options.readOnly || false;
         // Initialize operation executors with timeout and retry configuration
         this.operationExecutors = new StorageOperationExecutors(options.operationConfig);
         // Set up prefixes for different types of data
@@ -10184,9 +10355,10 @@ class S3CompatibleStorage extends BaseStorage {
             this.nounCacheManager.setStorageAdapters(nounStorageAdapter, nounStorageAdapter);
             this.verbCacheManager.setStorageAdapters(verbStorageAdapter, verbStorageAdapter);
             this.isInitialized = true;
+            this.logger.info(`Initialized ${this.serviceType} storage with bucket ${this.bucketName}`);
         }
         catch (error) {
-            console.error(`Failed to initialize ${this.serviceType} storage:`, error);
+            this.logger.error(`Failed to initialize ${this.serviceType} storage:`, error);
             throw new Error(`Failed to initialize ${this.serviceType} storage: ${error}`);
         }
     }
@@ -10202,7 +10374,7 @@ class S3CompatibleStorage extends BaseStorage {
     async saveNode(node) {
         await this.ensureInitialized();
         try {
-            console.log(`Saving node ${node.id} to bucket ${this.bucketName}`);
+            this.logger.trace(`Saving node ${node.id}`);
             // Convert connections Map to a serializable format
             const serializableNode = {
                 ...node,
@@ -10212,8 +10384,7 @@ class S3CompatibleStorage extends BaseStorage {
             const { PutObjectCommand } = await import('@aws-sdk/client-s3');
             const key = `${this.nounPrefix}${node.id}.json`;
             const body = JSON.stringify(serializableNode, null, 2);
-            console.log(`Saving node to key: ${key}`);
-            console.log(`Node data: ${body.substring(0, 100)}${body.length > 100 ? '...' : ''}`);
+            this.logger.trace(`Saving to key: ${key}`);
             // Save the node to S3-compatible storage
             const result = await this.s3Client.send(new PutObjectCommand({
                 Bucket: this.bucketName,
@@ -10221,7 +10392,7 @@ class S3CompatibleStorage extends BaseStorage {
                 Body: body,
                 ContentType: 'application/json'
             }));
-            console.log(`Node ${node.id} saved successfully:`, result);
+            this.logger.debug(`Node ${node.id} saved successfully`);
             // Log the change for efficient synchronization
             await this.appendToChangeLog({
                 timestamp: Date.now(),
@@ -10241,18 +10412,18 @@ class S3CompatibleStorage extends BaseStorage {
                     Key: key
                 }));
                 if (verifyResponse && verifyResponse.Body) {
-                    console.log(`Verified node ${node.id} was saved correctly`);
+                    this.logger.trace(`Verified node ${node.id} was saved correctly`);
                 }
                 else {
-                    console.error(`Failed to verify node ${node.id} was saved correctly: no response or body`);
+                    this.logger.warn(`Failed to verify node ${node.id} was saved correctly: no response or body`);
                 }
             }
             catch (verifyError) {
-                console.error(`Failed to verify node ${node.id} was saved correctly:`, verifyError);
+                this.logger.warn(`Failed to verify node ${node.id} was saved correctly:`, verifyError);
             }
         }
         catch (error) {
-            console.error(`Failed to save node ${node.id}:`, error);
+            this.logger.error(`Failed to save node ${node.id}:`, error);
             throw new Error(`Failed to save node ${node.id}: ${error}`);
         }
     }
@@ -10270,9 +10441,8 @@ class S3CompatibleStorage extends BaseStorage {
         try {
             // Import the GetObjectCommand only when needed
             const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            console.log(`Getting node ${id} from bucket ${this.bucketName}`);
             const key = `${this.nounPrefix}${id}.json`;
-            console.log(`Looking for node at key: ${key}`);
+            this.logger.trace(`Getting node ${id} from key: ${key}`);
             // Try to get the node from the nouns directory
             const response = await this.s3Client.send(new GetObjectCommand({
                 Bucket: this.bucketName,
@@ -10280,22 +10450,22 @@ class S3CompatibleStorage extends BaseStorage {
             }));
             // Check if response is null or undefined
             if (!response || !response.Body) {
-                console.log(`No node found for ${id}`);
+                this.logger.trace(`No node found for ${id}`);
                 return null;
             }
             // Convert the response body to a string
             const bodyContents = await response.Body.transformToString();
-            console.log(`Retrieved node body: ${bodyContents.substring(0, 100)}${bodyContents.length > 100 ? '...' : ''}`);
+            this.logger.trace(`Retrieved node body for ${id}`);
             // Parse the JSON string
             try {
                 const parsedNode = JSON.parse(bodyContents);
-                console.log(`Parsed node data for ${id}:`, parsedNode);
+                this.logger.trace(`Parsed node data for ${id}`);
                 // Ensure the parsed node has the expected properties
                 if (!parsedNode ||
                     !parsedNode.id ||
                     !parsedNode.vector ||
                     !parsedNode.connections) {
-                    console.error(`Invalid node data for ${id}:`, parsedNode);
+                    this.logger.warn(`Invalid node data for ${id}`);
                     return null;
                 }
                 // Convert serialized connections back to Map<number, Set<string>>
@@ -10306,19 +10476,20 @@ class S3CompatibleStorage extends BaseStorage {
                 const node = {
                     id: parsedNode.id,
                     vector: parsedNode.vector,
-                    connections
+                    connections,
+                    level: parsedNode.level || 0
                 };
-                console.log(`Successfully retrieved node ${id}:`, node);
+                this.logger.trace(`Successfully retrieved node ${id}`);
                 return node;
             }
             catch (parseError) {
-                console.error(`Failed to parse node data for ${id}:`, parseError);
+                this.logger.error(`Failed to parse node data for ${id}:`, parseError);
                 return null;
             }
         }
         catch (error) {
             // Node not found or other error
-            console.log(`Error getting node for ${id}:`, error);
+            this.logger.trace(`Node not found for ${id}`);
             return null;
         }
     }
@@ -10326,7 +10497,12 @@ class S3CompatibleStorage extends BaseStorage {
      * Get all nouns from storage (internal implementation)
      */
     async getAllNouns_internal() {
-        return this.getAllNodes();
+        // Use paginated method to avoid deprecation warning
+        const result = await this.getNodesWithPagination({
+            limit: 1000,
+            useCache: true
+        });
+        return result.nodes;
     }
     /**
      * Get all nodes from storage
@@ -10335,7 +10511,7 @@ class S3CompatibleStorage extends BaseStorage {
      */
     async getAllNodes() {
         await this.ensureInitialized();
-        console.warn('WARNING: getAllNodes() is deprecated and will be removed in a future version. Use getNodesWithPagination() instead.');
+        this.logger.warn('getAllNodes() is deprecated and will be removed in a future version. Use getNodesWithPagination() instead.');
         try {
             // Use the paginated method with a large limit to maintain backward compatibility
             // but warn about potential issues
@@ -10344,12 +10520,12 @@ class S3CompatibleStorage extends BaseStorage {
                 useCache: true
             });
             if (result.hasMore) {
-                console.warn(`WARNING: Only returning the first 1000 nodes. There are more nodes available. Use getNodesWithPagination() for proper pagination.`);
+                this.logger.warn(`Only returning the first 1000 nodes. There are more nodes available. Use getNodesWithPagination() for proper pagination.`);
             }
             return result.nodes;
         }
         catch (error) {
-            console.error('Failed to get all nodes:', error);
+            this.logger.error('Failed to get all nodes:', error);
             return [];
         }
     }
@@ -10437,7 +10613,7 @@ class S3CompatibleStorage extends BaseStorage {
             };
         }
         catch (error) {
-            console.error('Failed to get nodes with pagination:', error);
+            this.logger.error('Failed to get nodes with pagination:', error);
             return {
                 nodes: [],
                 hasMore: false
@@ -10483,14 +10659,14 @@ class S3CompatibleStorage extends BaseStorage {
                 cursor = result.nextCursor;
                 // Safety check to prevent infinite loops
                 if (!cursor && hasMore) {
-                    console.warn('No cursor returned but hasMore is true, breaking loop');
+                    this.logger.warn('No cursor returned but hasMore is true, breaking loop');
                     break;
                 }
             }
             return filteredNodes;
         }
         catch (error) {
-            console.error(`Failed to get nodes by noun type ${nounType}:`, error);
+            this.logger.error(`Failed to get nodes by noun type ${nounType}:`, error);
             return [];
         }
     }
@@ -10522,7 +10698,7 @@ class S3CompatibleStorage extends BaseStorage {
             });
         }
         catch (error) {
-            console.error(`Failed to delete node ${id}:`, error);
+            this.logger.error(`Failed to delete node ${id}:`, error);
             throw new Error(`Failed to delete node ${id}: ${error}`);
         }
     }
@@ -10564,7 +10740,7 @@ class S3CompatibleStorage extends BaseStorage {
             });
         }
         catch (error) {
-            console.error(`Failed to save edge ${edge.id}:`, error);
+            this.logger.error(`Failed to save edge ${edge.id}:`, error);
             throw new Error(`Failed to save edge ${edge.id}: ${error}`);
         }
     }
@@ -10582,9 +10758,8 @@ class S3CompatibleStorage extends BaseStorage {
         try {
             // Import the GetObjectCommand only when needed
             const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            console.log(`Getting edge ${id} from bucket ${this.bucketName}`);
             const key = `${this.verbPrefix}${id}.json`;
-            console.log(`Looking for edge at key: ${key}`);
+            this.logger.trace(`Getting edge ${id} from key: ${key}`);
             // Try to get the edge from the verbs directory
             const response = await this.s3Client.send(new GetObjectCommand({
                 Bucket: this.bucketName,
@@ -10592,22 +10767,22 @@ class S3CompatibleStorage extends BaseStorage {
             }));
             // Check if response is null or undefined
             if (!response || !response.Body) {
-                console.log(`No edge found for ${id}`);
+                this.logger.trace(`No edge found for ${id}`);
                 return null;
             }
             // Convert the response body to a string
             const bodyContents = await response.Body.transformToString();
-            console.log(`Retrieved edge body: ${bodyContents.substring(0, 100)}${bodyContents.length > 100 ? '...' : ''}`);
+            this.logger.trace(`Retrieved edge body for ${id}`);
             // Parse the JSON string
             try {
                 const parsedEdge = JSON.parse(bodyContents);
-                console.log(`Parsed edge data for ${id}:`, parsedEdge);
+                this.logger.trace(`Parsed edge data for ${id}`);
                 // Ensure the parsed edge has the expected properties
                 if (!parsedEdge ||
                     !parsedEdge.id ||
                     !parsedEdge.vector ||
                     !parsedEdge.connections) {
-                    console.error(`Invalid edge data for ${id}:`, parsedEdge);
+                    this.logger.warn(`Invalid edge data for ${id}`);
                     return null;
                 }
                 // Convert serialized connections back to Map<number, Set<string>>
@@ -10620,17 +10795,17 @@ class S3CompatibleStorage extends BaseStorage {
                     vector: parsedEdge.vector,
                     connections
                 };
-                console.log(`Successfully retrieved edge ${id}:`, edge);
+                this.logger.trace(`Successfully retrieved edge ${id}`);
                 return edge;
             }
             catch (parseError) {
-                console.error(`Failed to parse edge data for ${id}:`, parseError);
+                this.logger.error(`Failed to parse edge data for ${id}:`, parseError);
                 return null;
             }
         }
         catch (error) {
             // Edge not found or other error
-            console.log(`Error getting edge for ${id}:`, error);
+            this.logger.trace(`Edge not found for ${id}`);
             return null;
         }
     }
@@ -10640,7 +10815,7 @@ class S3CompatibleStorage extends BaseStorage {
      * It can cause memory issues with large datasets. Use getVerbsWithPagination() instead.
      */
     async getAllVerbs_internal() {
-        console.warn('WARNING: getAllVerbs_internal() is deprecated and will be removed in a future version. Use getVerbsWithPagination() instead.');
+        this.logger.warn('getAllVerbs_internal() is deprecated and will be removed in a future version. Use getVerbsWithPagination() instead.');
         return this.getAllEdges();
     }
     /**
@@ -10650,7 +10825,7 @@ class S3CompatibleStorage extends BaseStorage {
      */
     async getAllEdges() {
         await this.ensureInitialized();
-        console.warn('WARNING: getAllEdges() is deprecated and will be removed in a future version. Use getEdgesWithPagination() instead.');
+        this.logger.warn('getAllEdges() is deprecated and will be removed in a future version. Use getEdgesWithPagination() instead.');
         try {
             // Use the paginated method with a large limit to maintain backward compatibility
             // but warn about potential issues
@@ -10659,12 +10834,12 @@ class S3CompatibleStorage extends BaseStorage {
                 useCache: true
             });
             if (result.hasMore) {
-                console.warn(`WARNING: Only returning the first 1000 edges. There are more edges available. Use getEdgesWithPagination() for proper pagination.`);
+                this.logger.warn(`Only returning the first 1000 edges. There are more edges available. Use getEdgesWithPagination() for proper pagination.`);
             }
             return result.edges;
         }
         catch (error) {
-            console.error('Failed to get all edges:', error);
+            this.logger.error('Failed to get all edges:', error);
             return [];
         }
     }
@@ -10761,7 +10936,7 @@ class S3CompatibleStorage extends BaseStorage {
             };
         }
         catch (error) {
-            console.error('Failed to get edges with pagination:', error);
+            this.logger.error('Failed to get edges with pagination:', error);
             return {
                 edges: [],
                 hasMore: false
@@ -10777,7 +10952,7 @@ class S3CompatibleStorage extends BaseStorage {
     filterEdge(edge, filter) {
         // HNSWVerb filtering is not supported since metadata is stored separately
         // This method is deprecated and should not be used with the new storage pattern
-        console.warn('Edge filtering is deprecated and not supported with the new storage pattern');
+        this.logger.trace('Edge filtering is deprecated and not supported with the new storage pattern');
         return true; // Return all edges since filtering requires metadata
     }
     /**
@@ -10842,7 +11017,7 @@ class S3CompatibleStorage extends BaseStorage {
     async getEdgesBySource(sourceId) {
         // This method is deprecated and would require loading metadata for each edge
         // For now, return empty array since this is not efficiently implementable with new storage pattern
-        console.warn('getEdgesBySource is deprecated and not efficiently supported in new storage pattern');
+        this.logger.trace('getEdgesBySource is deprecated and not efficiently supported in new storage pattern');
         return [];
     }
     /**
@@ -10857,7 +11032,7 @@ class S3CompatibleStorage extends BaseStorage {
     async getEdgesByTarget(targetId) {
         // This method is deprecated and would require loading metadata for each edge
         // For now, return empty array since this is not efficiently implementable with new storage pattern
-        console.warn('getEdgesByTarget is deprecated and not efficiently supported in new storage pattern');
+        this.logger.trace('getEdgesByTarget is deprecated and not efficiently supported in new storage pattern');
         return [];
     }
     /**
@@ -10872,7 +11047,7 @@ class S3CompatibleStorage extends BaseStorage {
     async getEdgesByType(type) {
         // This method is deprecated and would require loading metadata for each edge
         // For now, return empty array since this is not efficiently implementable with new storage pattern
-        console.warn('getEdgesByType is deprecated and not efficiently supported in new storage pattern');
+        this.logger.trace('getEdgesByType is deprecated and not efficiently supported in new storage pattern');
         return [];
     }
     /**
@@ -10903,7 +11078,7 @@ class S3CompatibleStorage extends BaseStorage {
             });
         }
         catch (error) {
-            console.error(`Failed to delete edge ${id}:`, error);
+            this.logger.error(`Failed to delete edge ${id}:`, error);
             throw new Error(`Failed to delete edge ${id}: ${error}`);
         }
     }
@@ -10913,13 +11088,11 @@ class S3CompatibleStorage extends BaseStorage {
     async saveMetadata(id, metadata) {
         await this.ensureInitialized();
         try {
-            console.log(`Saving metadata for ${id} to bucket ${this.bucketName}`);
             // Import the PutObjectCommand only when needed
             const { PutObjectCommand } = await import('@aws-sdk/client-s3');
             const key = `${this.metadataPrefix}${id}.json`;
             const body = JSON.stringify(metadata, null, 2);
-            console.log(`Saving metadata to key: ${key}`);
-            console.log(`Metadata: ${body}`);
+            this.logger.trace(`Saving metadata for ${id} to key: ${key}`);
             // Save the metadata to S3-compatible storage
             const result = await this.s3Client.send(new PutObjectCommand({
                 Bucket: this.bucketName,
@@ -10927,7 +11100,7 @@ class S3CompatibleStorage extends BaseStorage {
                 Body: body,
                 ContentType: 'application/json'
             }));
-            console.log(`Metadata for ${id} saved successfully:`, result);
+            this.logger.debug(`Metadata for ${id} saved successfully`);
             // Log the change for efficient synchronization
             await this.appendToChangeLog({
                 timestamp: Date.now(),
@@ -10944,19 +11117,18 @@ class S3CompatibleStorage extends BaseStorage {
                     Key: key
                 }));
                 if (verifyResponse && verifyResponse.Body) {
-                    const bodyContents = await verifyResponse.Body.transformToString();
-                    console.log(`Verified metadata for ${id} was saved correctly: ${bodyContents}`);
+                    this.logger.trace(`Verified metadata for ${id} was saved correctly`);
                 }
                 else {
-                    console.error(`Failed to verify metadata for ${id} was saved correctly: no response or body`);
+                    this.logger.warn(`Failed to verify metadata for ${id} was saved correctly: no response or body`);
                 }
             }
             catch (verifyError) {
-                console.error(`Failed to verify metadata for ${id} was saved correctly:`, verifyError);
+                this.logger.warn(`Failed to verify metadata for ${id} was saved correctly:`, verifyError);
             }
         }
         catch (error) {
-            console.error(`Failed to save metadata for ${id}:`, error);
+            this.logger.error(`Failed to save metadata for ${id}:`, error);
             throw new Error(`Failed to save metadata for ${id}: ${error}`);
         }
     }
@@ -10966,13 +11138,11 @@ class S3CompatibleStorage extends BaseStorage {
     async saveVerbMetadata(id, metadata) {
         await this.ensureInitialized();
         try {
-            console.log(`Saving verb metadata for ${id} to bucket ${this.bucketName}`);
             // Import the PutObjectCommand only when needed
             const { PutObjectCommand } = await import('@aws-sdk/client-s3');
             const key = `verb-metadata/${id}.json`;
             const body = JSON.stringify(metadata, null, 2);
-            console.log(`Saving verb metadata to key: ${key}`);
-            console.log(`Verb Metadata: ${body}`);
+            this.logger.trace(`Saving verb metadata for ${id} to key: ${key}`);
             // Save the verb metadata to S3-compatible storage
             const result = await this.s3Client.send(new PutObjectCommand({
                 Bucket: this.bucketName,
@@ -10980,10 +11150,10 @@ class S3CompatibleStorage extends BaseStorage {
                 Body: body,
                 ContentType: 'application/json'
             }));
-            console.log(`Verb metadata for ${id} saved successfully:`, result);
+            this.logger.debug(`Verb metadata for ${id} saved successfully`);
         }
         catch (error) {
-            console.error(`Failed to save verb metadata for ${id}:`, error);
+            this.logger.error(`Failed to save verb metadata for ${id}:`, error);
             throw new Error(`Failed to save verb metadata for ${id}: ${error}`);
         }
     }
@@ -10995,9 +11165,8 @@ class S3CompatibleStorage extends BaseStorage {
         try {
             // Import the GetObjectCommand only when needed
             const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            console.log(`Getting verb metadata for ${id} from bucket ${this.bucketName}`);
             const key = `verb-metadata/${id}.json`;
-            console.log(`Looking for verb metadata at key: ${key}`);
+            this.logger.trace(`Getting verb metadata for ${id} from key: ${key}`);
             // Try to get the verb metadata
             const response = await this.s3Client.send(new GetObjectCommand({
                 Bucket: this.bucketName,
@@ -11005,20 +11174,20 @@ class S3CompatibleStorage extends BaseStorage {
             }));
             // Check if response is null or undefined
             if (!response || !response.Body) {
-                console.log(`No verb metadata found for ${id}`);
+                this.logger.trace(`No verb metadata found for ${id}`);
                 return null;
             }
             // Convert the response body to a string
             const bodyContents = await response.Body.transformToString();
-            console.log(`Retrieved verb metadata body: ${bodyContents}`);
+            this.logger.trace(`Retrieved verb metadata body for ${id}`);
             // Parse the JSON string
             try {
                 const parsedMetadata = JSON.parse(bodyContents);
-                console.log(`Successfully retrieved verb metadata for ${id}:`, parsedMetadata);
+                this.logger.trace(`Successfully retrieved verb metadata for ${id}`);
                 return parsedMetadata;
             }
             catch (parseError) {
-                console.error(`Failed to parse verb metadata for ${id}:`, parseError);
+                this.logger.error(`Failed to parse verb metadata for ${id}:`, parseError);
                 return null;
             }
         }
@@ -11029,7 +11198,7 @@ class S3CompatibleStorage extends BaseStorage {
                     (error.message.includes('NoSuchKey') ||
                         error.message.includes('not found') ||
                         error.message.includes('does not exist')))) {
-                console.log(`Verb metadata not found for ${id}`);
+                this.logger.trace(`Verb metadata not found for ${id}`);
                 return null;
             }
             // For other types of errors, convert to BrainyError for better classification
@@ -11042,13 +11211,11 @@ class S3CompatibleStorage extends BaseStorage {
     async saveNounMetadata(id, metadata) {
         await this.ensureInitialized();
         try {
-            console.log(`Saving noun metadata for ${id} to bucket ${this.bucketName}`);
             // Import the PutObjectCommand only when needed
             const { PutObjectCommand } = await import('@aws-sdk/client-s3');
             const key = `noun-metadata/${id}.json`;
             const body = JSON.stringify(metadata, null, 2);
-            console.log(`Saving noun metadata to key: ${key}`);
-            console.log(`Noun Metadata: ${body}`);
+            this.logger.trace(`Saving noun metadata for ${id} to key: ${key}`);
             // Save the noun metadata to S3-compatible storage
             const result = await this.s3Client.send(new PutObjectCommand({
                 Bucket: this.bucketName,
@@ -11056,10 +11223,10 @@ class S3CompatibleStorage extends BaseStorage {
                 Body: body,
                 ContentType: 'application/json'
             }));
-            console.log(`Noun metadata for ${id} saved successfully:`, result);
+            this.logger.debug(`Noun metadata for ${id} saved successfully`);
         }
         catch (error) {
-            console.error(`Failed to save noun metadata for ${id}:`, error);
+            this.logger.error(`Failed to save noun metadata for ${id}:`, error);
             throw new Error(`Failed to save noun metadata for ${id}: ${error}`);
         }
     }
@@ -11071,9 +11238,8 @@ class S3CompatibleStorage extends BaseStorage {
         try {
             // Import the GetObjectCommand only when needed
             const { GetObjectCommand } = await import('@aws-sdk/client-s3');
-            console.log(`Getting noun metadata for ${id} from bucket ${this.bucketName}`);
             const key = `noun-metadata/${id}.json`;
-            console.log(`Looking for noun metadata at key: ${key}`);
+            this.logger.trace(`Getting noun metadata for ${id} from key: ${key}`);
             // Try to get the noun metadata
             const response = await this.s3Client.send(new GetObjectCommand({
                 Bucket: this.bucketName,
@@ -11081,20 +11247,20 @@ class S3CompatibleStorage extends BaseStorage {
             }));
             // Check if response is null or undefined
             if (!response || !response.Body) {
-                console.log(`No noun metadata found for ${id}`);
+                this.logger.trace(`No noun metadata found for ${id}`);
                 return null;
             }
             // Convert the response body to a string
             const bodyContents = await response.Body.transformToString();
-            console.log(`Retrieved noun metadata body: ${bodyContents}`);
+            this.logger.trace(`Retrieved noun metadata body for ${id}`);
             // Parse the JSON string
             try {
                 const parsedMetadata = JSON.parse(bodyContents);
-                console.log(`Successfully retrieved noun metadata for ${id}:`, parsedMetadata);
+                this.logger.trace(`Successfully retrieved noun metadata for ${id}`);
                 return parsedMetadata;
             }
             catch (parseError) {
-                console.error(`Failed to parse noun metadata for ${id}:`, parseError);
+                this.logger.error(`Failed to parse noun metadata for ${id}:`, parseError);
                 return null;
             }
         }
@@ -11105,7 +11271,7 @@ class S3CompatibleStorage extends BaseStorage {
                     (error.message.includes('NoSuchKey') ||
                         error.message.includes('not found') ||
                         error.message.includes('does not exist')))) {
-                console.log(`Noun metadata not found for ${id}`);
+                this.logger.trace(`Noun metadata not found for ${id}`);
                 return null;
             }
             // For other types of errors, convert to BrainyError for better classification
@@ -11204,6 +11370,9 @@ class S3CompatibleStorage extends BaseStorage {
             await deleteObjectsWithPrefix(this.metadataPrefix);
             // Delete all objects in the index directory
             await deleteObjectsWithPrefix(this.indexPrefix);
+            // Clear the statistics cache
+            this.statisticsCache = null;
+            this.statisticsModified = false;
         }
         catch (error) {
             console.error('Failed to clear storage:', error);
@@ -11319,7 +11488,7 @@ class S3CompatibleStorage extends BaseStorage {
                             }
                         }
                         catch (error) {
-                            console.error(`Error getting metadata from ${object.Key}:`, error);
+                            this.logger.warn(`Error getting metadata from ${object.Key}:`, error);
                         }
                     }
                 }
@@ -11340,7 +11509,7 @@ class S3CompatibleStorage extends BaseStorage {
             };
         }
         catch (error) {
-            console.error('Failed to get storage status:', error);
+            this.logger.error('Failed to get storage status:', error);
             return {
                 type: this.serviceType,
                 used: 0,
@@ -11380,6 +11549,11 @@ class S3CompatibleStorage extends BaseStorage {
     scheduleBatchUpdate() {
         // Mark statistics as modified
         this.statisticsModified = true;
+        // If we're in read-only mode, don't update statistics
+        if (this.readOnly) {
+            this.logger.trace('Skipping statistics update in read-only mode');
+            return;
+        }
         // If a timer is already set, don't set another one
         if (this.statisticsBatchUpdateTimerId !== null) {
             return;
@@ -11416,7 +11590,7 @@ class S3CompatibleStorage extends BaseStorage {
         if (!lockAcquired) {
             // Another instance is updating statistics, skip this flush
             // but keep the modified flag so we'll try again later
-            console.log('Statistics flush skipped - another instance is updating');
+            this.logger.debug('Statistics flush skipped - another instance is updating');
             return;
         }
         try {
@@ -11435,7 +11609,7 @@ class S3CompatibleStorage extends BaseStorage {
             }
             catch (error) {
                 // If we can't read current stats, proceed with local cache
-                console.warn('Could not read current statistics from storage, using local cache:', error);
+                this.logger.warn('Could not read current statistics from storage, using local cache:', error);
             }
             // Merge local statistics with storage statistics
             let mergedStats = this.statisticsCache;
@@ -11473,7 +11647,7 @@ class S3CompatibleStorage extends BaseStorage {
             }
         }
         catch (error) {
-            console.error('Failed to flush statistics data:', error);
+            this.logger.error('Failed to flush statistics data:', error);
             // Mark as still modified so we'll try again later
             this.statisticsModified = true;
             // Don't throw the error to avoid disrupting the application
@@ -11538,7 +11712,7 @@ class S3CompatibleStorage extends BaseStorage {
             this.scheduleBatchUpdate();
         }
         catch (error) {
-            console.error('Failed to save statistics data:', error);
+            this.logger.error('Failed to save statistics data:', error);
             throw new Error(`Failed to save statistics data: ${error}`);
         }
     }
@@ -11548,8 +11722,11 @@ class S3CompatibleStorage extends BaseStorage {
      */
     async getStatisticsData() {
         await this.ensureInitialized();
-        // If we have cached statistics, return a deep copy
-        if (this.statisticsCache) {
+        // Always fetch fresh statistics from storage to avoid inconsistencies
+        // Only use cache if explicitly in read-only mode
+        const shouldUseCache = this.readOnly && this.statisticsCache &&
+            (Date.now() - this.lastStatisticsFlushTime < this.MIN_FLUSH_INTERVAL_MS);
+        if (shouldUseCache && this.statisticsCache) {
             return {
                 nounCount: { ...this.statisticsCache.nounCount },
                 verbCount: { ...this.statisticsCache.verbCount },
@@ -11590,7 +11767,7 @@ class S3CompatibleStorage extends BaseStorage {
             return statistics;
         }
         catch (error) {
-            console.error('Error getting statistics data:', error);
+            this.logger.error('Error getting statistics data:', error);
             throw error;
         }
     }
@@ -11660,7 +11837,7 @@ class S3CompatibleStorage extends BaseStorage {
             }));
         }
         catch (error) {
-            console.warn('Failed to append to change log:', error);
+            this.logger.warn('Failed to append to change log:', error);
             // Don't throw error to avoid disrupting main operations
         }
     }
@@ -11705,7 +11882,7 @@ class S3CompatibleStorage extends BaseStorage {
                     }
                 }
                 catch (error) {
-                    console.warn(`Failed to read change log entry ${object.Key}:`, error);
+                    this.logger.warn(`Failed to read change log entry ${object.Key}:`, error);
                     // Continue processing other entries
                 }
             }
@@ -11714,7 +11891,7 @@ class S3CompatibleStorage extends BaseStorage {
             return changes.slice(0, maxEntries);
         }
         catch (error) {
-            console.error('Failed to get changes from change log:', error);
+            this.logger.error('Failed to get changes from change log:', error);
             return [];
         }
     }
@@ -11761,15 +11938,15 @@ class S3CompatibleStorage extends BaseStorage {
                     }));
                 }
                 catch (error) {
-                    console.warn(`Failed to delete old change log entry ${key}:`, error);
+                    this.logger.warn(`Failed to delete old change log entry ${key}:`, error);
                 }
             }
             if (entriesToDelete.length > 0) {
-                console.log(`Cleaned up ${entriesToDelete.length} old change log entries`);
+                this.logger.debug(`Cleaned up ${entriesToDelete.length} old change log entries`);
             }
         }
         catch (error) {
-            console.warn('Failed to cleanup old change logs:', error);
+            this.logger.warn('Failed to cleanup old change logs:', error);
         }
     }
     /**
@@ -11824,13 +12001,13 @@ class S3CompatibleStorage extends BaseStorage {
             // Schedule automatic cleanup when lock expires
             setTimeout(() => {
                 this.releaseLock(lockKey, lockValue).catch((error) => {
-                    console.warn(`Failed to auto-release expired lock ${lockKey}:`, error);
+                    this.logger.warn(`Failed to auto-release expired lock ${lockKey}:`, error);
                 });
             }, ttl);
             return true;
         }
         catch (error) {
-            console.warn(`Failed to acquire lock ${lockKey}:`, error);
+            this.logger.warn(`Failed to acquire lock ${lockKey}:`, error);
             return false;
         }
     }
@@ -11879,7 +12056,7 @@ class S3CompatibleStorage extends BaseStorage {
             this.activeLocks.delete(lockKey);
         }
         catch (error) {
-            console.warn(`Failed to release lock ${lockKey}:`, error);
+            this.logger.warn(`Failed to release lock ${lockKey}:`, error);
         }
     }
     /**
@@ -11930,16 +12107,84 @@ class S3CompatibleStorage extends BaseStorage {
                     }));
                 }
                 catch (error) {
-                    console.warn(`Failed to delete expired lock ${lockKey}:`, error);
+                    this.logger.warn(`Failed to delete expired lock ${lockKey}:`, error);
                 }
             }
             if (expiredLocks.length > 0) {
-                console.log(`Cleaned up ${expiredLocks.length} expired locks`);
+                this.logger.debug(`Cleaned up ${expiredLocks.length} expired locks`);
             }
         }
         catch (error) {
-            console.warn('Failed to cleanup expired locks:', error);
+            this.logger.warn('Failed to cleanup expired locks:', error);
+        }
+    }
+    /**
+     * Get nouns with pagination support
+     * @param options Pagination options
+     * @returns Promise that resolves to a paginated result of nouns
+     */
+    async getNounsWithPagination(options = {}) {
+        await this.ensureInitialized();
+        const limit = options.limit || 100;
+        const cursor = options.cursor;
+        // Get paginated nodes
+        const result = await this.getNodesWithPagination({
+            limit,
+            cursor,
+            useCache: true
+        });
+        // Apply filters if provided
+        let filteredNodes = result.nodes;
+        if (options.filter) {
+            // Filter by noun type
+            if (options.filter.nounType) {
+                const nounTypes = Array.isArray(options.filter.nounType)
+                    ? options.filter.nounType
+                    : [options.filter.nounType];
+                const filteredByType = [];
+                for (const node of filteredNodes) {
+                    const metadata = await this.getNounMetadata(node.id);
+                    if (metadata && nounTypes.includes(metadata.type || metadata.noun)) {
+                        filteredByType.push(node);
+                    }
+                }
+                filteredNodes = filteredByType;
+            }
+            // Filter by service
+            if (options.filter.service) {
+                const services = Array.isArray(options.filter.service)
+                    ? options.filter.service
+                    : [options.filter.service];
+                const filteredByService = [];
+                for (const node of filteredNodes) {
+                    const metadata = await this.getNounMetadata(node.id);
+                    if (metadata && services.includes(metadata.service)) {
+                        filteredByService.push(node);
+                    }
+                }
+                filteredNodes = filteredByService;
+            }
+            // Filter by metadata
+            if (options.filter.metadata) {
+                const metadataFilter = options.filter.metadata;
+                const filteredByMetadata = [];
+                for (const node of filteredNodes) {
+                    const metadata = await this.getNounMetadata(node.id);
+                    if (metadata) {
+                        const matches = Object.entries(metadataFilter).every(([key, value]) => metadata[key] === value);
+                        if (matches) {
+                            filteredByMetadata.push(node);
+                        }
+                    }
+                }
+                filteredNodes = filteredByMetadata;
+            }
         }
+        return {
+            items: filteredNodes,
+            hasMore: result.hasMore,
+            nextCursor: result.nextCursor
+        };
     }
 }
 
@@ -12080,7 +12325,8 @@ class FileSystemStorage extends BaseStorage {
             return {
                 id: parsedNode.id,
                 vector: parsedNode.vector,
-                connections
+                connections,
+                level: parsedNode.level || 0
             };
         }
         catch (error) {
@@ -12111,7 +12357,8 @@ class FileSystemStorage extends BaseStorage {
                     allNodes.push({
                         id: parsedNode.id,
                         vector: parsedNode.vector,
-                        connections
+                        connections,
+                        level: parsedNode.level || 0
                     });
                 }
             }
@@ -12150,7 +12397,8 @@ class FileSystemStorage extends BaseStorage {
                         nouns.push({
                             id: parsedNode.id,
                             vector: parsedNode.vector,
-                            connections
+                            connections,
+                            level: parsedNode.level || 0
                         });
                     }
                 }
@@ -12415,6 +12663,9 @@ class FileSystemStorage extends BaseStorage {
         await removeDirectoryContents(this.verbMetadataDir);
         // Remove all files in the index directory
         await removeDirectoryContents(this.indexDir);
+        // Clear the statistics cache
+        this.statisticsCache = null;
+        this.statisticsModified = false;
     }
     /**
      * Get information about storage usage and capacity
@@ -17110,6 +17361,230 @@ class CacheAutoConfigurator {
     }
 }
 
+/**
+ * Lightweight statistics collector for Brainy
+ * Designed to have minimal performance impact even with millions of entries
+ */
+class StatisticsCollector {
+    constructor() {
+        // Content type tracking (lightweight counters)
+        this.contentTypes = new Map();
+        // Data freshness tracking (only track timestamps, not full data)
+        this.oldestTimestamp = Date.now();
+        this.newestTimestamp = Date.now();
+        this.updateTimestamps = [];
+        // Search performance tracking (rolling window)
+        this.searchMetrics = {
+            totalSearches: 0,
+            totalSearchTimeMs: 0,
+            searchTimestamps: [],
+            topSearchTerms: new Map()
+        };
+        // Verb type tracking
+        this.verbTypes = new Map();
+        // Storage size estimates (updated periodically, not on every operation)
+        this.storageSizeCache = {
+            lastUpdated: 0,
+            sizes: {
+                nouns: 0,
+                verbs: 0,
+                metadata: 0,
+                index: 0
+            }
+        };
+        this.MAX_TIMESTAMPS = 1000; // Keep last 1000 timestamps
+        this.MAX_SEARCH_TERMS = 100; // Track top 100 search terms
+        this.SIZE_UPDATE_INTERVAL = 60000; // Update sizes every minute
+    }
+    /**
+     * Track content type (very lightweight)
+     */
+    trackContentType(type) {
+        this.contentTypes.set(type, (this.contentTypes.get(type) || 0) + 1);
+    }
+    /**
+     * Track data update timestamp (lightweight)
+     */
+    trackUpdate(timestamp) {
+        const ts = timestamp || Date.now();
+        // Update oldest/newest
+        if (ts < this.oldestTimestamp)
+            this.oldestTimestamp = ts;
+        if (ts > this.newestTimestamp)
+            this.newestTimestamp = ts;
+        // Add to rolling window
+        this.updateTimestamps.push({ timestamp: ts, count: 1 });
+        // Keep window size limited
+        if (this.updateTimestamps.length > this.MAX_TIMESTAMPS) {
+            this.updateTimestamps.shift();
+        }
+    }
+    /**
+     * Track search performance (lightweight)
+     */
+    trackSearch(searchTerm, durationMs) {
+        this.searchMetrics.totalSearches++;
+        this.searchMetrics.totalSearchTimeMs += durationMs;
+        // Add to rolling window
+        this.searchMetrics.searchTimestamps.push({
+            timestamp: Date.now(),
+            count: 1
+        });
+        // Keep window size limited
+        if (this.searchMetrics.searchTimestamps.length > this.MAX_TIMESTAMPS) {
+            this.searchMetrics.searchTimestamps.shift();
+        }
+        // Track search term (limit to top N)
+        const termCount = (this.searchMetrics.topSearchTerms.get(searchTerm) || 0) + 1;
+        this.searchMetrics.topSearchTerms.set(searchTerm, termCount);
+        // Prune if too many terms
+        if (this.searchMetrics.topSearchTerms.size > this.MAX_SEARCH_TERMS * 2) {
+            this.pruneSearchTerms();
+        }
+    }
+    /**
+     * Track verb type (lightweight)
+     */
+    trackVerbType(type) {
+        this.verbTypes.set(type, (this.verbTypes.get(type) || 0) + 1);
+    }
+    /**
+     * Update storage size estimates (called periodically, not on every operation)
+     */
+    updateStorageSizes(sizes) {
+        this.storageSizeCache = {
+            lastUpdated: Date.now(),
+            sizes
+        };
+    }
+    /**
+     * Get comprehensive statistics
+     */
+    getStatistics() {
+        const now = Date.now();
+        const hourAgo = now - 3600000;
+        const dayAgo = now - 86400000;
+        const weekAgo = now - 604800000;
+        const monthAgo = now - 2592000000;
+        // Calculate data freshness
+        const updatesLastHour = this.updateTimestamps.filter(t => t.timestamp > hourAgo).length;
+        const updatesLastDay = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length;
+        // Calculate age distribution
+        const ageDistribution = {
+            last24h: 0,
+            last7d: 0,
+            last30d: 0,
+            older: 0
+        };
+        // Estimate based on update patterns (not scanning all data)
+        const totalUpdates = this.updateTimestamps.length;
+        if (totalUpdates > 0) {
+            const recentUpdates = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length;
+            const weekUpdates = this.updateTimestamps.filter(t => t.timestamp > weekAgo).length;
+            const monthUpdates = this.updateTimestamps.filter(t => t.timestamp > monthAgo).length;
+            ageDistribution.last24h = Math.round((recentUpdates / totalUpdates) * 100);
+            ageDistribution.last7d = Math.round(((weekUpdates - recentUpdates) / totalUpdates) * 100);
+            ageDistribution.last30d = Math.round(((monthUpdates - weekUpdates) / totalUpdates) * 100);
+            ageDistribution.older = 100 - ageDistribution.last24h - ageDistribution.last7d - ageDistribution.last30d;
+        }
+        // Calculate search metrics
+        const searchesLastHour = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > hourAgo).length;
+        const searchesLastDay = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > dayAgo).length;
+        const avgSearchTime = this.searchMetrics.totalSearches > 0
+            ? this.searchMetrics.totalSearchTimeMs / this.searchMetrics.totalSearches
+            : 0;
+        // Get top search terms
+        const topSearchTerms = Array.from(this.searchMetrics.topSearchTerms.entries())
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, 10)
+            .map(([term]) => term);
+        // Calculate storage metrics
+        const totalSize = Object.values(this.storageSizeCache.sizes).reduce((a, b) => a + b, 0);
+        return {
+            contentTypes: Object.fromEntries(this.contentTypes),
+            dataFreshness: {
+                oldestEntry: new Date(this.oldestTimestamp).toISOString(),
+                newestEntry: new Date(this.newestTimestamp).toISOString(),
+                updatesLastHour,
+                updatesLastDay,
+                ageDistribution
+            },
+            storageMetrics: {
+                totalSizeBytes: totalSize,
+                nounsSizeBytes: this.storageSizeCache.sizes.nouns,
+                verbsSizeBytes: this.storageSizeCache.sizes.verbs,
+                metadataSizeBytes: this.storageSizeCache.sizes.metadata,
+                indexSizeBytes: this.storageSizeCache.sizes.index
+            },
+            searchMetrics: {
+                totalSearches: this.searchMetrics.totalSearches,
+                averageSearchTimeMs: avgSearchTime,
+                searchesLastHour,
+                searchesLastDay,
+                topSearchTerms
+            },
+            verbStatistics: {
+                totalVerbs: Array.from(this.verbTypes.values()).reduce((a, b) => a + b, 0),
+                verbTypes: Object.fromEntries(this.verbTypes),
+                averageConnectionsPerVerb: 2 // Verbs connect 2 nouns
+            }
+        };
+    }
+    /**
+     * Merge statistics from storage (for distributed systems)
+     */
+    mergeFromStorage(stored) {
+        // Merge content types
+        if (stored.contentTypes) {
+            for (const [type, count] of Object.entries(stored.contentTypes)) {
+                this.contentTypes.set(type, count);
+            }
+        }
+        // Merge verb types
+        if (stored.verbStatistics?.verbTypes) {
+            for (const [type, count] of Object.entries(stored.verbStatistics.verbTypes)) {
+                this.verbTypes.set(type, count);
+            }
+        }
+        // Merge search metrics
+        if (stored.searchMetrics) {
+            this.searchMetrics.totalSearches = stored.searchMetrics.totalSearches || 0;
+            this.searchMetrics.totalSearchTimeMs = (stored.searchMetrics.averageSearchTimeMs || 0) * this.searchMetrics.totalSearches;
+        }
+        // Merge data freshness
+        if (stored.dataFreshness) {
+            this.oldestTimestamp = new Date(stored.dataFreshness.oldestEntry).getTime();
+            this.newestTimestamp = new Date(stored.dataFreshness.newestEntry).getTime();
+        }
+    }
+    /**
+     * Reset statistics (for testing)
+     */
+    reset() {
+        this.contentTypes.clear();
+        this.verbTypes.clear();
+        this.updateTimestamps = [];
+        this.searchMetrics = {
+            totalSearches: 0,
+            totalSearchTimeMs: 0,
+            searchTimestamps: [],
+            topSearchTerms: new Map()
+        };
+        this.oldestTimestamp = Date.now();
+        this.newestTimestamp = Date.now();
+    }
+    pruneSearchTerms() {
+        // Keep only top N search terms
+        const sorted = Array.from(this.searchMetrics.topSearchTerms.entries())
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, this.MAX_SEARCH_TERMS);
+        this.searchMetrics.topSearchTerms.clear();
+        for (const [term, count] of sorted) {
+            this.searchMetrics.topSearchTerms.set(term, count);
+        }
+    }
+}
+
 /**
  * BrainyData
  * Main class that provides the vector database functionality
@@ -17170,6 +17645,8 @@ class BrainyData {
         this.operationalMode = null;
         this.domainDetector = null;
         this.healthMonitor = null;
+        // Statistics collector
+        this.statisticsCollector = new StatisticsCollector();
         // Set dimensions to fixed value of 512 (Universal Sentence Encoder dimension)
         this._dimensions = 512;
         // Set distance function
@@ -17433,7 +17910,8 @@ class BrainyData {
             }
             // Adapt cache configuration based on performance (every few updates)
             // Only adapt every 5th update to avoid over-optimization
-            const updateCount = Math.floor((Date.now() - (this.lastUpdateTime || 0)) / this.realtimeUpdateConfig.interval);
+            const updateCount = Math.floor((Date.now() - (this.lastUpdateTime || 0)) /
+                this.realtimeUpdateConfig.interval);
             if (updateCount % 5 === 0) {
                 this.adaptCacheConfiguration();
             }
@@ -17751,6 +18229,16 @@ class BrainyData {
                     // Continue initialization even if remote connection fails
                 }
             }
+            // Initialize statistics collector with existing data
+            try {
+                const existingStats = await this.storage.getStatistics();
+                if (existingStats) {
+                    this.statisticsCollector.mergeFromStorage(existingStats);
+                }
+            }
+            catch (e) {
+                // Ignore errors loading existing statistics
+            }
             this.isInitialized = true;
             this.isInitializing = false;
             // Start real-time updates if enabled
@@ -17962,13 +18450,15 @@ class BrainyData {
                 try {
                     if (this.writeOnly) {
                         // In write-only mode, check storage directly
-                        existingNoun = await this.storage.getNoun(options.id) ?? undefined;
+                        existingNoun =
+                            (await this.storage.getNoun(options.id)) ?? undefined;
                     }
                     else {
                         // In normal mode, check index first, then storage
                         existingNoun = this.index.getNouns().get(options.id);
                         if (!existingNoun) {
-                            existingNoun = await this.storage.getNoun(options.id) ?? undefined;
+                            existingNoun =
+                                (await this.storage.getNoun(options.id)) ?? undefined;
                         }
                     }
                     if (existingNoun) {
@@ -18003,6 +18493,7 @@ class BrainyData {
                     id,
                     vector,
                     connections: new Map(),
+                    level: 0, // Default level for new nodes
                     metadata: undefined // Will be set separately
                 };
             }
@@ -18076,17 +18567,24 @@ class BrainyData {
                                 // Domain already specified, keep it
                                 const domainInfo = this.domainDetector.detectDomain(metadataToSave);
                                 if (domainInfo.domainMetadata) {
-                                    metadataToSave.domainMetadata = domainInfo.domainMetadata;
+                                    ;
+                                    metadataToSave.domainMetadata =
+                                        domainInfo.domainMetadata;
                                 }
                             }
                             else {
                                 // Try to detect domain from the data
-                                const dataToAnalyze = Array.isArray(vectorOrData) ? metadata : vectorOrData;
+                                const dataToAnalyze = Array.isArray(vectorOrData)
+                                    ? metadata
+                                    : vectorOrData;
                                 const domainInfo = this.domainDetector.detectDomain(dataToAnalyze);
                                 if (domainInfo.domain) {
+                                    ;
                                     metadataToSave.domain = domainInfo.domain;
                                     if (domainInfo.domainMetadata) {
-                                        metadataToSave.domainMetadata = domainInfo.domainMetadata;
+                                        ;
+                                        metadataToSave.domainMetadata =
+                                            domainInfo.domainMetadata;
                                     }
                                 }
                             }
@@ -18101,10 +18599,19 @@ class BrainyData {
                     // Track metadata statistics
                     const metadataService = this.getServiceName(options);
                     await this.storage.incrementStatistic('metadata', metadataService);
+                    // Track content type if it's a GraphNoun
+                    if (metadataToSave &&
+                        typeof metadataToSave === 'object' &&
+                        'noun' in metadataToSave) {
+                        this.statisticsCollector.trackContentType(metadataToSave.noun);
+                    }
+                    // Track update timestamp
+                    this.statisticsCollector.trackUpdate();
                 }
             }
-            // Update HNSW index size (excluding verbs)
-            await this.storage.updateHnswIndexSize(await this.getNounCount());
+            // Update HNSW index size with actual index size
+            const indexSize = this.index.size();
+            await this.storage.updateHnswIndexSize(indexSize);
             // Update health metrics if in distributed mode
             if (this.healthMonitor) {
                 const vectorCount = await this.getNounCount();
@@ -18578,7 +19085,7 @@ class BrainyData {
         let startIndex = 0;
         // If cursor provided, find starting position
         if (options.cursor) {
-            startIndex = allResults.findIndex(r => r.id === options.cursor.lastId &&
+            startIndex = allResults.findIndex((r) => r.id === options.cursor.lastId &&
                 Math.abs(r.score - options.cursor.lastScore) < 0.0001);
             if (startIndex >= 0) {
                 startIndex += 1; // Start after the cursor position
@@ -18595,7 +19102,8 @@ class BrainyData {
         }
         // Create cursor for next page
         let nextCursor;
-        const hasMoreResults = (startIndex + results.length) < allResults.length || allResults.length >= searchK;
+        const hasMoreResults = startIndex + results.length < allResults.length ||
+            allResults.length >= searchK;
         if (results.length > 0 && hasMoreResults) {
             const lastResult = results[results.length - 1];
             nextCursor = {
@@ -18679,7 +19187,7 @@ class BrainyData {
             });
         }
         // Filter out placeholder nouns from search results
-        searchResults = searchResults.filter(result => {
+        searchResults = searchResults.filter((result) => {
             if (result.metadata && typeof result.metadata === 'object') {
                 const metadata = result.metadata;
                 // Exclude placeholder nouns from search results
@@ -18787,7 +19295,7 @@ class BrainyData {
             // In write-only mode, query storage directly since index is not loaded
             if (this.writeOnly) {
                 try {
-                    noun = await this.storage.getNoun(id) ?? undefined;
+                    noun = (await this.storage.getNoun(id)) ?? undefined;
                 }
                 catch (storageError) {
                     // If storage lookup fails, return null (noun doesn't exist)
@@ -18800,7 +19308,7 @@ class BrainyData {
                 // If not found in index, fallback to storage (for race conditions)
                 if (!noun && this.storage) {
                     try {
-                        noun = await this.storage.getNoun(id) ?? undefined;
+                        noun = (await this.storage.getNoun(id)) ?? undefined;
                     }
                     catch (storageError) {
                         // Storage lookup failed, noun doesn't exist
@@ -19192,6 +19700,7 @@ class BrainyData {
                     id: sourceId,
                     vector: sourcePlaceholderVector,
                     connections: new Map(),
+                    level: 0,
                     metadata: sourceMetadata
                 };
                 // Create placeholder target noun
@@ -19212,6 +19721,7 @@ class BrainyData {
                     id: targetId,
                     vector: targetPlaceholderVector,
                     connections: new Map(),
+                    level: 0,
                     metadata: targetMetadata
                 };
                 // Save placeholder nouns to storage (but skip indexing for speed)
@@ -19448,8 +19958,11 @@ class BrainyData {
             // Track verb statistics
             const serviceForStats = this.getServiceName(options);
             await this.storage.incrementStatistic('verb', serviceForStats);
-            // Update HNSW index size (excluding verbs)
-            await this.storage.updateHnswIndexSize(await this.getNounCount());
+            // Track verb type
+            this.statisticsCollector.trackVerbType(verbMetadata.verb);
+            // Update HNSW index size with actual index size
+            const indexSize = this.index.size();
+            await this.storage.updateHnswIndexSize(indexSize);
             // Invalidate search cache since verb data has changed
             this.searchCache.invalidateOnDataChange('add');
             return id;
@@ -19674,6 +20187,8 @@ class BrainyData {
             await this.index.clear();
             // Clear storage
             await this.storage.clear();
+            // Reset statistics collector
+            this.statisticsCollector = new StatisticsCollector();
             // Clear search cache since all data has been removed
             this.searchCache.invalidateOnDataChange('delete');
         }
@@ -19727,7 +20242,8 @@ class BrainyData {
             // Apply new cache configuration
             this.searchCache.updateConfig(newConfig.cacheConfig);
             // Apply new real-time update configuration if needed
-            if (newConfig.realtimeConfig.enabled !== this.realtimeUpdateConfig.enabled ||
+            if (newConfig.realtimeConfig.enabled !==
+                this.realtimeUpdateConfig.enabled ||
                 newConfig.realtimeConfig.interval !== this.realtimeUpdateConfig.interval) {
                 const wasEnabled = this.realtimeUpdateConfig.enabled;
                 this.realtimeUpdateConfig = {
@@ -19817,6 +20333,41 @@ class BrainyData {
         // Call the flushStatisticsToStorage method on the storage adapter
         await this.storage.flushStatisticsToStorage();
     }
+    /**
+     * Update storage sizes if needed (called periodically for performance)
+     */
+    async updateStorageSizesIfNeeded() {
+        // Only update every minute to avoid performance impact
+        const now = Date.now();
+        const lastUpdate = this.lastStorageSizeUpdate || 0;
+        if (now - lastUpdate < 60000) {
+            return; // Skip if updated recently
+        }
+        this.lastStorageSizeUpdate = now;
+        try {
+            // Estimate sizes based on counts and average sizes
+            const stats = await this.storage.getStatistics();
+            if (stats) {
+                const avgNounSize = 2048; // ~2KB per noun (vector + metadata)
+                const avgVerbSize = 512; // ~0.5KB per verb
+                const avgMetadataSize = 256; // ~0.25KB per metadata entry
+                const avgIndexEntrySize = 128; // ~128 bytes per index entry
+                // Calculate total counts
+                const totalNouns = Object.values(stats.nounCount).reduce((a, b) => a + b, 0);
+                const totalVerbs = Object.values(stats.verbCount).reduce((a, b) => a + b, 0);
+                const totalMetadata = Object.values(stats.metadataCount).reduce((a, b) => a + b, 0);
+                this.statisticsCollector.updateStorageSizes({
+                    nouns: totalNouns * avgNounSize,
+                    verbs: totalVerbs * avgVerbSize,
+                    metadata: totalMetadata * avgMetadataSize,
+                    index: stats.hnswIndexSize * avgIndexEntrySize
+                });
+            }
+        }
+        catch (error) {
+            // Ignore errors in size calculation
+        }
+    }
     /**
      * Get statistics about the current state of the database
      * @param options Additional options for retrieving statistics
@@ -19891,6 +20442,40 @@ class BrainyData {
                     relate: result.verbCount,
                     total: result.nounCount + result.verbCount + result.metadataCount
                 };
+                // Add extended statistics if requested
+                if (true) {
+                    // Always include for now
+                    // Add index health metrics
+                    try {
+                        const indexHealth = this.index.getIndexHealth();
+                        result.indexHealth = indexHealth;
+                    }
+                    catch (e) {
+                        // Index health not available
+                    }
+                    // Add cache metrics
+                    try {
+                        const cacheStats = this.searchCache.getStats();
+                        result.cacheMetrics = cacheStats;
+                    }
+                    catch (e) {
+                        // Cache stats not available
+                    }
+                    // Add memory usage
+                    if (typeof process !== 'undefined' && process.memoryUsage) {
+                        ;
+                        result.memoryUsage = process.memoryUsage().heapUsed;
+                    }
+                    // Add last updated timestamp
+                    ;
+                    result.lastUpdated =
+                        stats.lastUpdated || new Date().toISOString();
+                    // Add enhanced statistics from collector
+                    const collectorStats = this.statisticsCollector.getStatistics();
+                    Object.assign(result, collectorStats);
+                    // Update storage sizes if needed (only periodically for performance)
+                    await this.updateStorageSizesIfNeeded();
+                }
                 return result;
             }
             // If statistics are not available, return zeros instead of calculating on-demand
@@ -20250,15 +20835,20 @@ class BrainyData {
         await this.ensureInitialized();
         // Check if database is in write-only mode
         this.checkWriteOnly();
+        const searchStartTime = Date.now();
         try {
             // Embed the query text
             const queryVector = await this.embed(query);
             // Search using the embedded vector
-            return await this.search(queryVector, k, {
+            const results = await this.search(queryVector, k, {
                 nounTypes: options.nounTypes,
                 includeVerbs: options.includeVerbs,
                 searchMode: options.searchMode
             });
+            // Track search performance
+            const duration = Date.now() - searchStartTime;
+            this.statisticsCollector.trackSearch(query, duration);
+            return results;
         }
         catch (error) {
             console.error('Failed to search with text query:', error);
@@ -127662,5 +128252,5 @@ var _osShim$1 = /*#__PURE__*/Object.freeze({
 	totalmem: totalmem
 });
 
-export { AugmentationType, BrainyData, BrainyMCPAdapter, BrainyMCPService, ExecutionMode, FileSystemStorage, FileSystemStorageAugmentation, HNSWIndex, HNSWIndexOptimized, MCPAugmentationToolset, MCPRequestType, MCP_VERSION, MemoryStorage, MemoryStorageAugmentation, NounType, OPFSStorage, OPFSStorageAugmentation, Pipeline, S3CompatibleStorage as R2Storage, S3CompatibleStorage, SequentialPipeline, ServerSearchActivationAugmentation, ServerSearchConduitAugmentation, StreamlinedExecutionMode, UniversalSentenceEncoder$1 as UniversalSentenceEncoder, VerbType, WebRTCConduitAugmentation, WebSocketConduitAugmentation, addWebSocketSupport, applyTensorFlowPatch, areWebWorkersAvailable, areWorkerThreadsAvailable, areWorkerThreadsAvailableSync, augmentationPipeline, availableAugmentations, cleanupWorkerPools, cosineDistance$1 as cosineDistance, createAugmentationRegistryPlugin, createAugmentationRegistryRollupPlugin, createConduitAugmentation, createEmbeddingFunction, createMemoryAugmentation, createPipeline, createSenseAugmentation, createServerSearchAugmentations, createStorage, createStreamingPipeline, createTensorFlowEmbeddingFunction, createThreadedEmbeddingFunction, defaultEmbeddingFunction, dotProductDistance, environment, euclideanDistance, executeAugmentation, executeByType, executeInThread, executeSingle, executeStreamlined, getAugmentationsByType, getNounTypeMap, getNounTypes, getStatistics, getVerbTypeMap, getVerbTypes, initializeAugmentationPipeline, isBrowser$1 as isBrowser, isNode, isThreadingAvailable, isThreadingAvailableAsync, isWebWorker, loadAugmentationModule, loadAugmentationsFromModules, manhattanDistance, pipeline, processStaticData, processStreamingData, registerAugmentation, sequentialPipeline, setAugmentationEnabled };
+export { AugmentationType, BrainyData, BrainyMCPAdapter, BrainyMCPService, ExecutionMode, FileSystemStorage, FileSystemStorageAugmentation, HNSWIndex, HNSWIndexOptimized, LogLevel, MCPAugmentationToolset, MCPRequestType, MCP_VERSION, MemoryStorage, MemoryStorageAugmentation, NounType, OPFSStorage, OPFSStorageAugmentation, Pipeline, S3CompatibleStorage as R2Storage, S3CompatibleStorage, SequentialPipeline, ServerSearchActivationAugmentation, ServerSearchConduitAugmentation, StreamlinedExecutionMode, UniversalSentenceEncoder$1 as UniversalSentenceEncoder, VerbType, WebRTCConduitAugmentation, WebSocketConduitAugmentation, addWebSocketSupport, applyTensorFlowPatch, areWebWorkersAvailable, areWorkerThreadsAvailable, areWorkerThreadsAvailableSync, augmentationPipeline, availableAugmentations, cleanupWorkerPools, configureLogger, cosineDistance$1 as cosineDistance, createAugmentationRegistryPlugin, createAugmentationRegistryRollupPlugin, createConduitAugmentation, createEmbeddingFunction, createMemoryAugmentation, createModuleLogger, createPipeline, createSenseAugmentation, createServerSearchAugmentations, createStorage, createStreamingPipeline, createTensorFlowEmbeddingFunction, createThreadedEmbeddingFunction, defaultEmbeddingFunction, dotProductDistance, environment, euclideanDistance, executeAugmentation, executeByType, executeInThread, executeSingle, executeStreamlined, getAugmentationsByType, getNounTypeMap, getNounTypes, getStatistics, getVerbTypeMap, getVerbTypes, initializeAugmentationPipeline, isBrowser$1 as isBrowser, isNode, isThreadingAvailable, isThreadingAvailableAsync, isWebWorker, loadAugmentationModule, loadAugmentationsFromModules, logger, manhattanDistance, pipeline, processStaticData, processStreamingData, registerAugmentation, sequentialPipeline, setAugmentationEnabled };
 //# sourceMappingURL=unified.js.map