@soulcraft/brainy 0.48.0 → 0.50.0

package/dist/brainyData.js

@@ -3,10 +3,13 @@
  * Main class that provides the vector database functionality
  */
 import { v4 as uuidv4 } from 'uuid';
+import { HNSWIndex } from './hnsw/hnswIndex.js';
 import { HNSWIndexOptimized } from './hnsw/hnswIndexOptimized.js';
 import { createStorage } from './storage/storageFactory.js';
 import { cosineDistance, defaultEmbeddingFunction, cleanupWorkerPools, batchEmbed } from './utils/index.js';
 import { getAugmentationVersion } from './utils/version.js';
+import { matchesMetadataFilter } from './utils/metadataFilter.js';
+import { MetadataIndexManager } from './utils/metadataIndex.js';
 import { NounType, VerbType } from './types/graphTypes.js';
 import { createServerSearchAugmentations } from './augmentations/serverSearchAugmentations.js';
 import { augmentationPipeline } from './augmentationPipeline.js';
@@ -41,6 +44,7 @@ export class BrainyData {
      */
     constructor(config = {}) {
         this.storage = null;
+        this.metadataIndex = null;
         this.isInitialized = false;
         this.isInitializing = false;
         this.storageConfig = {};
@@ -58,6 +62,7 @@ export class BrainyData {
             updateIndex: true
         };
         this.updateTimerId = null;
+        this.maintenanceIntervals = [];
         this.lastUpdateTime = 0;
         this.lastKnownNounCount = 0;
         // Remote server properties
@@ -73,6 +78,8 @@ export class BrainyData {
         this.healthMonitor = null;
         // Statistics collector
         this.statisticsCollector = new StatisticsCollector();
+        // Store config
+        this.config = config;
         // Set dimensions to fixed value of 384 (all-MiniLM-L6-v2 dimension)
         this._dimensions = 384;
         // Set distance function
@@ -83,8 +90,9 @@ export class BrainyData {
         if (config.storageAdapter) {
             hnswConfig.useDiskBasedIndex = true;
         }
-        this.index = new HNSWIndexOptimized(hnswConfig, this.distanceFunction, config.storageAdapter || null);
-        this.useOptimizedIndex = true;
+        // Temporarily use base HNSW index for metadata filtering
+        this.index = new HNSWIndex(hnswConfig, this.distanceFunction);
+        this.useOptimizedIndex = false;
         // Set storage if provided, otherwise it will be initialized in init()
         this.storage = config.storageAdapter || null;
         // Store logging configuration
@@ -106,6 +114,8 @@ export class BrainyData {
             config.storage?.requestPersistentStorage || false;
         // Set read-only flag
         this.readOnly = config.readOnly || false;
+        // Set frozen flag (defaults to false to allow optimizations in readOnly mode)
+        this.frozen = config.frozen || false;
         // Set lazy loading in read-only mode flag
         this.lazyLoadInReadOnlyMode = config.lazyLoadInReadOnlyMode || false;
         // Set write-only flag
@@ -198,6 +208,15 @@ export class BrainyData {
             throw new Error('Cannot perform write operation: database is in read-only mode');
         }
     }
+    /**
+     * Check if the database is frozen and throw an error if it is
+     * @throws Error if the database is frozen
+     */
+    checkFrozen() {
+        if (this.frozen) {
+            throw new Error('Cannot perform operation: database is frozen (no changes allowed)');
+        }
+    }
     /**
      * Check if the database is in write-only mode and throw an error if it is
      * @param allowExistenceChecks If true, allows existence checks (get operations) in write-only mode
@@ -217,6 +236,13 @@ export class BrainyData {
         if (!this.realtimeUpdateConfig.enabled) {
             return;
         }
+        // If the database is frozen, do not start real-time updates
+        if (this.frozen) {
+            if (this.loggingConfig?.verbose) {
+                console.log('Real-time updates disabled: database is frozen');
+            }
+            return;
+        }
         // If the update timer is already running, do nothing
         if (this.updateTimerId !== null) {
             return;
@@ -281,6 +307,27 @@ export class BrainyData {
             this.startRealtimeUpdates();
         }
     }
+    /**
+     * Start metadata index maintenance
+     */
+    startMetadataIndexMaintenance() {
+        if (!this.metadataIndex)
+            return;
+        // Flush index periodically to persist changes
+        const flushInterval = setInterval(async () => {
+            try {
+                await this.metadataIndex.flush();
+            }
+            catch (error) {
+                console.warn('Error flushing metadata index:', error);
+            }
+        }, 30000); // Flush every 30 seconds
+        // Store the interval ID for cleanup
+        if (!this.maintenanceIntervals) {
+            this.maintenanceIntervals = [];
+        }
+        this.maintenanceIntervals.push(flushInterval);
+    }
     /**
      * Disable real-time updates
      */
@@ -307,6 +354,10 @@ export class BrainyData {
         if (!this.isInitialized || !this.storage) {
             return;
         }
+        // If the database is frozen, do not perform updates
+        if (this.frozen) {
+            return;
+        }
         try {
             // Record the current time
             const startTime = Date.now();
@@ -438,40 +489,51 @@ export class BrainyData {
             const currentCount = await this.getNounCount();
             // If the noun count has changed, update the index
             if (currentCount !== this.lastKnownNounCount) {
-                // Get all nouns from storage
-                const nouns = await this.storage.getAllNouns();
                 // Get all nouns currently in the index
                 const indexNouns = this.index.getNouns();
                 const indexNounIds = new Set(indexNouns.keys());
-                // Find nouns that are in storage but not in the index
-                const newNouns = nouns.filter((noun) => !indexNounIds.has(noun.id));
-                // Add new nouns to the index
-                for (const noun of newNouns) {
-                    // Check if the vector dimensions match the expected dimensions
-                    if (noun.vector.length !== this._dimensions) {
-                        console.warn(`Skipping noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}`);
-                        continue;
-                    }
-                    // Add to index
-                    await this.index.addItem({
-                        id: noun.id,
-                        vector: noun.vector
+                // Use pagination to load nouns from storage
+                let offset = 0;
+                const limit = 100;
+                let hasMore = true;
+                let totalNewNouns = 0;
+                while (hasMore) {
+                    const result = await this.storage.getNouns({
+                        pagination: { offset, limit }
                     });
-                    if (this.loggingConfig?.verbose) {
-                        console.log(`Added new noun ${noun.id} to index during real-time update`);
+                    // Find nouns that are in storage but not in the index
+                    const newNouns = result.items.filter((noun) => !indexNounIds.has(noun.id));
+                    totalNewNouns += newNouns.length;
+                    // Add new nouns to the index
+                    for (const noun of newNouns) {
+                        // Check if the vector dimensions match the expected dimensions
+                        if (noun.vector.length !== this._dimensions) {
+                            console.warn(`Skipping noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}`);
+                            continue;
+                        }
+                        // Add to index
+                        await this.index.addItem({
+                            id: noun.id,
+                            vector: noun.vector
+                        });
+                        if (this.loggingConfig?.verbose) {
+                            console.log(`Added new noun ${noun.id} to index during real-time update`);
+                        }
                     }
+                    hasMore = result.hasMore;
+                    offset += limit;
                 }
                 // Update the last known noun count
                 this.lastKnownNounCount = currentCount;
                 // Invalidate search cache if new nouns were detected
-                if (newNouns.length > 0) {
+                if (totalNewNouns > 0) {
                     this.searchCache.invalidateOnDataChange('add');
                     if (this.loggingConfig?.verbose) {
                         console.log('Search cache invalidated due to external data changes');
                     }
                 }
-                if (this.loggingConfig?.verbose && newNouns.length > 0) {
-                    console.log(`Real-time update: Added ${newNouns.length} new nouns to index using full scan`);
+                if (this.loggingConfig?.verbose && totalNewNouns > 0) {
+                    console.log(`Real-time update: Added ${totalNewNouns} new nouns to index using full scan`);
                 }
             }
         }
@@ -624,23 +686,31 @@ export class BrainyData {
                 this.index.clear();
             }
             else {
-                // Load all nouns from storage
-                const nouns = await this.storage.getAllNouns();
-                // Clear the index and add all nouns
+                // Clear the index and load nouns using pagination
                 this.index.clear();
-                for (const noun of nouns) {
-                    // Check if the vector dimensions match the expected dimensions
-                    if (noun.vector.length !== this._dimensions) {
-                        console.warn(`Deleting noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}`);
-                        // Delete the mismatched noun from storage to prevent future issues
-                        await this.storage.deleteNoun(noun.id);
-                        continue;
-                    }
-                    // Add to index
-                    await this.index.addItem({
-                        id: noun.id,
-                        vector: noun.vector
+                let offset = 0;
+                const limit = 100;
+                let hasMore = true;
+                while (hasMore) {
+                    const result = await this.storage.getNouns({
+                        pagination: { offset, limit }
                     });
+                    for (const noun of result.items) {
+                        // Check if the vector dimensions match the expected dimensions
+                        if (noun.vector.length !== this._dimensions) {
+                            console.warn(`Deleting noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}`);
+                            // Delete the mismatched noun from storage to prevent future issues
+                            await this.storage.deleteNoun(noun.id);
+                            continue;
+                        }
+                        // Add to index
+                        await this.index.addItem({
+                            id: noun.id,
+                            vector: noun.vector
+                        });
+                    }
+                    hasMore = result.hasMore;
+                    offset += limit;
                 }
             }
             // Connect to remote server if configured with autoConnect
@@ -663,10 +733,46 @@ export class BrainyData {
             catch (e) {
                 // Ignore errors loading existing statistics
             }
+            // Initialize metadata index if not in write-only mode
+            if (!this.writeOnly) {
+                this.metadataIndex = new MetadataIndexManager(this.storage, this.config.metadataIndex);
+                // Check if we need to rebuild the index (for existing data)
+                // Skip rebuild for memory storage (starts empty) or when in read-only mode
+                // Also skip if index already has entries
+                const isMemoryStorage = this.storage?.constructor?.name === 'MemoryStorage';
+                const stats = await this.metadataIndex.getStats();
+                if (!isMemoryStorage && !this.readOnly && stats.totalEntries === 0) {
+                    // Check if we have existing data that needs indexing
+                    // Use a simple check to avoid expensive operations
+                    try {
+                        const testResult = await this.storage.getNouns({ pagination: { offset: 0, limit: 1 } });
+                        if (testResult.items.length > 0) {
+                            if (this.loggingConfig?.verbose) {
+                                console.log('Rebuilding metadata index for existing data...');
+                            }
+                            await this.metadataIndex.rebuild();
+                            if (this.loggingConfig?.verbose) {
+                                const newStats = await this.metadataIndex.getStats();
+                                console.log(`Metadata index rebuilt: ${newStats.totalEntries} entries, ${newStats.fieldsIndexed.length} fields`);
+                            }
+                        }
+                    }
+                    catch (error) {
+                        // If getNouns fails, skip rebuild
+                        if (this.loggingConfig?.verbose) {
+                            console.log('Skipping metadata index rebuild:', error);
+                        }
+                    }
+                }
+            }
             this.isInitialized = true;
             this.isInitializing = false;
             // Start real-time updates if enabled
             this.startRealtimeUpdates();
+            // Start metadata index maintenance
+            if (this.metadataIndex) {
+                this.startMetadataIndexMaintenance();
+            }
         }
         catch (error) {
             console.error('Failed to initialize BrainyData:', error);
@@ -1018,6 +1124,10 @@ export class BrainyData {
                         }
                     }
                     await this.storage.saveMetadata(id, metadataToSave);
+                    // Update metadata index
+                    if (this.metadataIndex && !this.readOnly && !this.frozen) {
+                        await this.metadataIndex.addToIndex(id, metadataToSave);
+                    }
                     // Track metadata statistics
                     const metadataService = this.getServiceName(options);
                     await this.storage.incrementStatistic('metadata', metadataService);
@@ -1300,10 +1410,11 @@ export class BrainyData {
                     }
                     // In lazy loading mode, we need to load some nodes to search
                     // Instead of loading all nodes, we'll load a subset of nodes
-                    // Since we don't have a specialized method to get top nodes for a query,
-                    // we'll load a limited number of nodes from storage
-                    const nouns = await this.storage.getAllNouns();
-                    const limitedNouns = nouns.slice(0, Math.min(nouns.length, k * 10)); // Get 10x more nodes than needed
+                    // Load a limited number of nodes from storage using pagination
+                    const result = await this.storage.getNouns({
+                        pagination: { offset: 0, limit: k * 10 } // Get 10x more nodes than needed
+                    });
+                    const limitedNouns = result.items;
                     // Add these nodes to the index
                     for (const node of limitedNouns) {
                         // Check if the vector dimensions match the expected dimensions
@@ -1321,11 +1432,79 @@ export class BrainyData {
                         console.log(`Lazy loading mode: Added ${limitedNouns.length} nodes to index for search`);
                     }
                 }
+                // Create filter function for HNSW search with metadata index optimization
+                const hasMetadataFilter = options.metadata && Object.keys(options.metadata).length > 0;
+                const hasServiceFilter = !!options.service;
+                let filterFunction;
+                let preFilteredIds;
+                // Use metadata index for pre-filtering if available
+                if (hasMetadataFilter && this.metadataIndex) {
+                    try {
+                        // Ensure metadata index is up to date
+                        await this.metadataIndex.flush();
+                        // Get candidate IDs from metadata index
+                        const candidateIds = await this.metadataIndex.getIdsForFilter(options.metadata);
+                        if (candidateIds.length > 0) {
+                            preFilteredIds = new Set(candidateIds);
+                            // Create a simple filter function that just checks the pre-filtered set
+                            filterFunction = async (id) => {
+                                if (!preFilteredIds.has(id))
+                                    return false;
+                                // Still apply service filter if needed
+                                if (hasServiceFilter) {
+                                    const metadata = await this.storage.getMetadata(id);
+                                    const noun = this.index.getNouns().get(id);
+                                    if (!noun || !metadata)
+                                        return false;
+                                    const result = { id, score: 0, vector: noun.vector, metadata };
+                                    return this.filterResultsByService([result], options.service).length > 0;
+                                }
+                                return true;
+                            };
+                        }
+                        else {
+                            // No items match the metadata criteria, return empty results immediately
+                            return [];
+                        }
+                    }
+                    catch (indexError) {
+                        console.warn('Metadata index error, falling back to full filtering:', indexError);
+                        // Fall back to full metadata filtering below
+                    }
+                }
+                // Fallback to full metadata filtering if index wasn't used
+                if (!filterFunction && (hasMetadataFilter || hasServiceFilter)) {
+                    filterFunction = async (id) => {
+                        // Get metadata for filtering
+                        let metadata = await this.storage.getMetadata(id);
+                        if (metadata === null) {
+                            metadata = {};
+                        }
+                        // Apply metadata filter
+                        if (hasMetadataFilter) {
+                            const matches = matchesMetadataFilter(metadata, options.metadata);
+                            if (!matches) {
+                                return false;
+                            }
+                        }
+                        // Apply service filter
+                        if (hasServiceFilter) {
+                            const noun = this.index.getNouns().get(id);
+                            if (!noun)
+                                return false;
+                            const result = { id, score: 0, vector: noun.vector, metadata };
+                            if (!this.filterResultsByService([result], options.service).length) {
+                                return false;
+                            }
+                        }
+                        return true;
+                    };
+                }
                 // When using offset, we need to fetch more results and then slice
                 const offset = options.offset || 0;
                 const totalNeeded = k + offset;
-                // Search in the index for totalNeeded results
-                const results = await this.index.search(queryVector, totalNeeded);
+                // Search in the index with filter
+                const results = await this.index.search(queryVector, totalNeeded, filterFunction);
                 // Skip the offset number of results
                 const paginatedResults = results.slice(offset, offset + k);
                 // Get metadata for each result
@@ -1351,8 +1530,7 @@ export class BrainyData {
                         metadata: metadata
                     });
                 }
-                // Filter results by service if specified
-                return this.filterResultsByService(searchResults, options.service);
+                return searchResults;
             }
             else {
                 // Get nouns for each noun type in parallel
@@ -1397,8 +1575,8 @@ export class BrainyData {
                         metadata: metadata
                     });
                 }
-                // Filter results by service if specified
-                return this.filterResultsByService(searchResults, options.service);
+                // Results are already filtered, just return them
+                return searchResults;
             }
         }
         catch (error) {
@@ -1467,22 +1645,29 @@ export class BrainyData {
         }
         // Default behavior (backward compatible): search locally
         try {
-            // Check cache first (transparent to user)
-            const cacheKey = this.searchCache.getCacheKey(queryVectorOrData, k, options);
-            const cachedResults = this.searchCache.get(cacheKey);
-            if (cachedResults) {
-                // Track cache hit in health monitor
-                if (this.healthMonitor) {
-                    const latency = Date.now() - startTime;
-                    this.healthMonitor.recordRequest(latency, false);
-                    this.healthMonitor.recordCacheAccess(true);
-                }
-                return cachedResults;
+            const hasMetadataFilter = options.metadata && Object.keys(options.metadata).length > 0;
+            // Check cache first (transparent to user) - but skip cache if we have metadata filters
+            if (!hasMetadataFilter) {
+                const cacheKey = this.searchCache.getCacheKey(queryVectorOrData, k, options);
+                const cachedResults = this.searchCache.get(cacheKey);
+                if (cachedResults) {
+                    // Track cache hit in health monitor
+                    if (this.healthMonitor) {
+                        const latency = Date.now() - startTime;
+                        this.healthMonitor.recordRequest(latency, false);
+                        this.healthMonitor.recordCacheAccess(true);
+                    }
+                    return cachedResults;
+                }
             }
             // Cache miss - perform actual search
-            const results = await this.searchLocal(queryVectorOrData, k, options);
-            // Cache results for future queries (unless explicitly disabled)
-            if (!options.skipCache) {
+            const results = await this.searchLocal(queryVectorOrData, k, {
+                ...options,
+                metadata: options.metadata
+            });
+            // Cache results for future queries (unless explicitly disabled or has metadata filter)
+            if (!options.skipCache && !hasMetadataFilter) {
+                const cacheKey = this.searchCache.getCacheKey(queryVectorOrData, k, options);
                 this.searchCache.set(cacheKey, results);
             }
             // Track successful search in health monitor
@@ -1611,6 +1796,7 @@ export class BrainyData {
             searchResults = await this.searchByNounTypes(queryToUse, k, options.nounTypes, {
                 forceEmbed: options.forceEmbed,
                 service: options.service,
+                metadata: options.metadata,
                 offset: options.offset
             });
         }
@@ -1619,6 +1805,7 @@ export class BrainyData {
             searchResults = await this.searchByNounTypes(queryToUse, k, null, {
                 forceEmbed: options.forceEmbed,
                 service: options.service,
+                metadata: options.metadata,
                 offset: options.offset
             });
         }
@@ -1947,6 +2134,12 @@ export class BrainyData {
             await this.storage.decrementStatistic('noun', service);
             // Try to remove metadata (ignore errors)
             try {
+                // Get metadata before removing for index cleanup
+                const existingMetadata = await this.storage.getMetadata(actualId);
+                // Remove from metadata index
+                if (this.metadataIndex && existingMetadata && !this.readOnly && !this.frozen) {
+                    await this.metadataIndex.removeFromIndex(actualId, existingMetadata);
+                }
                 await this.storage.saveMetadata(actualId, null);
                 await this.storage.decrementStatistic('metadata', service);
             }
@@ -2032,6 +2225,18 @@ export class BrainyData {
             }
             // Update metadata
             await this.storage.saveMetadata(id, metadata);
+            // Update metadata index
+            if (this.metadataIndex && !this.readOnly && !this.frozen) {
+                // Remove old metadata from index if it exists
+                const oldMetadata = await this.storage.getMetadata(id);
+                if (oldMetadata) {
+                    await this.metadataIndex.removeFromIndex(id, oldMetadata);
+                }
+                // Add new metadata to index
+                if (metadata) {
+                    await this.metadataIndex.addToIndex(id, metadata);
+                }
+            }
             // Track metadata statistics
             const service = this.getServiceName(options);
             await this.storage.incrementStatistic('metadata', service);
@@ -2379,6 +2584,10 @@ export class BrainyData {
             };
             // Save the complete verb (BaseStorage will handle the separation)
             await this.storage.saveVerb(fullVerb);
+            // Update metadata index
+            if (this.metadataIndex && verbMetadata) {
+                await this.metadataIndex.addToIndex(id, verbMetadata);
+            }
             // Track verb statistics
             const serviceForStats = this.getServiceName(options);
             await this.storage.incrementStatistic('verb', serviceForStats);
@@ -2582,11 +2791,17 @@ export class BrainyData {
         // Check if database is in read-only mode
         this.checkReadOnly();
         try {
+            // Get existing metadata before removal for index cleanup
+            const existingMetadata = await this.storage.getVerbMetadata(id);
             // Remove from index
             const removed = this.index.removeItem(id);
             if (!removed) {
                 return false;
             }
+            // Remove from metadata index
+            if (this.metadataIndex && existingMetadata) {
+                await this.metadataIndex.removeFromIndex(id, existingMetadata);
+            }
             // Remove from storage
             await this.storage.deleteVerb(id);
             // Track deletion statistics
@@ -2754,6 +2969,10 @@ export class BrainyData {
         if (!this.storage) {
             throw new Error('Storage not initialized');
         }
+        // If the database is frozen, do not flush statistics
+        if (this.frozen) {
+            return;
+        }
         // Call the flushStatisticsToStorage method on the storage adapter
         await this.storage.flushStatisticsToStorage();
     }
@@ -2761,6 +2980,10 @@ export class BrainyData {
      * Update storage sizes if needed (called periodically for performance)
      */
     async updateStorageSizesIfNeeded() {
+        // If the database is frozen, do not update storage sizes
+        if (this.frozen) {
+            return;
+        }
         // Only update every minute to avoid performance impact
         const now = Date.now();
         const lastUpdate = this.lastStorageSizeUpdate || 0;
@@ -2801,8 +3024,8 @@ export class BrainyData {
     async getStatistics(options = {}) {
         await this.ensureInitialized();
         try {
-            // If forceRefresh is true, flush statistics to storage first
-            if (options.forceRefresh && this.storage) {
+            // If forceRefresh is true and not frozen, flush statistics to storage first
+            if (options.forceRefresh && this.storage && !this.frozen) {
                 await this.storage.flushStatisticsToStorage();
             }
             // Get statistics from storage
@@ -2945,6 +3168,137 @@ export class BrainyData {
             throw new Error(`Failed to get statistics: ${error}`);
         }
     }
+    /**
+     * List all services that have written data to the database
+     * @returns Array of service statistics
+     */
+    async listServices() {
+        await this.ensureInitialized();
+        try {
+            const stats = await this.storage.getStatistics();
+            if (!stats) {
+                return [];
+            }
+            // Get unique service names from all counters
+            const services = new Set();
+            Object.keys(stats.nounCount).forEach(s => services.add(s));
+            Object.keys(stats.verbCount).forEach(s => services.add(s));
+            Object.keys(stats.metadataCount).forEach(s => services.add(s));
+            // Build service statistics for each service
+            const result = [];
+            for (const service of services) {
+                const serviceStats = {
+                    name: service,
+                    totalNouns: stats.nounCount[service] || 0,
+                    totalVerbs: stats.verbCount[service] || 0,
+                    totalMetadata: stats.metadataCount[service] || 0
+                };
+                // Add activity timestamps if available
+                if (stats.serviceActivity && stats.serviceActivity[service]) {
+                    const activity = stats.serviceActivity[service];
+                    serviceStats.firstActivity = activity.firstActivity;
+                    serviceStats.lastActivity = activity.lastActivity;
+                    serviceStats.operations = {
+                        adds: activity.totalOperations,
+                        updates: 0,
+                        deletes: 0
+                    };
+                }
+                // Determine status based on recent activity
+                if (serviceStats.lastActivity) {
+                    const lastActivityTime = new Date(serviceStats.lastActivity).getTime();
+                    const now = Date.now();
+                    const hourAgo = now - 3600000;
+                    if (lastActivityTime > hourAgo) {
+                        serviceStats.status = 'active';
+                    }
+                    else {
+                        serviceStats.status = 'inactive';
+                    }
+                }
+                else {
+                    serviceStats.status = 'inactive';
+                }
+                // Check if service is read-only (has no write operations)
+                if (serviceStats.totalNouns === 0 && serviceStats.totalVerbs === 0) {
+                    serviceStats.status = 'read-only';
+                }
+                result.push(serviceStats);
+            }
+            // Sort by last activity (most recent first)
+            result.sort((a, b) => {
+                if (!a.lastActivity && !b.lastActivity)
+                    return 0;
+                if (!a.lastActivity)
+                    return 1;
+                if (!b.lastActivity)
+                    return -1;
+                return new Date(b.lastActivity).getTime() - new Date(a.lastActivity).getTime();
+            });
+            return result;
+        }
+        catch (error) {
+            console.error('Failed to list services:', error);
+            throw new Error(`Failed to list services: ${error}`);
+        }
+    }
+    /**
+     * Get statistics for a specific service
+     * @param service The service name to get statistics for
+     * @returns Service statistics or null if service not found
+     */
+    async getServiceStatistics(service) {
+        await this.ensureInitialized();
+        try {
+            const stats = await this.storage.getStatistics();
+            if (!stats) {
+                return null;
+            }
+            // Check if service exists in any counter
+            const hasData = (stats.nounCount[service] || 0) > 0 ||
+                (stats.verbCount[service] || 0) > 0 ||
+                (stats.metadataCount[service] || 0) > 0;
+            if (!hasData && !stats.serviceActivity?.[service]) {
+                return null;
+            }
+            const serviceStats = {
+                name: service,
+                totalNouns: stats.nounCount[service] || 0,
+                totalVerbs: stats.verbCount[service] || 0,
+                totalMetadata: stats.metadataCount[service] || 0
+            };
+            // Add activity timestamps if available
+            if (stats.serviceActivity && stats.serviceActivity[service]) {
+                const activity = stats.serviceActivity[service];
+                serviceStats.firstActivity = activity.firstActivity;
+                serviceStats.lastActivity = activity.lastActivity;
+                serviceStats.operations = {
+                    adds: activity.totalOperations,
+                    updates: 0,
+                    deletes: 0
+                };
+            }
+            // Determine status
+            if (serviceStats.lastActivity) {
+                const lastActivityTime = new Date(serviceStats.lastActivity).getTime();
+                const now = Date.now();
+                const hourAgo = now - 3600000;
+                serviceStats.status = lastActivityTime > hourAgo ? 'active' : 'inactive';
+            }
+            else {
+                serviceStats.status = 'inactive';
+            }
+            // Check if service is read-only
+            if (serviceStats.totalNouns === 0 && serviceStats.totalVerbs === 0) {
+                serviceStats.status = 'read-only';
+            }
+            return serviceStats;
+        }
+        catch (error) {
+            console.error(`Failed to get statistics for service ${service}:`, error);
+            throw new Error(`Failed to get statistics for service ${service}: ${error}`);
+        }
+    }
     /**
      * Check if the database is in read-only mode
      * @returns True if the database is in read-only mode, false otherwise
@@ -2963,6 +3317,29 @@ export class BrainyData {
             this.writeOnly = false;
         }
     }
+    /**
+     * Check if the database is frozen (completely immutable)
+     * @returns True if the database is frozen, false otherwise
+     */
+    isFrozen() {
+        return this.frozen;
+    }
+    /**
+     * Set the database to frozen mode (completely immutable)
+     * When frozen, no changes are allowed including statistics updates and index optimizations
+     * @param frozen True to freeze the database, false to allow optimizations
+     */
+    setFrozen(frozen) {
+        this.frozen = frozen;
+        // If unfreezing and real-time updates are configured, restart them
+        if (!frozen && this.realtimeUpdateConfig.enabled && this.isInitialized) {
+            this.startRealtimeUpdates();
+        }
+        // If freezing, stop real-time updates
+        else if (frozen && this.updateTimerId !== null) {
+            this.stopRealtimeUpdates();
+        }
+    }
     /**
      * Check if the database is in write-only mode
      * @returns True if the database is in write-only mode, false otherwise
@@ -3090,7 +3467,7 @@ export class BrainyData {
             // First use the HNSW index to find similar vectors efficiently
             const searchResults = await this.index.search(queryVector, k * 2);
             // Get all verbs for filtering
-            const allVerbs = await this.storage.getAllVerbs();
+            const allVerbs = await this.getAllVerbs();
             // Create a map of verb IDs for faster lookup
             const verbMap = new Map();
             for (const verb of allVerbs) {
@@ -3247,6 +3624,83 @@ export class BrainyData {
             throw new Error(`Failed to search nouns by verbs: ${error}`);
         }
     }
+    /**
+     * Get available filter values for a field
+     * Useful for building dynamic filter UIs
+     *
+     * @param field The field name to get values for
+     * @returns Array of available values for that field
+     */
+    async getFilterValues(field) {
+        await this.ensureInitialized();
+        if (!this.metadataIndex) {
+            return [];
+        }
+        return this.metadataIndex.getFilterValues(field);
+    }
+    /**
+     * Get all available filter fields
+     * Useful for discovering what metadata fields are indexed
+     *
+     * @returns Array of indexed field names
+     */
+    async getFilterFields() {
+        await this.ensureInitialized();
+        if (!this.metadataIndex) {
+            return [];
+        }
+        return this.metadataIndex.getFilterFields();
+    }
+    /**
+     * Search within a specific set of items
+     * This is useful when you've pre-filtered items and want to search only within them
+     *
+     * @param queryVectorOrData Query vector or data to search for
+     * @param itemIds Array of item IDs to search within
+     * @param k Number of results to return
+     * @param options Additional options
+     * @returns Array of search results
+     */
+    async searchWithinItems(queryVectorOrData, itemIds, k = 10, options = {}) {
+        await this.ensureInitialized();
+        // Check if database is in write-only mode
+        this.checkWriteOnly();
+        // Create a Set for fast lookups
+        const allowedIds = new Set(itemIds);
+        // Create filter function that only allows specified items
+        const filterFunction = async (id) => allowedIds.has(id);
+        // Get query vector
+        let queryVector;
+        if (Array.isArray(queryVectorOrData) && !options.forceEmbed) {
+            queryVector = queryVectorOrData;
+        }
+        else {
+            queryVector = await this.embeddingFunction(queryVectorOrData);
+        }
+        // Search with the filter
+        const results = await this.index.search(queryVector, Math.min(k, itemIds.length), filterFunction);
+        // Get metadata for each result
+        const searchResults = [];
+        for (const [id, score] of results) {
+            const noun = this.index.getNouns().get(id);
+            if (!noun)
+                continue;
+            let metadata = await this.storage.getMetadata(id);
+            if (metadata === null) {
+                metadata = {};
+            }
+            if (metadata && typeof metadata === 'object') {
+                metadata = { ...metadata, id };
+            }
+            searchResults.push({
+                id,
+                score,
+                vector: noun.vector,
+                metadata: metadata
+            });
+        }
+        return searchResults;
+    }
     /**
      * Search for similar documents using a text query
      * This is a convenience method that embeds the query text and performs a search
@@ -3264,11 +3718,13 @@ export class BrainyData {
         try {
             // Embed the query text
             const queryVector = await this.embed(query);
-            // Search using the embedded vector
+            // Search using the embedded vector with metadata filtering
             const results = await this.search(queryVector, k, {
                 nounTypes: options.nounTypes,
                 includeVerbs: options.includeVerbs,
-                searchMode: options.searchMode
+                searchMode: options.searchMode,
+                metadata: options.metadata,
+                forceEmbed: false // Already embedded
             });
             // Track search performance
             const duration = Date.now() - searchStartTime;
@@ -3981,6 +4437,20 @@ export class BrainyData {
             clearInterval(this.updateTimerId);
             this.updateTimerId = null;
         }
+        // Stop maintenance intervals
+        for (const intervalId of this.maintenanceIntervals) {
+            clearInterval(intervalId);
+        }
+        this.maintenanceIntervals = [];
+        // Flush metadata index one last time
+        if (this.metadataIndex) {
+            try {
+                await this.metadataIndex.flush();
+            }
+            catch (error) {
+                console.warn('Error flushing metadata index during cleanup:', error);
+            }
+        }
         // Clean up distributed mode resources
         if (this.healthMonitor) {
             this.healthMonitor.stop();