@soulcraft/brainy 3.44.0 → 3.46.0

package/dist/utils/metadataIndex.js

@@ -6,6 +6,7 @@
 import { MetadataIndexCache } from './metadataIndexCache.js';
 import { prodLog } from './logger.js';
 import { getGlobalCache } from './unifiedCache.js';
+import { TypeUtils, NOUN_TYPE_COUNT, VERB_TYPE_COUNT } from '../types/graphTypes.js';
 import { SparseIndex, ChunkManager, AdaptiveChunkingStrategy } from './metadataIndexChunking.js';
 import { EntityIdMapper } from './entityIdMapper.js';
 import { RoaringBitmap32 } from 'roaring-wasm';
@@ -27,14 +28,17 @@ export class MetadataIndexManager {
         // Type-Field Affinity Tracking for intelligent NLP
         this.typeFieldAffinity = new Map(); // nounType -> field -> count
         this.totalEntitiesByType = new Map(); // nounType -> total count
+        // Phase 1b: Fixed-size type tracking (99.76% memory reduction vs Maps)
+        // Uint32Array provides O(1) access via type enum index
+        // 31 noun types × 4 bytes = 124 bytes (vs ~15KB with Map overhead)
+        // 40 verb types × 4 bytes = 160 bytes (vs ~20KB with Map overhead)
+        // Total: 284 bytes (vs ~35KB) = 99.2% memory reduction
+        this.entityCountsByTypeFixed = new Uint32Array(NOUN_TYPE_COUNT); // 124 bytes
+        this.verbCountsByTypeFixed = new Uint32Array(VERB_TYPE_COUNT); // 160 bytes
         // File locking for concurrent write protection (prevents race conditions)
         this.activeLocks = new Map();
         this.lockPromises = new Map();
         this.lockTimers = new Map(); // Track timers for cleanup
-        // Adaptive Chunked Sparse Indexing (v3.42.0)
-        // Reduces file count from 560k → 89 files (630x reduction)
-        // ALL fields now use chunking - no more flat files
-        this.sparseIndices = new Map(); // field -> sparse index
         this.storage = storage;
         this.config = {
             maxIndexSize: config.maxIndexSize ?? 10000,
@@ -87,6 +91,77 @@ export class MetadataIndexManager {
     async init() {
         // Initialize EntityIdMapper (loads UUID ↔ integer mappings from storage)
         await this.idMapper.init();
+        // Phase 1b: Sync loaded counts to fixed-size arrays
+        // This populates the Uint32Arrays from the Maps loaded by lazyLoadCounts()
+        this.syncTypeCountsToFixed();
+        // Warm the cache with common fields (v3.44.1 - lazy loading optimization)
+        await this.warmCache();
+    }
+    /**
+     * Warm the cache by preloading common field sparse indices (v3.44.1)
+     * This improves cache hit rates by loading frequently-accessed fields at startup
+     * Target: >80% cache hit rate for typical workloads
+     */
+    async warmCache() {
+        // Common fields used in most queries
+        const commonFields = ['noun', 'type', 'service', 'createdAt'];
+        prodLog.debug(`🔥 Warming metadata cache with common fields: ${commonFields.join(', ')}`);
+        // Preload in parallel for speed
+        await Promise.all(commonFields.map(async (field) => {
+            try {
+                await this.loadSparseIndex(field);
+            }
+            catch (error) {
+                // Silently ignore if field doesn't exist yet
+                // This maintains zero-configuration principle
+                prodLog.debug(`Cache warming: field '${field}' not yet indexed`);
+            }
+        }));
+        prodLog.debug('✅ Metadata cache warmed successfully');
+        // Phase 1b: Also warm cache for top types (type-aware optimization)
+        await this.warmCacheForTopTypes(3);
+    }
+    /**
+     * Phase 1b: Warm cache for top types (type-aware optimization)
+     * Preloads metadata indices for the most common entity types and their top fields
+     * This significantly improves query performance for the most frequently accessed data
+     *
+     * @param topN Number of top types to warm (default: 3)
+     */
+    async warmCacheForTopTypes(topN = 3) {
+        // Get top noun types by entity count
+        const topTypes = this.getTopNounTypes(topN);
+        if (topTypes.length === 0) {
+            prodLog.debug('⏭️  Skipping type-aware cache warming: no types found yet');
+            return;
+        }
+        prodLog.debug(`🔥 Warming cache for top ${topTypes.length} types: ${topTypes.join(', ')}`);
+        // For each top type, warm cache for its top fields
+        for (const type of topTypes) {
+            // Get fields with high affinity to this type
+            const typeFields = this.typeFieldAffinity.get(type);
+            if (!typeFields)
+                continue;
+            // Sort fields by count (most common first)
+            const topFields = Array.from(typeFields.entries())
+                .sort((a, b) => b[1] - a[1])
+                .slice(0, 5) // Top 5 fields per type
+                .map(([field]) => field);
+            if (topFields.length === 0)
+                continue;
+            prodLog.debug(`  📊 Type '${type}' - warming fields: ${topFields.join(', ')}`);
+            // Preload sparse indices for these fields in parallel
+            await Promise.all(topFields.map(async (field) => {
+                try {
+                    await this.loadSparseIndex(field);
+                }
+                catch (error) {
+                    // Silently ignore if field doesn't exist yet
+                    prodLog.debug(`  ⏭️  Field '${field}' not yet indexed for type '${type}'`);
+                }
+            }));
+        }
+        prodLog.debug('✅ Type-aware cache warming completed');
     }
     /**
      * Acquire an in-memory lock for coordinating concurrent metadata index writes
@@ -166,6 +241,49 @@ export class MetadataIndexManager {
             // This maintains zero-configuration principle
         }
     }
+    /**
+     * Phase 1b: Sync Map-based counts to fixed-size Uint32Arrays
+     * This enables gradual migration from Maps to arrays while maintaining backward compatibility
+     * Called periodically and on demand to keep both representations in sync
+     */
+    syncTypeCountsToFixed() {
+        // Sync noun counts from totalEntitiesByType Map to entityCountsByTypeFixed array
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const type = TypeUtils.getNounFromIndex(i);
+            const count = this.totalEntitiesByType.get(type) || 0;
+            this.entityCountsByTypeFixed[i] = count;
+        }
+        // Sync verb counts from totalEntitiesByType Map to verbCountsByTypeFixed array
+        // Note: Verb counts are currently tracked alongside noun counts in totalEntitiesByType
+        // In the future, we may want a separate Map for verb counts
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const type = TypeUtils.getVerbFromIndex(i);
+            const count = this.totalEntitiesByType.get(type) || 0;
+            this.verbCountsByTypeFixed[i] = count;
+        }
+    }
+    /**
+     * Phase 1b: Sync from fixed-size arrays back to Maps (reverse direction)
+     * Used when Uint32Arrays are the source of truth and need to update Maps
+     */
+    syncTypeCountsFromFixed() {
+        // Sync noun counts from array to Map
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const count = this.entityCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getNounFromIndex(i);
+                this.totalEntitiesByType.set(type, count);
+            }
+        }
+        // Sync verb counts from array to Map
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const count = this.verbCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getVerbFromIndex(i);
+                this.totalEntitiesByType.set(type, count);
+            }
+        }
+    }
     /**
      * Update cardinality statistics for a field
      */
@@ -303,16 +421,13 @@ export class MetadataIndexManager {
     }
     /**
      * Get IDs for a value using chunked sparse index with roaring bitmaps (v3.43.0)
+     * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
      */
     async getIdsFromChunks(field, value) {
-        // Load sparse index
-        let sparseIndex = this.sparseIndices.get(field);
+        // Load sparse index via UnifiedCache (lazy loading)
+        const sparseIndex = await this.loadSparseIndex(field);
         if (!sparseIndex) {
-            sparseIndex = await this.loadSparseIndex(field);
-            if (!sparseIndex) {
-                return []; // No chunked index exists yet
-            }
-            this.sparseIndices.set(field, sparseIndex);
+            return []; // No chunked index exists yet
         }
         // Find candidate chunks using zone maps and bloom filters
         const normalizedValue = this.normalizeValue(value, field);
@@ -339,16 +454,13 @@ export class MetadataIndexManager {
     }
     /**
      * Get IDs for a range using chunked sparse index with zone maps and roaring bitmaps (v3.43.0)
+     * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
      */
     async getIdsFromChunksForRange(field, min, max, includeMin = true, includeMax = true) {
-        // Load sparse index
-        let sparseIndex = this.sparseIndices.get(field);
+        // Load sparse index via UnifiedCache (lazy loading)
+        const sparseIndex = await this.loadSparseIndex(field);
         if (!sparseIndex) {
-            sparseIndex = await this.loadSparseIndex(field);
-            if (!sparseIndex) {
-                return []; // No chunked index exists yet
-            }
-            this.sparseIndices.set(field, sparseIndex);
+            return []; // No chunked index exists yet
         }
         // Find candidate chunks using zone maps
         const candidateChunkIds = sparseIndex.findChunksForRange(min, max);
@@ -384,17 +496,14 @@ export class MetadataIndexManager {
     /**
      * Get roaring bitmap for a field-value pair without converting to UUIDs (v3.43.0)
      * This is used for fast multi-field intersection queries using hardware-accelerated bitmap AND
+     * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
      * @returns RoaringBitmap32 containing integer IDs, or null if no matches
      */
     async getBitmapFromChunks(field, value) {
-        // Load sparse index
-        let sparseIndex = this.sparseIndices.get(field);
+        // Load sparse index via UnifiedCache (lazy loading)
+        const sparseIndex = await this.loadSparseIndex(field);
         if (!sparseIndex) {
-            sparseIndex = await this.loadSparseIndex(field);
-            if (!sparseIndex) {
-                return null; // No chunked index exists yet
-            }
-            this.sparseIndices.set(field, sparseIndex);
+            return null; // No chunked index exists yet
         }
         // Find candidate chunks using zone maps and bloom filters
         const normalizedValue = this.normalizeValue(value, field);
@@ -481,25 +590,22 @@ export class MetadataIndexManager {
     }
     /**
      * Add value-ID mapping to chunked index
+     * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
      */
     async addToChunkedIndex(field, value, id) {
-        // Load or create sparse index
-        let sparseIndex = this.sparseIndices.get(field);
+        // Load or create sparse index via UnifiedCache (lazy loading)
+        let sparseIndex = await this.loadSparseIndex(field);
         if (!sparseIndex) {
-            sparseIndex = await this.loadSparseIndex(field);
-            if (!sparseIndex) {
-                // Create new sparse index
-                const stats = this.fieldStats.get(field);
-                const chunkSize = stats
-                    ? this.chunkingStrategy.getOptimalChunkSize({
-                        uniqueValues: stats.cardinality.uniqueValues,
-                        distribution: stats.cardinality.distribution,
-                        avgIdsPerValue: stats.cardinality.totalValues / Math.max(1, stats.cardinality.uniqueValues)
-                    })
-                    : 50;
-                sparseIndex = new SparseIndex(field, chunkSize);
-            }
-            this.sparseIndices.set(field, sparseIndex);
+            // Create new sparse index
+            const stats = this.fieldStats.get(field);
+            const chunkSize = stats
+                ? this.chunkingStrategy.getOptimalChunkSize({
+                    uniqueValues: stats.cardinality.uniqueValues,
+                    distribution: stats.cardinality.distribution,
+                    avgIdsPerValue: stats.cardinality.totalValues / Math.max(1, stats.cardinality.uniqueValues)
+                })
+                : 50;
+            sparseIndex = new SparseIndex(field, chunkSize);
         }
         const normalizedValue = this.normalizeValue(value, field);
         // Find existing chunk for this value (check zone maps)
@@ -571,9 +677,11 @@ export class MetadataIndexManager {
     }
     /**
      * Remove ID from chunked index
+     * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
      */
     async removeFromChunkedIndex(field, value, id) {
-        const sparseIndex = this.sparseIndices.get(field) || await this.loadSparseIndex(field);
+        // Load sparse index via UnifiedCache (lazy loading)
+        const sparseIndex = await this.loadSparseIndex(field);
         if (!sparseIndex) {
             return; // No chunked index exists
         }
@@ -834,21 +942,25 @@ export class MetadataIndexManager {
             }
         }
         else {
-            // Remove from all indexes (slower, requires scanning all chunks)
+            // Remove from all indexes (slower, requires scanning all field indexes)
             // This should be rare - prefer providing metadata when removing
-            prodLog.warn(`Removing ID ${id} without metadata requires scanning all sparse indices (slow)`);
-            // Scan all sparse indices
-            for (const [field, sparseIndex] of this.sparseIndices.entries()) {
-                for (const chunkId of sparseIndex.getAllChunkIds()) {
-                    const chunk = await this.chunkManager.loadChunk(field, chunkId);
-                    if (chunk) {
-                        // Convert UUID to integer for bitmap checking
-                        const intId = this.idMapper.getInt(id);
-                        if (intId !== undefined) {
-                            // Check all values in this chunk
-                            for (const [value, bitmap] of chunk.entries) {
-                                if (bitmap.has(intId)) {
-                                    await this.removeFromChunkedIndex(field, value, id);
+            // v3.44.1: Scan via fieldIndexes, load sparse indices on-demand
+            prodLog.warn(`Removing ID ${id} without metadata requires scanning all fields (slow)`);
+            // Scan all fields via fieldIndexes
+            for (const field of this.fieldIndexes.keys()) {
+                const sparseIndex = await this.loadSparseIndex(field);
+                if (sparseIndex) {
+                    for (const chunkId of sparseIndex.getAllChunkIds()) {
+                        const chunk = await this.chunkManager.loadChunk(field, chunkId);
+                        if (chunk) {
+                            // Convert UUID to integer for bitmap checking
+                            const intId = this.idMapper.getInt(id);
+                            if (intId !== undefined) {
+                                // Check all values in this chunk
+                                for (const [value, bitmap] of chunk.entries) {
+                                    if (bitmap.has(intId)) {
+                                        await this.removeFromChunkedIndex(field, value, id);
+                                    }
                                 }
                             }
                         }
@@ -1087,9 +1199,10 @@ export class MetadataIndexManager {
                         case 'exists':
                             if (operand) {
                                 // Get all IDs that have this field (any value) from chunked sparse index with roaring bitmaps (v3.43.0)
+                                // v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
                                 const allIntIds = new Set();
-                                // Load sparse index for this field
-                                const sparseIndex = this.sparseIndices.get(field) || await this.loadSparseIndex(field);
+                                // Load sparse index via UnifiedCache (lazy loading)
+                                const sparseIndex = await this.loadSparseIndex(field);
                                 if (sparseIndex) {
                                     // Iterate through all chunks for this field
                                     for (const chunkId of sparseIndex.getAllChunkIds()) {
@@ -1333,6 +1446,102 @@ export class MetadataIndexManager {
     getAllEntityCounts() {
         return new Map(this.totalEntitiesByType);
     }
+    // ============================================================================
+    // Phase 1b: Type Enum Methods (O(1) access via Uint32Arrays)
+    // ============================================================================
+    /**
+     * Get entity count for a noun type using type enum (O(1) array access)
+     * More efficient than Map-based getEntityCountByType
+     * @param type Noun type from NounTypeEnum
+     * @returns Count of entities of this type
+     */
+    getEntityCountByTypeEnum(type) {
+        const index = TypeUtils.getNounIndex(type);
+        return this.entityCountsByTypeFixed[index];
+    }
+    /**
+     * Get verb count for a verb type using type enum (O(1) array access)
+     * @param type Verb type from VerbTypeEnum
+     * @returns Count of verbs of this type
+     */
+    getVerbCountByTypeEnum(type) {
+        const index = TypeUtils.getVerbIndex(type);
+        return this.verbCountsByTypeFixed[index];
+    }
+    /**
+     * Get top N noun types by entity count (using fixed-size arrays)
+     * Useful for type-aware cache warming and query optimization
+     * @param n Number of top types to return
+     * @returns Array of noun types sorted by count (highest first)
+     */
+    getTopNounTypes(n) {
+        const types = [];
+        // Iterate through all noun types
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const count = this.entityCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getNounFromIndex(i);
+                types.push({ type, count });
+            }
+        }
+        // Sort by count (descending) and return top N
+        return types
+            .sort((a, b) => b.count - a.count)
+            .slice(0, n)
+            .map(t => t.type);
+    }
+    /**
+     * Get top N verb types by count (using fixed-size arrays)
+     * @param n Number of top types to return
+     * @returns Array of verb types sorted by count (highest first)
+     */
+    getTopVerbTypes(n) {
+        const types = [];
+        // Iterate through all verb types
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const count = this.verbCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getVerbFromIndex(i);
+                types.push({ type, count });
+            }
+        }
+        // Sort by count (descending) and return top N
+        return types
+            .sort((a, b) => b.count - a.count)
+            .slice(0, n)
+            .map(t => t.type);
+    }
+    /**
+     * Get all noun type counts as a Map (using fixed-size arrays)
+     * More efficient than getAllEntityCounts for type-aware queries
+     * @returns Map of noun type to count
+     */
+    getAllNounTypeCounts() {
+        const counts = new Map();
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const count = this.entityCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getNounFromIndex(i);
+                counts.set(type, count);
+            }
+        }
+        return counts;
+    }
+    /**
+     * Get all verb type counts as a Map (using fixed-size arrays)
+     * @returns Map of verb type to count
+     */
+    getAllVerbTypeCounts() {
+        const counts = new Map();
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const count = this.verbCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getVerbFromIndex(i);
+                counts.set(type, count);
+            }
+        }
+        return counts;
+    }
     /**
      * Get count of entities matching field-value criteria - queries chunked sparse index
      */
@@ -1343,29 +1552,31 @@ export class MetadataIndexManager {
     }
     /**
      * Get index statistics with enhanced counting information
+     * v3.44.1: Sparse indices now lazy-loaded via UnifiedCache
+     * Note: This method may load sparse indices to calculate stats
      */
     async getStats() {
         const fields = new Set();
         let totalEntries = 0;
         let totalIds = 0;
-        // Collect stats from sparse indices (v3.42.0 - removed indexCache)
-        for (const [field, sparseIndex] of this.sparseIndices.entries()) {
+        // Collect stats from field indexes (lightweight - always in memory)
+        for (const field of this.fieldIndexes.keys()) {
             fields.add(field);
-            // Count entries and IDs from all chunks
-            for (const chunkId of sparseIndex.getAllChunkIds()) {
-                const chunk = await this.chunkManager.loadChunk(field, chunkId);
-                if (chunk) {
-                    totalEntries += chunk.entries.size;
-                    for (const ids of chunk.entries.values()) {
-                        totalIds += ids.size;
+            // Load sparse index to count entries (may trigger lazy load)
+            const sparseIndex = await this.loadSparseIndex(field);
+            if (sparseIndex) {
+                // Count entries and IDs from all chunks
+                for (const chunkId of sparseIndex.getAllChunkIds()) {
+                    const chunk = await this.chunkManager.loadChunk(field, chunkId);
+                    if (chunk) {
+                        totalEntries += chunk.entries.size;
+                        for (const ids of chunk.entries.values()) {
+                            totalIds += ids.size;
+                        }
                     }
                 }
             }
         }
-        // Also include fields from fieldIndexes that might not have sparse indices yet
-        for (const field of this.fieldIndexes.keys()) {
-            fields.add(field);
-        }
         return {
             totalEntries,
             totalIds,
@@ -1377,6 +1588,7 @@ export class MetadataIndexManager {
     /**
      * Rebuild entire index from scratch using pagination
      * Non-blocking version that yields control back to event loop
+     * v3.44.1: Sparse indices now lazy-loaded via UnifiedCache (no need to clear Map)
      */
     async rebuild() {
         if (this.isRebuilding)
@@ -1387,9 +1599,12 @@ export class MetadataIndexManager {
             prodLog.info(`📊 Storage adapter: ${this.storage.constructor.name}`);
             prodLog.info(`🔧 Batch processing available: ${!!this.storage.getMetadataBatch}`);
             // Clear existing indexes (v3.42.0 - use sparse indices instead of flat files)
-            this.sparseIndices.clear();
+            // v3.44.1: No sparseIndices Map to clear - UnifiedCache handles eviction
             this.fieldIndexes.clear();
             this.dirtyFields.clear();
+            // Clear all cached sparse indices in UnifiedCache
+            // This ensures rebuild starts fresh (v3.44.1)
+            this.unifiedCache.clear('metadata');
             // Rebuild noun metadata indexes using pagination
             let nounOffset = 0;
             const nounLimit = 25; // Even smaller batches during initialization to prevent socket exhaustion
@@ -1742,7 +1957,17 @@ export class MetadataIndexManager {
             typeFields.set(field, currentCount + 1);
             // Update total entities of this type (only count once per entity)
             if (field === 'noun') {
-                this.totalEntitiesByType.set(entityType, this.totalEntitiesByType.get(entityType) + 1);
+                const newCount = this.totalEntitiesByType.get(entityType) + 1;
+                this.totalEntitiesByType.set(entityType, newCount);
+                // Phase 1b: Also update fixed-size array
+                // Try to parse as noun type - if it matches a known type, update the array
+                try {
+                    const nounTypeIndex = TypeUtils.getNounIndex(entityType);
+                    this.entityCountsByTypeFixed[nounTypeIndex] = newCount;
+                }
+                catch {
+                    // Not a recognized noun type, skip fixed-size array update
+                }
             }
         }
         else if (operation === 'remove') {
@@ -1758,11 +1983,28 @@ export class MetadataIndexManager {
             if (field === 'noun') {
                 const total = this.totalEntitiesByType.get(entityType);
                 if (total > 1) {
-                    this.totalEntitiesByType.set(entityType, total - 1);
+                    const newCount = total - 1;
+                    this.totalEntitiesByType.set(entityType, newCount);
+                    // Phase 1b: Also update fixed-size array
+                    try {
+                        const nounTypeIndex = TypeUtils.getNounIndex(entityType);
+                        this.entityCountsByTypeFixed[nounTypeIndex] = newCount;
+                    }
+                    catch {
+                        // Not a recognized noun type, skip fixed-size array update
+                    }
                 }
                 else {
                     this.totalEntitiesByType.delete(entityType);
                     this.typeFieldAffinity.delete(entityType);
+                    // Phase 1b: Also zero out fixed-size array
+                    try {
+                        const nounTypeIndex = TypeUtils.getNounIndex(entityType);
+                        this.entityCountsByTypeFixed[nounTypeIndex] = 0;
+                    }
+                    catch {
+                        // Not a recognized noun type, skip fixed-size array update
+                    }
                 }
             }
         }