@soulcraft/brainy 3.45.0 → 3.47.0

package/dist/utils/metadataIndex.js

@@ -6,6 +6,7 @@
 import { MetadataIndexCache } from './metadataIndexCache.js';
 import { prodLog } from './logger.js';
 import { getGlobalCache } from './unifiedCache.js';
+import { TypeUtils, NOUN_TYPE_COUNT, VERB_TYPE_COUNT } from '../types/graphTypes.js';
 import { SparseIndex, ChunkManager, AdaptiveChunkingStrategy } from './metadataIndexChunking.js';
 import { EntityIdMapper } from './entityIdMapper.js';
 import { RoaringBitmap32 } from 'roaring-wasm';
@@ -27,6 +28,13 @@ export class MetadataIndexManager {
         // Type-Field Affinity Tracking for intelligent NLP
         this.typeFieldAffinity = new Map(); // nounType -> field -> count
         this.totalEntitiesByType = new Map(); // nounType -> total count
+        // Phase 1b: Fixed-size type tracking (99.76% memory reduction vs Maps)
+        // Uint32Array provides O(1) access via type enum index
+        // 31 noun types × 4 bytes = 124 bytes (vs ~15KB with Map overhead)
+        // 40 verb types × 4 bytes = 160 bytes (vs ~20KB with Map overhead)
+        // Total: 284 bytes (vs ~35KB) = 99.2% memory reduction
+        this.entityCountsByTypeFixed = new Uint32Array(NOUN_TYPE_COUNT); // 124 bytes
+        this.verbCountsByTypeFixed = new Uint32Array(VERB_TYPE_COUNT); // 160 bytes
         // File locking for concurrent write protection (prevents race conditions)
         this.activeLocks = new Map();
         this.lockPromises = new Map();
@@ -83,6 +91,9 @@ export class MetadataIndexManager {
     async init() {
         // Initialize EntityIdMapper (loads UUID ↔ integer mappings from storage)
         await this.idMapper.init();
+        // Phase 1b: Sync loaded counts to fixed-size arrays
+        // This populates the Uint32Arrays from the Maps loaded by lazyLoadCounts()
+        this.syncTypeCountsToFixed();
         // Warm the cache with common fields (v3.44.1 - lazy loading optimization)
         await this.warmCache();
     }
@@ -107,6 +118,50 @@ export class MetadataIndexManager {
             }
         }));
         prodLog.debug('✅ Metadata cache warmed successfully');
+        // Phase 1b: Also warm cache for top types (type-aware optimization)
+        await this.warmCacheForTopTypes(3);
+    }
+    /**
+     * Phase 1b: Warm cache for top types (type-aware optimization)
+     * Preloads metadata indices for the most common entity types and their top fields
+     * This significantly improves query performance for the most frequently accessed data
+     *
+     * @param topN Number of top types to warm (default: 3)
+     */
+    async warmCacheForTopTypes(topN = 3) {
+        // Get top noun types by entity count
+        const topTypes = this.getTopNounTypes(topN);
+        if (topTypes.length === 0) {
+            prodLog.debug('⏭️  Skipping type-aware cache warming: no types found yet');
+            return;
+        }
+        prodLog.debug(`🔥 Warming cache for top ${topTypes.length} types: ${topTypes.join(', ')}`);
+        // For each top type, warm cache for its top fields
+        for (const type of topTypes) {
+            // Get fields with high affinity to this type
+            const typeFields = this.typeFieldAffinity.get(type);
+            if (!typeFields)
+                continue;
+            // Sort fields by count (most common first)
+            const topFields = Array.from(typeFields.entries())
+                .sort((a, b) => b[1] - a[1])
+                .slice(0, 5) // Top 5 fields per type
+                .map(([field]) => field);
+            if (topFields.length === 0)
+                continue;
+            prodLog.debug(`  📊 Type '${type}' - warming fields: ${topFields.join(', ')}`);
+            // Preload sparse indices for these fields in parallel
+            await Promise.all(topFields.map(async (field) => {
+                try {
+                    await this.loadSparseIndex(field);
+                }
+                catch (error) {
+                    // Silently ignore if field doesn't exist yet
+                    prodLog.debug(`  ⏭️  Field '${field}' not yet indexed for type '${type}'`);
+                }
+            }));
+        }
+        prodLog.debug('✅ Type-aware cache warming completed');
     }
     /**
      * Acquire an in-memory lock for coordinating concurrent metadata index writes
@@ -186,6 +241,49 @@ export class MetadataIndexManager {
             // This maintains zero-configuration principle
         }
     }
+    /**
+     * Phase 1b: Sync Map-based counts to fixed-size Uint32Arrays
+     * This enables gradual migration from Maps to arrays while maintaining backward compatibility
+     * Called periodically and on demand to keep both representations in sync
+     */
+    syncTypeCountsToFixed() {
+        // Sync noun counts from totalEntitiesByType Map to entityCountsByTypeFixed array
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const type = TypeUtils.getNounFromIndex(i);
+            const count = this.totalEntitiesByType.get(type) || 0;
+            this.entityCountsByTypeFixed[i] = count;
+        }
+        // Sync verb counts from totalEntitiesByType Map to verbCountsByTypeFixed array
+        // Note: Verb counts are currently tracked alongside noun counts in totalEntitiesByType
+        // In the future, we may want a separate Map for verb counts
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const type = TypeUtils.getVerbFromIndex(i);
+            const count = this.totalEntitiesByType.get(type) || 0;
+            this.verbCountsByTypeFixed[i] = count;
+        }
+    }
+    /**
+     * Phase 1b: Sync from fixed-size arrays back to Maps (reverse direction)
+     * Used when Uint32Arrays are the source of truth and need to update Maps
+     */
+    syncTypeCountsFromFixed() {
+        // Sync noun counts from array to Map
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const count = this.entityCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getNounFromIndex(i);
+                this.totalEntitiesByType.set(type, count);
+            }
+        }
+        // Sync verb counts from array to Map
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const count = this.verbCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getVerbFromIndex(i);
+                this.totalEntitiesByType.set(type, count);
+            }
+        }
+    }
     /**
      * Update cardinality statistics for a field
      */
@@ -1348,6 +1446,102 @@ export class MetadataIndexManager {
     getAllEntityCounts() {
         return new Map(this.totalEntitiesByType);
     }
+    // ============================================================================
+    // Phase 1b: Type Enum Methods (O(1) access via Uint32Arrays)
+    // ============================================================================
+    /**
+     * Get entity count for a noun type using type enum (O(1) array access)
+     * More efficient than Map-based getEntityCountByType
+     * @param type Noun type from NounTypeEnum
+     * @returns Count of entities of this type
+     */
+    getEntityCountByTypeEnum(type) {
+        const index = TypeUtils.getNounIndex(type);
+        return this.entityCountsByTypeFixed[index];
+    }
+    /**
+     * Get verb count for a verb type using type enum (O(1) array access)
+     * @param type Verb type from VerbTypeEnum
+     * @returns Count of verbs of this type
+     */
+    getVerbCountByTypeEnum(type) {
+        const index = TypeUtils.getVerbIndex(type);
+        return this.verbCountsByTypeFixed[index];
+    }
+    /**
+     * Get top N noun types by entity count (using fixed-size arrays)
+     * Useful for type-aware cache warming and query optimization
+     * @param n Number of top types to return
+     * @returns Array of noun types sorted by count (highest first)
+     */
+    getTopNounTypes(n) {
+        const types = [];
+        // Iterate through all noun types
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const count = this.entityCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getNounFromIndex(i);
+                types.push({ type, count });
+            }
+        }
+        // Sort by count (descending) and return top N
+        return types
+            .sort((a, b) => b.count - a.count)
+            .slice(0, n)
+            .map(t => t.type);
+    }
+    /**
+     * Get top N verb types by count (using fixed-size arrays)
+     * @param n Number of top types to return
+     * @returns Array of verb types sorted by count (highest first)
+     */
+    getTopVerbTypes(n) {
+        const types = [];
+        // Iterate through all verb types
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const count = this.verbCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getVerbFromIndex(i);
+                types.push({ type, count });
+            }
+        }
+        // Sort by count (descending) and return top N
+        return types
+            .sort((a, b) => b.count - a.count)
+            .slice(0, n)
+            .map(t => t.type);
+    }
+    /**
+     * Get all noun type counts as a Map (using fixed-size arrays)
+     * More efficient than getAllEntityCounts for type-aware queries
+     * @returns Map of noun type to count
+     */
+    getAllNounTypeCounts() {
+        const counts = new Map();
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const count = this.entityCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getNounFromIndex(i);
+                counts.set(type, count);
+            }
+        }
+        return counts;
+    }
+    /**
+     * Get all verb type counts as a Map (using fixed-size arrays)
+     * @returns Map of verb type to count
+     */
+    getAllVerbTypeCounts() {
+        const counts = new Map();
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const count = this.verbCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getVerbFromIndex(i);
+                counts.set(type, count);
+            }
+        }
+        return counts;
+    }
     /**
      * Get count of entities matching field-value criteria - queries chunked sparse index
      */
@@ -1763,7 +1957,17 @@ export class MetadataIndexManager {
             typeFields.set(field, currentCount + 1);
             // Update total entities of this type (only count once per entity)
             if (field === 'noun') {
-                this.totalEntitiesByType.set(entityType, this.totalEntitiesByType.get(entityType) + 1);
+                const newCount = this.totalEntitiesByType.get(entityType) + 1;
+                this.totalEntitiesByType.set(entityType, newCount);
+                // Phase 1b: Also update fixed-size array
+                // Try to parse as noun type - if it matches a known type, update the array
+                try {
+                    const nounTypeIndex = TypeUtils.getNounIndex(entityType);
+                    this.entityCountsByTypeFixed[nounTypeIndex] = newCount;
+                }
+                catch {
+                    // Not a recognized noun type, skip fixed-size array update
+                }
             }
         }
         else if (operation === 'remove') {
@@ -1779,11 +1983,28 @@ export class MetadataIndexManager {
             if (field === 'noun') {
                 const total = this.totalEntitiesByType.get(entityType);
                 if (total > 1) {
-                    this.totalEntitiesByType.set(entityType, total - 1);
+                    const newCount = total - 1;
+                    this.totalEntitiesByType.set(entityType, newCount);
+                    // Phase 1b: Also update fixed-size array
+                    try {
+                        const nounTypeIndex = TypeUtils.getNounIndex(entityType);
+                        this.entityCountsByTypeFixed[nounTypeIndex] = newCount;
+                    }
+                    catch {
+                        // Not a recognized noun type, skip fixed-size array update
+                    }
                 }
                 else {
                     this.totalEntitiesByType.delete(entityType);
                     this.typeFieldAffinity.delete(entityType);
+                    // Phase 1b: Also zero out fixed-size array
+                    try {
+                        const nounTypeIndex = TypeUtils.getNounIndex(entityType);
+                        this.entityCountsByTypeFixed[nounTypeIndex] = 0;
+                    }
+                    catch {
+                        // Not a recognized noun type, skip fixed-size array update
+                    }
                 }
             }
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.45.0",
+  "version": "3.47.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",