@soulcraft/cortex 1.3.1

package/dist/utils/NativeMetadataIndex.js

@@ -0,0 +1,1274 @@
+/**
+ * NativeMetadataIndex — TypeScript wrapper around the Rust NativeMetadataIndex.
+ *
+ * Implements the same public API as the old MetadataIndexManager (3,721 lines),
+ * delegating core operations (query, mutation, normalization) to Rust.
+ *
+ * Architecture:
+ * - Rust owns: bitmap operations, filter evaluation, value normalization,
+ *   field extraction, chunk management, text search, entity ID mapping
+ * - TS owns: async storage I/O, lazy field loading, rebuild orchestration,
+ *   flush, getSortedIdsForFilter (loads sort values from storage)
+ *
+ * Buffer exchange pattern: TS loads data from storage, passes JSON to Rust.
+ * Rust operates in-memory, returns serialized state for TS to persist.
+ */
+import { prodLog, getGlobalCache, FieldTypeInference } from '@soulcraft/brainy/internals';
+import { TypeUtils, NOUN_TYPE_COUNT, VERB_TYPE_COUNT } from '@soulcraft/brainy/types/graphTypes';
+import { loadNativeModule } from '../native/index.js';
+/**
+ * MetadataIndexManager — native Rust implementation with TS storage bridge.
+ *
+ * Drop-in replacement for the old pure-TS MetadataIndexManager.
+ * All bitmap AND/OR/NOT operations execute in Rust without crossing FFI.
+ */
+export class MetadataIndexManager {
+    storage;
+    config;
+    native;
+    isRebuilding = false;
+    lastFlushTime = Date.now();
+    autoFlushThreshold = 10;
+    dirtyFields = new Set();
+    // Lazy field loading: tracks which fields have been loaded into Rust
+    loadedFields = new Set();
+    // Tracks which field indexes are known (from field registry)
+    knownFields = new Set();
+    // TS-only analytics (not in Rust)
+    fieldStats = new Map();
+    typeFieldAffinity = new Map();
+    totalEntitiesByType = new Map();
+    entityCountsByTypeFixed = new Uint32Array(NOUN_TYPE_COUNT);
+    verbCountsByTypeFixed = new Uint32Array(VERB_TYPE_COUNT);
+    // Unified cache for coordinated memory management
+    unifiedCache;
+    // Field Type Inference
+    fieldTypeInference;
+    constructor(storage, config = {}) {
+        this.storage = storage;
+        this.config = {
+            maxIndexSize: config.maxIndexSize ?? 10000,
+            rebuildThreshold: config.rebuildThreshold ?? 0.1,
+            autoOptimize: config.autoOptimize ?? true,
+            indexedFields: config.indexedFields ?? [],
+            excludeFields: config.excludeFields ?? [
+                'embedding', 'vector', 'embeddings', 'vectors',
+                'content', 'data', 'originalData', '_data',
+                'id'
+            ]
+        };
+        // Build Rust config JSON
+        const nativeConfig = {};
+        if (this.config.excludeFields.length > 0) {
+            nativeConfig.excludeFields = this.config.excludeFields;
+        }
+        if (this.config.indexedFields.length > 0) {
+            nativeConfig.indexedFields = this.config.indexedFields;
+        }
+        const bindings = loadNativeModule();
+        this.native = new bindings.NativeMetadataIndex(Object.keys(nativeConfig).length > 0 ? JSON.stringify(nativeConfig) : null);
+        this.unifiedCache = getGlobalCache();
+        this.fieldTypeInference = new FieldTypeInference(storage);
+    }
+    // ==========================================================================
+    // Initialization
+    // ==========================================================================
+    async init() {
+        // Load field registry to discover persisted indices
+        await this.loadFieldRegistry();
+        // Load entity ID mapper
+        await this.loadEntityIdMapper();
+        const hasFields = this.knownFields.size > 0;
+        if (hasFields) {
+            await this.warmCache();
+            await this.lazyLoadCounts();
+            this.syncTypeCountsToFixed();
+        }
+    }
+    // ==========================================================================
+    // Storage I/O helpers
+    // ==========================================================================
+    async loadEntityIdMapper() {
+        try {
+            const data = await this.storage.getMetadata('brainy:entityIdMapper');
+            if (data && data.nextId !== undefined) {
+                this.native.loadEntityIdMapper(JSON.stringify(data));
+            }
+        }
+        catch {
+            // First time — mapper starts empty
+        }
+    }
+    async saveEntityIdMapper() {
+        if (!this.native.isEntityIdMapperDirty())
+            return;
+        const json = this.native.saveEntityIdMapper();
+        const data = JSON.parse(json);
+        await this.storage.saveMetadata('brainy:entityIdMapper', data);
+        // Reload to clear dirty flag
+        this.native.loadEntityIdMapper(json);
+    }
+    async loadFieldRegistry() {
+        try {
+            const registry = await this.storage.getMetadata('__metadata_field_registry__');
+            if (!registry?.fields || !Array.isArray(registry.fields)) {
+                prodLog.debug('No field registry found - will build on first flush');
+                return;
+            }
+            // Load into Rust
+            this.native.loadFieldRegistry(JSON.stringify(registry));
+            // Track known fields
+            for (const field of registry.fields) {
+                if (typeof field === 'string' && field.length > 0) {
+                    this.knownFields.add(field);
+                }
+            }
+            prodLog.info(`Loaded field registry: ${registry.fields.length} persisted fields discovered\n` +
+                `   Fields: ${registry.fields.slice(0, 5).join(', ')}${registry.fields.length > 5 ? '...' : ''}`);
+        }
+        catch (error) {
+            prodLog.debug('Could not load field registry:', error);
+        }
+    }
+    async saveFieldRegistry() {
+        if (this.knownFields.size === 0)
+            return;
+        try {
+            const json = this.native.saveFieldRegistry();
+            const data = JSON.parse(json);
+            await this.storage.saveMetadata('__metadata_field_registry__', data);
+        }
+        catch (error) {
+            prodLog.warn('Failed to save field registry:', error);
+        }
+    }
+    /**
+     * Ensure a field's sparse index + all chunks are loaded into Rust.
+     * Matches the lazy-loading pattern of the old UnifiedCache approach.
+     */
+    async ensureFieldLoaded(field) {
+        if (this.loadedFields.has(field))
+            return;
+        const indexPath = `__sparse_index__${field}`;
+        try {
+            const sparseData = await this.storage.getMetadata(indexPath);
+            if (sparseData) {
+                this.native.loadSparseIndex(field, JSON.stringify(sparseData));
+                const chunkIds = this.native.getSparseIndexChunkIds(field);
+                // Load all chunks in parallel
+                await Promise.all(chunkIds.map(async (chunkId) => {
+                    if (!this.native.isChunkLoaded(field, chunkId)) {
+                        const chunkPath = `__chunk__${field}_${chunkId}`;
+                        const chunkData = await this.storage.getMetadata(chunkPath);
+                        if (chunkData) {
+                            this.native.loadChunk(field, chunkId, JSON.stringify(chunkData));
+                        }
+                    }
+                }));
+            }
+        }
+        catch (error) {
+            prodLog.debug(`Failed to load field '${field}':`, error);
+        }
+        this.loadedFields.add(field);
+    }
+    /**
+     * Ensure multiple fields are loaded (parallel).
+     */
+    async ensureFieldsLoaded(fields) {
+        const unloaded = fields.filter(f => !this.loadedFields.has(f));
+        if (unloaded.length === 0)
+            return;
+        await Promise.all(unloaded.map(f => this.ensureFieldLoaded(f)));
+    }
+    /**
+     * Extract filter field names for lazy loading.
+     */
+    extractFilterFields(filter) {
+        if (!filter || typeof filter !== 'object')
+            return [];
+        const fields = [];
+        if (filter.allOf && Array.isArray(filter.allOf)) {
+            for (const sub of filter.allOf) {
+                fields.push(...this.extractFilterFields(sub));
+            }
+        }
+        if (filter.anyOf && Array.isArray(filter.anyOf)) {
+            for (const sub of filter.anyOf) {
+                fields.push(...this.extractFilterFields(sub));
+            }
+        }
+        for (const key of Object.keys(filter)) {
+            if (key !== 'allOf' && key !== 'anyOf' && key !== 'not') {
+                fields.push(key);
+            }
+        }
+        return [...new Set(fields)];
+    }
+    // ==========================================================================
+    // Dirty data persistence
+    // ==========================================================================
+    /**
+     * Persist dirty chunks, sparse indices, field indexes from a MutationResult.
+     */
+    async persistMutationResult(result) {
+        const promises = [];
+        // Save dirty chunks
+        for (const { field, chunkId } of result.dirtyChunks) {
+            const json = this.native.saveChunk(field, chunkId);
+            if (json) {
+                const chunkPath = `__chunk__${field}_${chunkId}`;
+                promises.push(this.storage.saveMetadata(chunkPath, JSON.parse(json)));
+            }
+        }
+        // Save dirty sparse indices
+        for (const field of result.dirtySparseIndices) {
+            const json = this.native.saveSparseIndex(field);
+            if (json) {
+                const indexPath = `__sparse_index__${field}`;
+                promises.push(this.storage.saveMetadata(indexPath, JSON.parse(json)));
+            }
+        }
+        // Save dirty field indexes
+        for (const field of result.dirtyFieldIndexes) {
+            const json = this.native.saveFieldIndex(field);
+            if (json) {
+                const indexId = `__metadata_field_index__field_${field}`;
+                promises.push(this.storage.saveMetadata(indexId, JSON.parse(json)));
+            }
+        }
+        // Track new fields
+        for (const field of result.newFields) {
+            this.knownFields.add(field);
+            this.dirtyFields.add(field);
+        }
+        if (result.dirtyFieldRegistry) {
+            promises.push(this.saveFieldRegistry());
+        }
+        if (result.dirtyEntityIdMapper) {
+            promises.push(this.saveEntityIdMapper());
+        }
+        await Promise.all(promises);
+    }
+    // ==========================================================================
+    // Cache warming
+    // ==========================================================================
+    async warmCache() {
+        const commonFields = ['noun', 'type', 'service', 'createdAt'];
+        await Promise.all(commonFields.map(async (field) => {
+            try {
+                await this.ensureFieldLoaded(field);
+            }
+            catch {
+                prodLog.debug(`Cache warming: field '${field}' not yet indexed`);
+            }
+        }));
+        await this.warmCacheForTopTypes(3);
+    }
+    async warmCacheForTopTypes(topN = 3) {
+        const topTypes = this.getTopNounTypes(topN);
+        if (topTypes.length === 0)
+            return;
+        for (const type of topTypes) {
+            const typeFields = this.typeFieldAffinity.get(type);
+            if (!typeFields)
+                continue;
+            const topFields = Array.from(typeFields.entries())
+                .sort((a, b) => b[1] - a[1])
+                .slice(0, 5)
+                .map(([field]) => field);
+            if (topFields.length === 0)
+                continue;
+            await Promise.all(topFields.map(async (field) => {
+                try {
+                    await this.ensureFieldLoaded(field);
+                }
+                catch {
+                    // Silently ignore
+                }
+            }));
+        }
+    }
+    // ==========================================================================
+    // Lazy load counts
+    // ==========================================================================
+    async lazyLoadCounts() {
+        try {
+            this.totalEntitiesByType.clear();
+            this.entityCountsByTypeFixed.fill(0);
+            this.verbCountsByTypeFixed.fill(0);
+            // Ensure 'noun' field is loaded into Rust
+            await this.ensureFieldLoaded('noun');
+            // Use Rust to get type counts via the noun field
+            const nounValues = this.native.getFilterValues('noun');
+            for (const typeName of nounValues) {
+                const ids = this.native.getIds('noun', JSON.stringify(typeName));
+                if (ids.length > 0) {
+                    this.totalEntitiesByType.set(typeName, ids.length);
+                }
+            }
+            prodLog.debug(`Loaded type counts: ${this.totalEntitiesByType.size} types`);
+        }
+        catch (error) {
+            prodLog.debug('Could not load type counts:', error);
+        }
+    }
+    syncTypeCountsToFixed() {
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const type = TypeUtils.getNounFromIndex(i);
+            const count = this.totalEntitiesByType.get(type) || 0;
+            this.entityCountsByTypeFixed[i] = count;
+        }
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const type = TypeUtils.getVerbFromIndex(i);
+            const count = this.totalEntitiesByType.get(type) || 0;
+            this.verbCountsByTypeFixed[i] = count;
+        }
+    }
+    syncTypeCountsFromFixed() {
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const count = this.entityCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getNounFromIndex(i);
+                this.totalEntitiesByType.set(type, count);
+            }
+        }
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const count = this.verbCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getVerbFromIndex(i);
+                this.totalEntitiesByType.set(type, count);
+            }
+        }
+    }
+    // ==========================================================================
+    // Core query methods
+    // ==========================================================================
+    async getIds(field, value) {
+        await this.ensureFieldLoaded(field);
+        return this.native.getIds(field, JSON.stringify(value));
+    }
+    async getIdsForFilter(filter) {
+        if (!filter || Object.keys(filter).length === 0)
+            return [];
+        // Extract fields and ensure loaded
+        const fields = this.extractFilterFields(filter);
+        await this.ensureFieldsLoaded(fields);
+        // For ne/exists:false/missing operators, we need allIds
+        const needsAllIds = this.filterNeedsAllIds(filter);
+        let allIdsJson = null;
+        if (needsAllIds) {
+            const allIds = await this.getAllIds();
+            allIdsJson = JSON.stringify(allIds);
+        }
+        return this.native.getIdsForFilter(JSON.stringify(filter), allIdsJson);
+    }
+    filterNeedsAllIds(filter) {
+        if (!filter || typeof filter !== 'object')
+            return false;
+        if (filter.allOf && Array.isArray(filter.allOf)) {
+            return filter.allOf.some((sub) => this.filterNeedsAllIds(sub));
+        }
+        if (filter.anyOf && Array.isArray(filter.anyOf)) {
+            return filter.anyOf.some((sub) => this.filterNeedsAllIds(sub));
+        }
+        for (const [_field, condition] of Object.entries(filter)) {
+            if (_field === 'allOf' || _field === 'anyOf' || _field === 'not')
+                continue;
+            if (condition && typeof condition === 'object' && !Array.isArray(condition)) {
+                for (const op of Object.keys(condition)) {
+                    if (op === 'ne' || op === 'isNot' || op === 'notEquals')
+                        return true;
+                    if (op === 'exists' && !condition[op])
+                        return true;
+                    if (op === 'missing' && condition[op])
+                        return true;
+                }
+            }
+        }
+        return false;
+    }
+    async getIdsForMultipleFields(fieldValuePairs) {
+        if (fieldValuePairs.length === 0)
+            return [];
+        const fields = fieldValuePairs.map(p => p.field);
+        await this.ensureFieldsLoaded(fields);
+        const pairsJson = JSON.stringify(fieldValuePairs.map(p => ({
+            field: p.field,
+            value: p.value
+        })));
+        return this.native.getIdsForMultipleFields(pairsJson);
+    }
+    async getIdsForTextQuery(query) {
+        await this.ensureFieldLoaded('__words__');
+        const resultJson = this.native.getIdsForTextQuery(query);
+        return JSON.parse(resultJson);
+    }
+    async getSortedIdsForFilter(filter, orderBy, order = 'asc') {
+        const filteredIds = await this.getIdsForFilter(filter);
+        if (filteredIds.length === 0)
+            return [];
+        const idValuePairs = [];
+        for (const id of filteredIds) {
+            const value = await this.getFieldValueForEntity(id, orderBy);
+            idValuePairs.push({ id, value });
+        }
+        idValuePairs.sort((a, b) => {
+            if (a.value == null && b.value == null)
+                return 0;
+            if (a.value == null)
+                return order === 'asc' ? 1 : -1;
+            if (b.value == null)
+                return order === 'asc' ? -1 : 1;
+            if (a.value === b.value)
+                return 0;
+            const comparison = a.value < b.value ? -1 : 1;
+            return order === 'asc' ? comparison : -comparison;
+        });
+        return idValuePairs.map(p => p.id);
+    }
+    async getFieldValueForEntity(entityId, field) {
+        // For timestamp fields, load actual value from entity metadata
+        if (field === 'createdAt' || field === 'updatedAt' || field === 'accessed' || field === 'modified') {
+            try {
+                const noun = await this.storage.getNoun(entityId);
+                if (noun && noun.metadata) {
+                    return noun.metadata[field];
+                }
+            }
+            catch {
+                // Fall back to index value
+            }
+        }
+        // For non-timestamp fields, scan chunks for this entity's value
+        await this.ensureFieldLoaded(field);
+        const ids = this.native.getIds(field, JSON.stringify(entityId));
+        // getIds returns IDs matching field=value, but we need value for field+entityId
+        // We need to scan through values — use getFilterValues and check membership
+        const values = this.native.getFilterValues(field);
+        for (const val of values) {
+            const matchIds = this.native.getIds(field, JSON.stringify(val));
+            if (matchIds.includes(entityId)) {
+                return this.denormalizeValue(val, field);
+            }
+        }
+        return undefined;
+    }
+    denormalizeValue(normalized, _field) {
+        const asNumber = Number(normalized);
+        if (!isNaN(asNumber))
+            return asNumber;
+        return normalized;
+    }
+    // ==========================================================================
+    // Filter discovery
+    // ==========================================================================
+    async getFilterValues(field) {
+        await this.ensureFieldLoaded(field);
+        return this.native.getFilterValues(field);
+    }
+    async getFilterFields() {
+        return this.native.getFilterFields();
+    }
+    // ==========================================================================
+    // Mutation methods
+    // ==========================================================================
+    async addToIndex(id, entityOrMetadata, skipFlush = false, deferWrites = false) {
+        // Extract field names from entity to know which fields to load
+        const fieldNames = this.native.extractFieldNames(JSON.stringify(entityOrMetadata));
+        await this.ensureFieldsLoaded(fieldNames);
+        // Execute mutation in Rust (single FFI call)
+        const resultJson = this.native.addToIndex(id, JSON.stringify(entityOrMetadata));
+        const result = JSON.parse(resultJson);
+        // Track new fields in loadedFields
+        for (const field of result.newFields) {
+            this.loadedFields.add(field);
+            this.knownFields.add(field);
+        }
+        // Update TS-side type tracking
+        this.updateTypeTracking(id, entityOrMetadata, 'add');
+        // Persist dirty data (unless deferred for batch operations)
+        if (!deferWrites) {
+            await this.persistMutationResult(result);
+        }
+        // Adaptive auto-flush
+        if (!skipFlush) {
+            for (const field of fieldNames) {
+                this.dirtyFields.add(field);
+            }
+            const timeSinceLastFlush = Date.now() - this.lastFlushTime;
+            const shouldAutoFlush = this.dirtyFields.size >= this.autoFlushThreshold ||
+                (this.dirtyFields.size > 10 && timeSinceLastFlush > 5000);
+            if (shouldAutoFlush) {
+                const startTime = Date.now();
+                await this.flush();
+                const flushTime = Date.now() - startTime;
+                if (flushTime < 50) {
+                    this.autoFlushThreshold = Math.min(200, this.autoFlushThreshold * 1.2);
+                }
+                else if (flushTime > 200) {
+                    this.autoFlushThreshold = Math.max(20, this.autoFlushThreshold * 0.8);
+                }
+            }
+        }
+    }
+    async removeFromIndex(id, metadata) {
+        if (metadata) {
+            const fieldNames = this.native.extractFieldNames(JSON.stringify(metadata));
+            await this.ensureFieldsLoaded(fieldNames);
+            const resultJson = this.native.removeFromIndex(id, JSON.stringify(metadata));
+            const result = JSON.parse(resultJson);
+            this.updateTypeTracking(id, metadata, 'remove');
+            await this.persistMutationResult(result);
+        }
+        else {
+            // Remove from all loaded fields (slower path)
+            prodLog.warn(`Removing ID ${id} without metadata requires scanning all fields (slow)`);
+            const loadedFields = this.native.getLoadedFields();
+            for (const field of loadedFields) {
+                await this.ensureFieldLoaded(field);
+            }
+            const resultJson = this.native.removeFromIndex(id);
+            const result = JSON.parse(resultJson);
+            await this.persistMutationResult(result);
+        }
+    }
+    updateTypeTracking(id, entityOrMetadata, operation) {
+        const nounValue = entityOrMetadata?.type || entityOrMetadata?.noun;
+        if (!nounValue)
+            return;
+        const entityType = String(nounValue).toLowerCase().trim();
+        if (operation === 'add') {
+            const current = this.totalEntitiesByType.get(entityType) || 0;
+            const newCount = current + 1;
+            this.totalEntitiesByType.set(entityType, newCount);
+            try {
+                const nounTypeIndex = TypeUtils.getNounIndex(entityType);
+                this.entityCountsByTypeFixed[nounTypeIndex] = newCount;
+            }
+            catch {
+                // Not a recognized noun type
+            }
+            // Update type-field affinity
+            if (!this.typeFieldAffinity.has(entityType)) {
+                this.typeFieldAffinity.set(entityType, new Map());
+            }
+            const typeFields = this.typeFieldAffinity.get(entityType);
+            const fieldNames = this.native.extractFieldNames(JSON.stringify(entityOrMetadata));
+            for (const field of fieldNames) {
+                const count = typeFields.get(field) || 0;
+                typeFields.set(field, count + 1);
+            }
+        }
+        else if (operation === 'remove') {
+            const current = this.totalEntitiesByType.get(entityType) || 0;
+            if (current > 1) {
+                const newCount = current - 1;
+                this.totalEntitiesByType.set(entityType, newCount);
+                try {
+                    const nounTypeIndex = TypeUtils.getNounIndex(entityType);
+                    this.entityCountsByTypeFixed[nounTypeIndex] = newCount;
+                }
+                catch { }
+            }
+            else {
+                this.totalEntitiesByType.delete(entityType);
+                this.typeFieldAffinity.delete(entityType);
+                try {
+                    const nounTypeIndex = TypeUtils.getNounIndex(entityType);
+                    this.entityCountsByTypeFixed[nounTypeIndex] = 0;
+                }
+                catch { }
+            }
+        }
+    }
+    // ==========================================================================
+    // Tokenization (delegates to Rust via extractFieldNames, but also exposed)
+    // ==========================================================================
+    tokenize(text) {
+        if (!text)
+            return [];
+        return text
+            .toLowerCase()
+            .replace(/[^\w\s]/g, ' ')
+            .split(/\s+/)
+            .filter(w => w.length >= 2 && w.length <= 50)
+            .filter((w, i, arr) => arr.indexOf(w) === i);
+    }
+    hashWord(word) {
+        let hash = 2166136261;
+        for (let i = 0; i < word.length; i++) {
+            hash ^= word.charCodeAt(i);
+            hash = Math.imul(hash, 16777619);
+        }
+        return hash | 0;
+    }
+    extractTextContent(data) {
+        if (data === null || data === undefined)
+            return '';
+        if (typeof data === 'string')
+            return data;
+        if (typeof data === 'number' || typeof data === 'boolean')
+            return String(data);
+        if (Array.isArray(data)) {
+            if (data.length > 0 && typeof data[0] === 'number')
+                return '';
+            return data.map(d => this.extractTextContent(d)).filter(Boolean).join(' ');
+        }
+        if (typeof data === 'object') {
+            const skipKeys = new Set(['vector', 'embedding', 'embeddings', 'connections', 'level', 'id']);
+            const texts = [];
+            for (const [key, value] of Object.entries(data)) {
+                if (skipKeys.has(key) || /^\d+$/.test(key))
+                    continue;
+                const text = this.extractTextContent(value);
+                if (text)
+                    texts.push(text);
+            }
+            return texts.join(' ');
+        }
+        return '';
+    }
+    // ==========================================================================
+    // Entity counting
+    // ==========================================================================
+    getEntityCountByType(type) {
+        return this.totalEntitiesByType.get(type) || 0;
+    }
+    getTotalEntityCount() {
+        let total = 0;
+        for (const count of this.totalEntitiesByType.values()) {
+            total += count;
+        }
+        return total;
+    }
+    getAllEntityCounts() {
+        return new Map(this.totalEntitiesByType);
+    }
+    getEntityCountByTypeEnum(type) {
+        const index = TypeUtils.getNounIndex(type);
+        return this.entityCountsByTypeFixed[index];
+    }
+    getVerbCountByTypeEnum(type) {
+        const index = TypeUtils.getVerbIndex(type);
+        return this.verbCountsByTypeFixed[index];
+    }
+    getTopNounTypes(n) {
+        const types = [];
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const count = this.entityCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getNounFromIndex(i);
+                types.push({ type, count });
+            }
+        }
+        return types.sort((a, b) => b.count - a.count).slice(0, n).map(t => t.type);
+    }
+    getTopVerbTypes(n) {
+        const types = [];
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const count = this.verbCountsByTypeFixed[i];
+            if (count > 0) {
+                const type = TypeUtils.getVerbFromIndex(i);
+                types.push({ type, count });
+            }
+        }
+        return types.sort((a, b) => b.count - a.count).slice(0, n).map(t => t.type);
+    }
+    getAllNounTypeCounts() {
+        const counts = new Map();
+        for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
+            const count = this.entityCountsByTypeFixed[i];
+            if (count > 0) {
+                counts.set(TypeUtils.getNounFromIndex(i), count);
+            }
+        }
+        return counts;
+    }
+    getAllVerbTypeCounts() {
+        const counts = new Map();
+        for (let i = 0; i < VERB_TYPE_COUNT; i++) {
+            const count = this.verbCountsByTypeFixed[i];
+            if (count > 0) {
+                counts.set(TypeUtils.getVerbFromIndex(i), count);
+            }
+        }
+        return counts;
+    }
+    // ==========================================================================
+    // VFS Statistics (roaring bitmap intersection via Rust)
+    // ==========================================================================
+    async getVFSEntityCountByType(type) {
+        await this.ensureFieldsLoaded(['isVFSEntity', 'noun']);
+        const vfsIds = this.native.getIds('isVFSEntity', JSON.stringify(true));
+        const typeIds = this.native.getIds('noun', JSON.stringify(type));
+        const vfsSet = new Set(vfsIds);
+        return typeIds.filter(id => vfsSet.has(id)).length;
+    }
+    async getAllVFSEntityCounts() {
+        await this.ensureFieldsLoaded(['isVFSEntity', 'noun']);
+        const vfsIds = this.native.getIds('isVFSEntity', JSON.stringify(true));
+        if (vfsIds.length === 0)
+            return new Map();
+        const vfsSet = new Set(vfsIds);
+        const result = new Map();
+        for (const type of this.totalEntitiesByType.keys()) {
+            const typeIds = this.native.getIds('noun', JSON.stringify(type));
+            const count = typeIds.filter(id => vfsSet.has(id)).length;
+            if (count > 0)
+                result.set(type, count);
+        }
+        return result;
+    }
+    async getTotalVFSEntityCount() {
+        await this.ensureFieldLoaded('isVFSEntity');
+        const vfsIds = this.native.getIds('isVFSEntity', JSON.stringify(true));
+        return vfsIds.length;
+    }
+    // ==========================================================================
+    // Criteria-based counting
+    // ==========================================================================
+    async getCountForCriteria(field, value) {
+        const ids = await this.getIds(field, value);
+        return ids.length;
+    }
+    // ==========================================================================
+    // getAllIds
+    // ==========================================================================
+    async getAllIds() {
+        const allIds = new Set();
+        if (this.storage && typeof this.storage.getNouns === 'function') {
+            try {
+                const result = await this.storage.getNouns({
+                    pagination: { limit: 100000 }
+                });
+                if (result && result.items) {
+                    result.items.forEach((item) => {
+                        if (item.id)
+                            allIds.add(item.id);
+                    });
+                }
+            }
+            catch (e) {
+                prodLog.warn('Failed to get all IDs from storage:', e);
+                return [];
+            }
+        }
+        return Array.from(allIds);
+    }
+    // ==========================================================================
+    // Flush
+    // ==========================================================================
+    async flush() {
+        if (this.dirtyFields.size === 0)
+            return;
+        // Save dirty field indexes
+        const promises = [];
+        for (const field of this.dirtyFields) {
+            const json = this.native.saveFieldIndex(field);
+            if (json) {
+                const indexId = `__metadata_field_index__field_${field}`;
+                promises.push(this.storage.saveMetadata(indexId, JSON.parse(json)));
+            }
+        }
+        await Promise.all(promises);
+        // Save entity ID mapper
+        await this.saveEntityIdMapper();
+        // Save field registry
+        await this.saveFieldRegistry();
+        this.dirtyFields.clear();
+        this.lastFlushTime = Date.now();
+    }
+    // ==========================================================================
+    // Stats
+    // ==========================================================================
+    async getStats() {
+        const statsJson = this.native.getStats();
+        const stats = JSON.parse(statsJson);
+        return {
+            totalEntries: stats.totalEntries || 0,
+            totalIds: 0,
+            fieldsIndexed: stats.fieldsIndexed || [],
+            lastRebuild: Date.now(),
+            indexSize: (stats.totalEntries || 0) * 100
+        };
+    }
+    // ==========================================================================
+    // Validation
+    // ==========================================================================
+    async validateConsistency() {
+        const entityCount = this.native.entityIdMapperSize();
+        if (entityCount === 0) {
+            return {
+                healthy: true,
+                avgEntriesPerEntity: 0,
+                entityCount: 0,
+                indexEntryCount: 0,
+                recommendation: null
+            };
+        }
+        // Load all fields to count entries
+        for (const field of this.knownFields) {
+            await this.ensureFieldLoaded(field);
+        }
+        const statsJson = this.native.getStats();
+        const stats = JSON.parse(statsJson);
+        const indexEntryCount = stats.totalEntries || 0;
+        const avgEntriesPerEntity = indexEntryCount / entityCount;
+        const CORRUPTION_THRESHOLD = 100;
+        const healthy = avgEntriesPerEntity <= CORRUPTION_THRESHOLD;
+        let recommendation = null;
+        if (!healthy) {
+            recommendation = `Index corruption detected (${avgEntriesPerEntity.toFixed(1)} avg entries/entity, expected ~30). ` +
+                `Run brain.index.clearAllIndexData() followed by brain.index.rebuild() to repair.`;
+        }
+        return { healthy, avgEntriesPerEntity, entityCount, indexEntryCount, recommendation };
+    }
+    async detectAndRepairCorruption() {
+        const validation = await this.validateConsistency();
+        if (!validation.healthy) {
+            prodLog.warn(`Index corruption detected (${validation.avgEntriesPerEntity.toFixed(1)} avg entries/entity)`);
+            prodLog.warn('Auto-rebuilding index to repair...');
+            await this.clearAllIndexData();
+            await this.rebuild();
+            const postRebuild = await this.validateConsistency();
+            if (postRebuild.healthy) {
+                prodLog.info(`Index rebuilt successfully (${postRebuild.avgEntriesPerEntity.toFixed(1)} avg entries/entity)`);
+            }
+            else {
+                prodLog.error(`Index still appears corrupted after rebuild (${postRebuild.avgEntriesPerEntity.toFixed(1)} avg entries/entity).`);
+            }
+        }
+    }
+    // ==========================================================================
+    // Rebuild
+    // ==========================================================================
+    async rebuild() {
+        if (this.isRebuilding)
+            return;
+        this.isRebuilding = true;
+        try {
+            prodLog.info('Starting metadata index rebuild with batch processing...');
+            // Clear in-memory state
+            this.native.clear();
+            this.loadedFields.clear();
+            this.knownFields.clear();
+            this.dirtyFields.clear();
+            this.totalEntitiesByType.clear();
+            this.entityCountsByTypeFixed.fill(0);
+            this.verbCountsByTypeFixed.fill(0);
+            this.typeFieldAffinity.clear();
+            this.unifiedCache.clear('metadata');
+            // Delete existing chunk files from storage
+            prodLog.info('Clearing existing metadata index chunks from storage...');
+            const existingFields = await this.getPersistedFieldList();
+            if (existingFields.length > 0) {
+                for (const field of existingFields) {
+                    await this.deleteFieldChunks(field);
+                }
+                try {
+                    await this.storage.saveMetadata('__metadata_field_registry__', null);
+                }
+                catch { }
+                prodLog.info(`Cleared ${existingFields.length} field indexes from storage`);
+            }
+            // Clear entity ID mapper in storage
+            try {
+                await this.storage.saveMetadata('brainy:entityIdMapper', null);
+            }
+            catch { }
+            // Adaptive rebuild strategy
+            const storageType = this.storage.constructor.name;
+            const isLocalStorage = storageType === 'FileSystemStorage' || storageType === 'MemoryStorage';
+            let totalNounsProcessed = 0;
+            if (isLocalStorage) {
+                const result = await this.storage.getNouns({
+                    pagination: { offset: 0, limit: 1000000 }
+                });
+                prodLog.info(`Loading ${result.items.length} nouns with metadata...`);
+                const nounIds = result.items.map(noun => noun.id);
+                let metadataBatch;
+                if (this.storage.getMetadataBatch) {
+                    metadataBatch = await this.storage.getMetadataBatch(nounIds);
+                }
+                else {
+                    metadataBatch = new Map();
+                    for (const id of nounIds) {
+                        try {
+                            const metadata = await this.storage.getNounMetadata(id);
+                            if (metadata)
+                                metadataBatch.set(id, metadata);
+                        }
+                        catch { }
+                    }
+                }
+                let localCount = 0;
+                for (const noun of result.items) {
+                    const metadata = metadataBatch.get(noun.id);
+                    if (metadata) {
+                        await this.addToIndex(noun.id, metadata, true, true);
+                        localCount++;
+                        if (localCount % 5000 === 0) {
+                            await this.flushRebuildDirty();
+                        }
+                    }
+                }
+                totalNounsProcessed = result.items.length;
+                prodLog.info(`Indexed ${totalNounsProcessed} nouns`);
+            }
+            else {
+                // Cloud storage: conservative batching
+                const nounLimit = 25;
+                let nounOffset = 0;
+                let hasMoreNouns = true;
+                let consecutiveEmptyBatches = 0;
+                const MAX_ITERATIONS = 10000;
+                let iterations = 0;
+                while (hasMoreNouns && iterations < MAX_ITERATIONS) {
+                    iterations++;
+                    const result = await this.storage.getNouns({
+                        pagination: { offset: nounOffset, limit: nounLimit }
+                    });
+                    if (result.items.length === 0) {
+                        consecutiveEmptyBatches++;
+                        if (consecutiveEmptyBatches >= 3)
+                            break;
+                        if (result.hasMore) {
+                            hasMoreNouns = false;
+                            break;
+                        }
+                    }
+                    else {
+                        consecutiveEmptyBatches = 0;
+                    }
+                    const nounIds = result.items.map(noun => noun.id);
+                    let metadataBatch;
+                    if (this.storage.getMetadataBatch) {
+                        metadataBatch = await this.storage.getMetadataBatch(nounIds);
+                    }
+                    else {
+                        metadataBatch = new Map();
+                        for (const id of nounIds) {
+                            try {
+                                const metadata = await this.storage.getNounMetadata(id);
+                                if (metadata)
+                                    metadataBatch.set(id, metadata);
+                            }
+                            catch { }
+                        }
+                    }
+                    for (const noun of result.items) {
+                        const metadata = metadataBatch.get(noun.id);
+                        if (metadata) {
+                            await this.addToIndex(noun.id, metadata, true, true);
+                        }
+                    }
+                    totalNounsProcessed += result.items.length;
+                    if (totalNounsProcessed % 5000 === 0) {
+                        await this.flushRebuildDirty();
+                    }
+                    hasMoreNouns = result.hasMore;
+                    nounOffset += nounLimit;
+                    await this.yieldToEventLoop();
+                }
+            }
+            // Rebuild verb metadata indexes
+            let totalVerbsProcessed = 0;
+            if (isLocalStorage) {
+                const result = await this.storage.getVerbs({
+                    pagination: { offset: 0, limit: 1000000 }
+                });
+                const verbIds = result.items.map(verb => verb.id);
+                let verbMetadataBatch;
+                if (this.storage.getVerbMetadataBatch) {
+                    verbMetadataBatch = await this.storage.getVerbMetadataBatch(verbIds);
+                }
+                else {
+                    verbMetadataBatch = new Map();
+                    for (const id of verbIds) {
+                        try {
+                            const metadata = await this.storage.getVerbMetadata(id);
+                            if (metadata)
+                                verbMetadataBatch.set(id, metadata);
+                        }
+                        catch { }
+                    }
+                }
+                let verbLocalCount = 0;
+                for (const verb of result.items) {
+                    const metadata = verbMetadataBatch.get(verb.id);
+                    if (metadata) {
+                        await this.addToIndex(verb.id, metadata, true, true);
+                        verbLocalCount++;
+                        if (verbLocalCount % 5000 === 0) {
+                            await this.flushRebuildDirty();
+                        }
+                    }
+                }
+                totalVerbsProcessed = result.items.length;
+            }
+            else {
+                let verbOffset = 0;
+                const verbLimit = 25;
+                let hasMoreVerbs = true;
+                let consecutiveEmptyVerbBatches = 0;
+                let verbIterations = 0;
+                const MAX_ITERATIONS = 10000;
+                while (hasMoreVerbs && verbIterations < MAX_ITERATIONS) {
+                    verbIterations++;
+                    const result = await this.storage.getVerbs({
+                        pagination: { offset: verbOffset, limit: verbLimit }
+                    });
+                    if (result.items.length === 0) {
+                        consecutiveEmptyVerbBatches++;
+                        if (consecutiveEmptyVerbBatches >= 3)
+                            break;
+                        if (result.hasMore) {
+                            hasMoreVerbs = false;
+                            break;
+                        }
+                    }
+                    else {
+                        consecutiveEmptyVerbBatches = 0;
+                    }
+                    const verbIds = result.items.map(verb => verb.id);
+                    let verbMetadataBatch;
+                    if (this.storage.getVerbMetadataBatch) {
+                        verbMetadataBatch = await this.storage.getVerbMetadataBatch(verbIds);
+                    }
+                    else {
+                        verbMetadataBatch = new Map();
+                        for (const id of verbIds) {
+                            try {
+                                const metadata = await this.storage.getVerbMetadata(id);
+                                if (metadata)
+                                    verbMetadataBatch.set(id, metadata);
+                            }
+                            catch { }
+                        }
+                    }
+                    for (const verb of result.items) {
+                        const metadata = verbMetadataBatch.get(verb.id);
+                        if (metadata) {
+                            await this.addToIndex(verb.id, metadata, true, true);
+                        }
+                    }
+                    totalVerbsProcessed += result.items.length;
+                    if (totalVerbsProcessed % 5000 === 0) {
+                        await this.flushRebuildDirty();
+                    }
+                    hasMoreVerbs = result.hasMore;
+                    verbOffset += verbLimit;
+                    await this.yieldToEventLoop();
+                }
+            }
+            // Final flush
+            await this.flushRebuildDirty();
+            await this.flush();
+            prodLog.info(`Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`);
+        }
+        finally {
+            this.isRebuilding = false;
+        }
+    }
+    /**
+     * During rebuild, persist all dirty data accumulated so far.
+     */
+    async flushRebuildDirty() {
+        // Save all dirty data currently in Rust
+        const promises = [];
+        // Save all loaded sparse indices
+        for (const field of this.loadedFields) {
+            const json = this.native.saveSparseIndex(field);
+            if (json) {
+                const indexPath = `__sparse_index__${field}`;
+                promises.push(this.storage.saveMetadata(indexPath, JSON.parse(json)));
+            }
+            // Save all chunks for this field
+            const chunkIds = this.native.getSparseIndexChunkIds(field);
+            for (const chunkId of chunkIds) {
+                const chunkJson = this.native.saveChunk(field, chunkId);
+                if (chunkJson) {
+                    const chunkPath = `__chunk__${field}_${chunkId}`;
+                    promises.push(this.storage.saveMetadata(chunkPath, JSON.parse(chunkJson)));
+                }
+            }
+        }
+        // Save entity ID mapper
+        promises.push(this.saveEntityIdMapper());
+        await Promise.all(promises);
+    }
+    // ==========================================================================
+    // Clear
+    // ==========================================================================
+    async clearAllIndexData() {
+        prodLog.warn('Clearing ALL metadata index data from storage...');
+        const fields = await this.getPersistedFieldList();
+        for (const field of fields) {
+            await this.deleteFieldChunks(field);
+        }
+        try {
+            await this.storage.saveMetadata('__metadata_field_registry__', null);
+        }
+        catch { }
+        // Clear in-memory state
+        this.native.clear();
+        this.loadedFields.clear();
+        this.knownFields.clear();
+        this.dirtyFields.clear();
+        this.totalEntitiesByType.clear();
+        this.entityCountsByTypeFixed.fill(0);
+        this.verbCountsByTypeFixed.fill(0);
+        this.typeFieldAffinity.clear();
+        this.unifiedCache.clear('metadata');
+        // Clear entity ID mapper in storage
+        try {
+            await this.storage.saveMetadata('brainy:entityIdMapper', null);
+        }
+        catch { }
+        prodLog.info(`Cleared ${fields.length} field indexes and all in-memory state`);
+    }
+    async getPersistedFieldList() {
+        try {
+            const registry = await this.storage.getMetadata('__metadata_field_registry__');
+            if (!registry?.fields || !Array.isArray(registry.fields))
+                return [];
+            return registry.fields.filter((f) => typeof f === 'string' && f.length > 0);
+        }
+        catch {
+            return [];
+        }
+    }
+    async deleteFieldChunks(field) {
+        try {
+            const indexPath = `__sparse_index__${field}`;
+            const sparseData = await this.storage.getMetadata(indexPath);
+            if (sparseData) {
+                // Parse chunk IDs from sparse index
+                let chunkIds = [];
+                if (sparseData.chunks && Array.isArray(sparseData.chunks)) {
+                    chunkIds = sparseData.chunks.map((c) => c.chunkId).filter((id) => typeof id === 'number');
+                }
+                // Delete all chunk files
+                for (const chunkId of chunkIds) {
+                    const chunkPath = `__chunk__${field}_${chunkId}`;
+                    try {
+                        await this.storage.saveMetadata(chunkPath, null);
+                    }
+                    catch { }
+                }
+                // Delete sparse index
+                try {
+                    await this.storage.saveMetadata(indexPath, null);
+                }
+                catch { }
+            }
+            // Delete field index
+            try {
+                await this.storage.saveMetadata(`__metadata_field_index__field_${field}`, null);
+            }
+            catch { }
+        }
+        catch {
+            prodLog.debug(`Could not clear chunks for field '${field}'`);
+        }
+    }
+    // ==========================================================================
+    // Field statistics (TS-only analytics)
+    // ==========================================================================
+    async getFieldStatistics() {
+        for (const field of this.knownFields) {
+            if (!this.fieldStats.has(field)) {
+                this.fieldStats.set(field, {
+                    cardinality: {
+                        uniqueValues: 0,
+                        totalValues: 0,
+                        distribution: 'uniform',
+                        updateFrequency: 0,
+                        lastAnalyzed: Date.now()
+                    },
+                    queryCount: 0,
+                    rangeQueryCount: 0,
+                    exactQueryCount: 0,
+                    avgQueryTime: 0,
+                    indexType: 'hash'
+                });
+            }
+        }
+        return new Map(this.fieldStats);
+    }
+    async getFieldCardinality(field) {
+        const stats = this.fieldStats.get(field);
+        return stats ? stats.cardinality : null;
+    }
+    async getFieldsWithCardinality() {
+        const fields = [];
+        for (const [field, stats] of this.fieldStats) {
+            fields.push({
+                field,
+                cardinality: stats.cardinality.uniqueValues,
+                distribution: stats.cardinality.distribution
+            });
+        }
+        fields.sort((a, b) => a.cardinality - b.cardinality);
+        return fields;
+    }
+    async getOptimalQueryPlan(filters) {
+        const fieldOrder = [];
+        let hasRangeQueries = false;
+        let totalEstimatedCost = 0;
+        for (const [field, value] of Object.entries(filters)) {
+            const stats = this.fieldStats.get(field);
+            if (!stats)
+                continue;
+            if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+                hasRangeQueries = true;
+            }
+            const cardinality = stats.cardinality.uniqueValues;
+            totalEstimatedCost += Math.log2(Math.max(1, cardinality));
+            fieldOrder.push(field);
+        }
+        fieldOrder.sort((a, b) => {
+            const statsA = this.fieldStats.get(a);
+            const statsB = this.fieldStats.get(b);
+            if (!statsA || !statsB)
+                return 0;
+            return statsA.cardinality.uniqueValues - statsB.cardinality.uniqueValues;
+        });
+        return {
+            strategy: hasRangeQueries ? 'hybrid' : 'exact',
+            fieldOrder,
+            estimatedCost: totalEstimatedCost
+        };
+    }
+    async getFieldsForType(nounType) {
+        const typeFields = this.typeFieldAffinity.get(nounType);
+        const totalEntities = this.totalEntitiesByType.get(nounType);
+        if (!typeFields || !totalEntities)
+            return [];
+        const result = [];
+        for (const [field, count] of typeFields.entries()) {
+            result.push({
+                field,
+                affinity: count / totalEntities,
+                occurrences: count,
+                totalEntities
+            });
+        }
+        result.sort((a, b) => b.affinity - a.affinity);
+        return result;
+    }
+    // ==========================================================================
+    // Deprecated / compatibility
+    // ==========================================================================
+    async getIdsForCriteria(criteria) {
+        return this.getIdsForFilter(criteria);
+    }
+    // ==========================================================================
+    // Private helpers
+    // ==========================================================================
+    async yieldToEventLoop() {
+        return new Promise(resolve => setImmediate(resolve));
+    }
+}
+//# sourceMappingURL=NativeMetadataIndex.js.map