@soulcraft/brainy 3.0.0 → 3.0.1

package/dist/utils/metadataIndex.js

@@ -18,6 +18,17 @@ export class MetadataIndexManager {
         // Sorted indices for range queries (only for numeric/date fields)
         this.sortedIndices = new Map();
         this.numericFields = new Set(); // Track which fields are numeric
+        // Cardinality and field statistics tracking
+        this.fieldStats = new Map();
+        this.cardinalityUpdateInterval = 100; // Update cardinality every N operations
+        this.operationCount = 0;
+        // Smart normalization thresholds
+        this.HIGH_CARDINALITY_THRESHOLD = 1000;
+        this.TIMESTAMP_PRECISION_MS = 60000; // 1 minute buckets
+        this.FLOAT_PRECISION = 2; // decimal places
+        // Type-Field Affinity Tracking for intelligent NLP
+        this.typeFieldAffinity = new Map(); // nounType -> field -> count
+        this.totalEntitiesByType = new Map(); // nounType -> total count
         this.storage = storage;
         this.config = {
             maxIndexSize: config.maxIndexSize ?? 10000,
@@ -53,10 +64,10 @@ export class MetadataIndexManager {
                 this.sortedIndices.set(field, loaded);
             }
             else {
-                // Create new sorted index
+                // Create new sorted index - NOT dirty since we maintain incrementally
                 this.sortedIndices.set(field, {
                     values: [],
-                    isDirty: true,
+                    isDirty: false, // Clean by default with incremental updates
                     fieldType: 'mixed'
                 });
             }
@@ -108,6 +119,107 @@ export class MetadataIndexManager {
         sortedIndex.values = sorted;
         sortedIndex.isDirty = false;
     }
+    /**
+     * Detect field type from value
+     */
+    detectFieldType(value) {
+        if (typeof value === 'number' && !isNaN(value))
+            return 'number';
+        if (value instanceof Date)
+            return 'date';
+        return 'string';
+    }
+    /**
+     * Compare two values based on field type for sorting
+     */
+    compareValues(a, b, fieldType) {
+        switch (fieldType) {
+            case 'number':
+                return a - b;
+            case 'date':
+                return a.getTime() - b.getTime();
+            case 'string':
+            default:
+                const aStr = String(a);
+                const bStr = String(b);
+                return aStr < bStr ? -1 : aStr > bStr ? 1 : 0;
+        }
+    }
+    /**
+     * Binary search to find insertion position for a value
+     * Returns the index where the value should be inserted to maintain sorted order
+     */
+    findInsertPosition(sortedArray, value, fieldType) {
+        if (sortedArray.length === 0)
+            return 0;
+        let left = 0;
+        let right = sortedArray.length - 1;
+        while (left <= right) {
+            const mid = Math.floor((left + right) / 2);
+            const midVal = sortedArray[mid][0];
+            const comparison = this.compareValues(midVal, value, fieldType);
+            if (comparison < 0) {
+                left = mid + 1;
+            }
+            else if (comparison > 0) {
+                right = mid - 1;
+            }
+            else {
+                return mid; // Value already exists at this position
+            }
+        }
+        return left; // Insert position
+    }
+    /**
+     * Incrementally update sorted index when adding an ID
+     */
+    updateSortedIndexAdd(field, value, id) {
+        // Ensure sorted index exists
+        if (!this.sortedIndices.has(field)) {
+            this.sortedIndices.set(field, {
+                values: [],
+                isDirty: false,
+                fieldType: this.detectFieldType(value)
+            });
+        }
+        const sortedIndex = this.sortedIndices.get(field);
+        const normalizedValue = this.normalizeValue(value);
+        // Find where this value should be in the sorted array
+        const insertPos = this.findInsertPosition(sortedIndex.values, normalizedValue, sortedIndex.fieldType);
+        if (insertPos < sortedIndex.values.length &&
+            sortedIndex.values[insertPos][0] === normalizedValue) {
+            // Value already exists, just add the ID to the existing Set
+            sortedIndex.values[insertPos][1].add(id);
+        }
+        else {
+            // New value, insert at the correct position
+            sortedIndex.values.splice(insertPos, 0, [normalizedValue, new Set([id])]);
+        }
+        // Mark as clean since we're maintaining it incrementally
+        sortedIndex.isDirty = false;
+    }
+    /**
+     * Incrementally update sorted index when removing an ID
+     */
+    updateSortedIndexRemove(field, value, id) {
+        const sortedIndex = this.sortedIndices.get(field);
+        if (!sortedIndex || sortedIndex.values.length === 0)
+            return;
+        const normalizedValue = this.normalizeValue(value);
+        // Binary search to find the value
+        const pos = this.findInsertPosition(sortedIndex.values, normalizedValue, sortedIndex.fieldType);
+        if (pos < sortedIndex.values.length &&
+            sortedIndex.values[pos][0] === normalizedValue) {
+            // Remove the ID from the Set
+            sortedIndex.values[pos][1].delete(id);
+            // If no IDs left for this value, remove the entire entry
+            if (sortedIndex.values[pos][1].size === 0) {
+                sortedIndex.values.splice(pos, 1);
+            }
+        }
+        // Keep it clean
+        sortedIndex.isDirty = false;
+    }
     /**
      * Binary search for range start (inclusive or exclusive)
      */
@@ -148,14 +260,126 @@ export class MetadataIndexManager {
         }
         return result;
     }
+    /**
+     * Update cardinality statistics for a field
+     */
+    updateCardinalityStats(field, value, operation) {
+        // Initialize field stats if needed
+        if (!this.fieldStats.has(field)) {
+            this.fieldStats.set(field, {
+                cardinality: {
+                    uniqueValues: 0,
+                    totalValues: 0,
+                    distribution: 'uniform',
+                    updateFrequency: 0,
+                    lastAnalyzed: Date.now()
+                },
+                queryCount: 0,
+                rangeQueryCount: 0,
+                exactQueryCount: 0,
+                avgQueryTime: 0,
+                indexType: 'hash'
+            });
+        }
+        const stats = this.fieldStats.get(field);
+        const cardinality = stats.cardinality;
+        // Track unique values
+        const fieldIndexKey = `${field}:${String(value)}`;
+        const entry = this.indexCache.get(fieldIndexKey);
+        if (operation === 'add') {
+            if (!entry || entry.ids.size === 1) {
+                cardinality.uniqueValues++;
+            }
+            cardinality.totalValues++;
+        }
+        else if (operation === 'remove') {
+            if (entry && entry.ids.size === 0) {
+                cardinality.uniqueValues--;
+            }
+            cardinality.totalValues = Math.max(0, cardinality.totalValues - 1);
+        }
+        // Update frequency tracking
+        cardinality.updateFrequency++;
+        // Periodically analyze distribution
+        if (++this.operationCount % this.cardinalityUpdateInterval === 0) {
+            this.analyzeFieldDistribution(field);
+        }
+        // Determine optimal index type based on cardinality
+        this.updateIndexStrategy(field, stats);
+    }
+    /**
+     * Analyze field distribution for optimization
+     */
+    analyzeFieldDistribution(field) {
+        const stats = this.fieldStats.get(field);
+        if (!stats)
+            return;
+        const cardinality = stats.cardinality;
+        const ratio = cardinality.uniqueValues / Math.max(1, cardinality.totalValues);
+        // Determine distribution type
+        if (ratio > 0.9) {
+            cardinality.distribution = 'sparse'; // High uniqueness (like IDs, timestamps)
+        }
+        else if (ratio < 0.1) {
+            cardinality.distribution = 'skewed'; // Low uniqueness (like status, type)
+        }
+        else {
+            cardinality.distribution = 'uniform'; // Balanced distribution
+        }
+        cardinality.lastAnalyzed = Date.now();
+    }
+    /**
+     * Update index strategy based on field statistics
+     */
+    updateIndexStrategy(field, stats) {
+        const isNumeric = this.numericFields.has(field);
+        const hasHighCardinality = stats.cardinality.uniqueValues > this.HIGH_CARDINALITY_THRESHOLD;
+        const hasRangeQueries = stats.rangeQueryCount > stats.exactQueryCount * 0.3;
+        // Determine optimal index type
+        if (isNumeric && hasRangeQueries) {
+            stats.indexType = 'both'; // Need both hash and sorted
+        }
+        else if (hasRangeQueries) {
+            stats.indexType = 'sorted';
+        }
+        else {
+            stats.indexType = 'hash';
+        }
+        // Determine normalization strategy for high cardinality fields
+        if (hasHighCardinality) {
+            if (field.toLowerCase().includes('time') || field.toLowerCase().includes('date')) {
+                stats.normalizationStrategy = 'bucket'; // Time bucketing
+            }
+            else if (isNumeric) {
+                stats.normalizationStrategy = 'precision'; // Reduce float precision
+            }
+            else {
+                stats.normalizationStrategy = 'none'; // Keep as-is for strings
+            }
+        }
+        else {
+            stats.normalizationStrategy = 'none';
+        }
+    }
     /**
      * Get IDs matching a range query
      */
     async getIdsForRange(field, min, max, includeMin = true, includeMax = true) {
-        // Ensure sorted index exists and is up to date
+        // Track range query for field statistics
+        if (this.fieldStats.has(field)) {
+            const stats = this.fieldStats.get(field);
+            stats.rangeQueryCount++;
+        }
+        // Ensure sorted index exists
         await this.ensureSortedIndex(field);
-        await this.buildSortedIndex(field);
-        const sortedIndex = this.sortedIndices.get(field);
+        // With incremental updates, we should rarely need to rebuild
+        // Only rebuild if it's marked dirty (e.g., after a bulk load or migration)
+        let sortedIndex = this.sortedIndices.get(field);
+        if (sortedIndex?.isDirty) {
+            prodLog.warn(`MetadataIndex: Sorted index for field '${field}' was dirty, rebuilding...`);
+            await this.buildSortedIndex(field);
+            sortedIndex = this.sortedIndices.get(field);
+        }
         if (!sortedIndex || sortedIndex.values.length === 0)
             return [];
         const sorted = sortedIndex.values;
@@ -201,17 +425,36 @@ export class MetadataIndexManager {
             .toLowerCase();
     }
     /**
-     * Normalize value for consistent indexing
+     * Normalize value for consistent indexing with smart optimization
      */
-    normalizeValue(value) {
+    normalizeValue(value, field) {
         if (value === null || value === undefined)
             return '__NULL__';
         if (typeof value === 'boolean')
             return value ? '__TRUE__' : '__FALSE__';
+        // Apply smart normalization based on field statistics
+        if (field && this.fieldStats.has(field)) {
+            const stats = this.fieldStats.get(field);
+            const strategy = stats.normalizationStrategy;
+            if (strategy === 'bucket' && typeof value === 'number') {
+                // Time bucketing for timestamps
+                if (field.toLowerCase().includes('time') || field.toLowerCase().includes('date')) {
+                    const bucketSize = this.TIMESTAMP_PRECISION_MS;
+                    const bucketed = Math.floor(value / bucketSize) * bucketSize;
+                    return bucketed.toString();
+                }
+            }
+            else if (strategy === 'precision' && typeof value === 'number') {
+                // Reduce float precision for high cardinality numeric fields
+                const rounded = Math.round(value * Math.pow(10, this.FLOAT_PRECISION)) / Math.pow(10, this.FLOAT_PRECISION);
+                return rounded.toString();
+            }
+        }
+        // Default normalization
         if (typeof value === 'number')
             return value.toString();
         if (Array.isArray(value)) {
-            const joined = value.map(v => this.normalizeValue(v)).join(',');
+            const joined = value.map(v => this.normalizeValue(v, field)).join(',');
             // Hash very long array values to avoid filesystem limits
             if (joined.length > 100) {
                 return this.hashValue(joined);
@@ -285,13 +528,16 @@ export class MetadataIndexManager {
      */
     async addToIndex(id, metadata, skipFlush = false) {
         const fields = this.extractIndexableFields(metadata);
-        // Mark sorted indices as dirty when adding new data
-        for (const { field } of fields) {
-            const sortedIndex = this.sortedIndices.get(field);
-            if (sortedIndex) {
-                sortedIndex.isDirty = true;
-            }
-        }
+        // Sort fields to process 'noun' field first for type-field affinity tracking
+        fields.sort((a, b) => {
+            if (a.field === 'noun')
+                return -1;
+            if (b.field === 'noun')
+                return 1;
+            return 0;
+        });
+        // Track which fields we're updating for incremental sorted index maintenance
+        const updatedFields = new Set();
         for (let i = 0; i < fields.length; i++) {
             const { field, value } = fields[i];
             const key = this.getIndexKey(field, value);
@@ -308,9 +554,25 @@ export class MetadataIndexManager {
                 this.indexCache.set(key, entry);
             }
             // Add ID to entry
+            const wasNew = entry.ids.size === 0;
             entry.ids.add(id);
             entry.lastUpdated = Date.now();
             this.dirtyEntries.add(key);
+            // Update cardinality statistics
+            if (wasNew) {
+                this.updateCardinalityStats(field, value, 'add');
+            }
+            // Track type-field affinity for intelligent NLP
+            this.updateTypeFieldAffinity(id, field, value, 'add');
+            // Incrementally update sorted index for this field
+            if (!updatedFields.has(field)) {
+                this.updateSortedIndexAdd(field, value, id);
+                updatedFields.add(field);
+            }
+            else {
+                // Multiple values for same field - still update
+                this.updateSortedIndexAdd(field, value, id);
+            }
             // Update field index
             await this.updateFieldIndex(field, value, 1);
             // Yield to event loop every 5 fields to prevent blocking
@@ -382,9 +644,20 @@ export class MetadataIndexManager {
                     entry = loadedEntry ?? undefined;
                 }
                 if (entry) {
+                    const hadId = entry.ids.has(id);
                     entry.ids.delete(id);
                     entry.lastUpdated = Date.now();
                     this.dirtyEntries.add(key);
+                    // Update cardinality statistics
+                    if (hadId && entry.ids.size === 0) {
+                        this.updateCardinalityStats(field, value, 'remove');
+                    }
+                    // Track type-field affinity for intelligent NLP
+                    if (hadId) {
+                        this.updateTypeFieldAffinity(id, field, value, 'remove');
+                    }
+                    // Incrementally update sorted index when removing
+                    this.updateSortedIndexRemove(field, value, id);
                     // Update field index
                     await this.updateFieldIndex(field, value, -1);
                     // If no IDs left, mark for cleanup
@@ -404,6 +677,8 @@ export class MetadataIndexManager {
                     entry.ids.delete(id);
                     entry.lastUpdated = Date.now();
                     this.dirtyEntries.add(key);
+                    // Incrementally update sorted index
+                    this.updateSortedIndexRemove(entry.field, entry.value, id);
                     if (entry.ids.size === 0) {
                         this.indexCache.delete(key);
                         await this.deleteIndexEntry(key);
@@ -446,6 +721,11 @@ export class MetadataIndexManager {
      * Get IDs for a specific field-value combination with caching
      */
     async getIds(field, value) {
+        // Track exact query for field statistics
+        if (this.fieldStats.has(field)) {
+            const stats = this.fieldStats.get(field);
+            stats.exactQueryCount++;
+        }
         const key = this.getIndexKey(field, value);
         // Check metadata cache first
         const cacheKey = `ids_${key}`;
@@ -944,7 +1224,7 @@ export class MetadataIndexManager {
             totalEntries,
             totalIds,
             fieldsIndexed: Array.from(fields),
-            lastRebuild: 0, // TODO: track rebuild timestamp
+            lastRebuild: Date.now(),
             indexSize: totalEntries * 100 // rough estimate
         };
     }
@@ -1174,5 +1454,248 @@ export class MetadataIndexManager {
             // Entry might not exist
         }
     }
+    /**
+     * Get field statistics for optimization and discovery
+     */
+    async getFieldStatistics() {
+        // Initialize stats for fields we haven't seen yet
+        for (const field of this.fieldIndexes.keys()) {
+            if (!this.fieldStats.has(field)) {
+                this.fieldStats.set(field, {
+                    cardinality: {
+                        uniqueValues: 0,
+                        totalValues: 0,
+                        distribution: 'uniform',
+                        updateFrequency: 0,
+                        lastAnalyzed: Date.now()
+                    },
+                    queryCount: 0,
+                    rangeQueryCount: 0,
+                    exactQueryCount: 0,
+                    avgQueryTime: 0,
+                    indexType: 'hash'
+                });
+            }
+        }
+        return new Map(this.fieldStats);
+    }
+    /**
+     * Get field cardinality information
+     */
+    async getFieldCardinality(field) {
+        const stats = this.fieldStats.get(field);
+        return stats ? stats.cardinality : null;
+    }
+    /**
+     * Get all field names with their cardinality (for query optimization)
+     */
+    async getFieldsWithCardinality() {
+        const fields = [];
+        for (const [field, stats] of this.fieldStats) {
+            fields.push({
+                field,
+                cardinality: stats.cardinality.uniqueValues,
+                distribution: stats.cardinality.distribution
+            });
+        }
+        // Sort by cardinality (low cardinality fields are better for filtering)
+        fields.sort((a, b) => a.cardinality - b.cardinality);
+        return fields;
+    }
+    /**
+     * Get optimal query plan based on field statistics
+     */
+    async getOptimalQueryPlan(filters) {
+        const fieldOrder = [];
+        let hasRangeQueries = false;
+        let totalEstimatedCost = 0;
+        // Analyze each filter
+        for (const [field, value] of Object.entries(filters)) {
+            const stats = this.fieldStats.get(field);
+            if (!stats)
+                continue;
+            // Check if this is a range query
+            if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+                hasRangeQueries = true;
+            }
+            // Estimate cost based on cardinality
+            const cardinality = stats.cardinality.uniqueValues;
+            const estimatedCost = Math.log2(Math.max(1, cardinality));
+            totalEstimatedCost += estimatedCost;
+            fieldOrder.push(field);
+        }
+        // Sort fields by cardinality (process low cardinality first)
+        fieldOrder.sort((a, b) => {
+            const statsA = this.fieldStats.get(a);
+            const statsB = this.fieldStats.get(b);
+            if (!statsA || !statsB)
+                return 0;
+            return statsA.cardinality.uniqueValues - statsB.cardinality.uniqueValues;
+        });
+        return {
+            strategy: hasRangeQueries ? 'hybrid' : 'exact',
+            fieldOrder,
+            estimatedCost: totalEstimatedCost
+        };
+    }
+    /**
+     * Export field statistics for analysis
+     */
+    async exportFieldStats() {
+        const stats = {
+            fields: {},
+            summary: {
+                totalFields: this.fieldStats.size,
+                highCardinalityFields: 0,
+                sparseFields: 0,
+                skewedFields: 0,
+                uniformFields: 0
+            }
+        };
+        for (const [field, fieldStats] of this.fieldStats) {
+            stats.fields[field] = {
+                cardinality: fieldStats.cardinality,
+                queryStats: {
+                    total: fieldStats.queryCount,
+                    exact: fieldStats.exactQueryCount,
+                    range: fieldStats.rangeQueryCount,
+                    avgTime: fieldStats.avgQueryTime
+                },
+                indexType: fieldStats.indexType,
+                normalization: fieldStats.normalizationStrategy
+            };
+            // Update summary
+            if (fieldStats.cardinality.uniqueValues > this.HIGH_CARDINALITY_THRESHOLD) {
+                stats.summary.highCardinalityFields++;
+            }
+            switch (fieldStats.cardinality.distribution) {
+                case 'sparse':
+                    stats.summary.sparseFields++;
+                    break;
+                case 'skewed':
+                    stats.summary.skewedFields++;
+                    break;
+                case 'uniform':
+                    stats.summary.uniformFields++;
+                    break;
+            }
+        }
+        return stats;
+    }
+    /**
+     * Update type-field affinity tracking for intelligent NLP
+     * Tracks which fields commonly appear with which entity types
+     */
+    updateTypeFieldAffinity(entityId, field, value, operation) {
+        // Only track affinity for non-system fields (but allow 'noun' for type detection)
+        if (this.config.excludeFields.includes(field) && field !== 'noun')
+            return;
+        // For the 'noun' field, the value IS the entity type
+        let entityType = null;
+        if (field === 'noun') {
+            // This is the type definition itself
+            entityType = this.normalizeValue(value);
+        }
+        else {
+            // Find the noun type for this entity by looking for entries with this entityId
+            for (const [key, entry] of this.indexCache.entries()) {
+                if (key.startsWith('noun:') && entry.ids.has(entityId)) {
+                    entityType = key.split(':', 2)[1];
+                    break;
+                }
+            }
+        }
+        if (!entityType)
+            return; // No type found, skip affinity tracking
+        // Initialize affinity tracking for this type
+        if (!this.typeFieldAffinity.has(entityType)) {
+            this.typeFieldAffinity.set(entityType, new Map());
+        }
+        if (!this.totalEntitiesByType.has(entityType)) {
+            this.totalEntitiesByType.set(entityType, 0);
+        }
+        const typeFields = this.typeFieldAffinity.get(entityType);
+        if (operation === 'add') {
+            // Increment field count for this type
+            const currentCount = typeFields.get(field) || 0;
+            typeFields.set(field, currentCount + 1);
+            // Update total entities of this type (only count once per entity)
+            if (field === 'noun') {
+                this.totalEntitiesByType.set(entityType, this.totalEntitiesByType.get(entityType) + 1);
+            }
+        }
+        else if (operation === 'remove') {
+            // Decrement field count for this type
+            const currentCount = typeFields.get(field) || 0;
+            if (currentCount > 1) {
+                typeFields.set(field, currentCount - 1);
+            }
+            else {
+                typeFields.delete(field);
+            }
+            // Update total entities of this type
+            if (field === 'noun') {
+                const total = this.totalEntitiesByType.get(entityType);
+                if (total > 1) {
+                    this.totalEntitiesByType.set(entityType, total - 1);
+                }
+                else {
+                    this.totalEntitiesByType.delete(entityType);
+                    this.typeFieldAffinity.delete(entityType);
+                }
+            }
+        }
+    }
+    /**
+     * Get fields that commonly appear with a specific entity type
+     * Returns fields with their affinity scores (0-1)
+     */
+    async getFieldsForType(nounType) {
+        const typeFields = this.typeFieldAffinity.get(nounType);
+        const totalEntities = this.totalEntitiesByType.get(nounType);
+        if (!typeFields || !totalEntities) {
+            return [];
+        }
+        const fieldsWithAffinity = [];
+        for (const [field, count] of typeFields.entries()) {
+            const affinity = count / totalEntities; // 0-1 score
+            fieldsWithAffinity.push({
+                field,
+                affinity,
+                occurrences: count,
+                totalEntities
+            });
+        }
+        // Sort by affinity (most common fields first)
+        fieldsWithAffinity.sort((a, b) => b.affinity - a.affinity);
+        return fieldsWithAffinity;
+    }
+    /**
+     * Get type-field affinity statistics for analysis
+     */
+    async getTypeFieldAffinityStats() {
+        const typeBreakdown = {};
+        let totalFields = 0;
+        for (const [nounType, fieldsMap] of this.typeFieldAffinity.entries()) {
+            const totalEntities = this.totalEntitiesByType.get(nounType) || 0;
+            const fields = Array.from(fieldsMap.entries());
+            // Get top 5 fields for this type
+            const topFields = fields
+                .map(([field, count]) => ({ field, affinity: count / totalEntities }))
+                .sort((a, b) => b.affinity - a.affinity)
+                .slice(0, 5);
+            typeBreakdown[nounType] = {
+                totalEntities,
+                uniqueFields: fieldsMap.size,
+                topFields
+            };
+            totalFields += fieldsMap.size;
+        }
+        return {
+            totalTypes: this.typeFieldAffinity.size,
+            averageFieldsPerType: totalFields / Math.max(1, this.typeFieldAffinity.size),
+            typeBreakdown
+        };
+    }
 }
 //# sourceMappingURL=metadataIndex.js.map

package/dist/utils/nodeVersionCheck.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Node.js Version Compatibility Check
+ *
+ * Brainy requires Node.js 22.x LTS for maximum stability with ONNX Runtime.
+ * This prevents V8 HandleScope locking issues in worker threads.
+ */
+export interface VersionInfo {
+    current: string;
+    major: number;
+    isSupported: boolean;
+    recommendation: string;
+}
+/**
+ * Check if the current Node.js version is supported
+ */
+export declare function checkNodeVersion(): VersionInfo;
+/**
+ * Enforce Node.js version requirement with helpful error messaging
+ */
+export declare function enforceNodeVersion(): void;
+/**
+ * Soft warning for version issues (non-blocking)
+ */
+export declare function warnNodeVersion(): boolean;