@soulcraft/brainy 3.40.2 → 3.41.0

package/CHANGELOG.md

@@ -2,6 +2,18 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [3.41.0](https://github.com/soulcraftlabs/brainy/compare/v3.40.3...v3.41.0) (2025-10-13)
+
+
+### ✨ Features
+
+* automatic temporal bucketing for metadata indexes ([b3edd4b](https://github.com/soulcraftlabs/brainy/commit/b3edd4b60a49d26d1ca776d459aa013736a0db9d))
+
+### [3.40.3](https://github.com/soulcraftlabs/brainy/compare/v3.40.2...v3.40.3) (2025-10-13)
+
+- fix: prevent metadata index file pollution by excluding high-cardinality fields (0c86c4f)
+
+
 ### [3.40.2](https://github.com/soulcraftlabs/brainy/compare/v3.40.1...v3.40.2) (2025-10-13)
 
 

package/dist/utils/metadataIndex.js

@@ -39,7 +39,25 @@ export class MetadataIndexManager {
             rebuildThreshold: config.rebuildThreshold ?? 0.1,
             autoOptimize: config.autoOptimize ?? true,
             indexedFields: config.indexedFields ?? [],
-            excludeFields: config.excludeFields ?? ['id', 'createdAt', 'updatedAt', 'embedding', 'vector', 'embeddings', 'vectors']
+            excludeFields: config.excludeFields ?? [
+                // ONLY exclude truly un-indexable fields (binary data, large content)
+                // Timestamps are NOW indexed with automatic bucketing (prevents pollution)
+                // Vectors and embeddings (binary data, already have HNSW indexes)
+                'embedding',
+                'vector',
+                'embeddings',
+                'vectors',
+                // Large content fields (too large for metadata indexing)
+                'content',
+                'data',
+                'originalData',
+                '_data',
+                // Primary keys (use direct lookups instead)
+                'id'
+                // NOTE: 'accessed', 'modified', 'createdAt', etc. are NO LONGER excluded!
+                // They are now indexed with automatic 1-minute bucketing to prevent file pollution
+                // This enables range queries like: modified > yesterday
+            ]
         };
         // Initialize metadata cache with similar config to search cache
         this.metadataCache = new MetadataIndexCache({
@@ -134,7 +152,7 @@ export class MetadataIndexManager {
      * Get index key for field and value
      */
     getIndexKey(field, value) {
-        const normalizedValue = this.normalizeValue(value);
+        const normalizedValue = this.normalizeValue(value, field); // Pass field for bucketing!
         return `${field}:${normalizedValue}`;
     }
     /**
@@ -267,7 +285,7 @@ export class MetadataIndexManager {
             });
         }
         const sortedIndex = this.sortedIndices.get(field);
-        const normalizedValue = this.normalizeValue(value);
+        const normalizedValue = this.normalizeValue(value, field); // Pass field for bucketing!
         // Find where this value should be in the sorted array
         const insertPos = this.findInsertPosition(sortedIndex.values, normalizedValue, sortedIndex.fieldType);
         if (insertPos < sortedIndex.values.length &&
@@ -289,7 +307,7 @@ export class MetadataIndexManager {
         const sortedIndex = this.sortedIndices.get(field);
         if (!sortedIndex || sortedIndex.values.length === 0)
             return;
-        const normalizedValue = this.normalizeValue(value);
+        const normalizedValue = this.normalizeValue(value, field); // Pass field for bucketing!
         // Binary search to find the value
         const pos = this.findInsertPosition(sortedIndex.values, normalizedValue, sortedIndex.fieldType);
         if (pos < sortedIndex.values.length &&
@@ -429,12 +447,10 @@ export class MetadataIndexManager {
         else {
             stats.indexType = 'hash';
         }
-        // Determine normalization strategy for high cardinality fields
+        // Determine normalization strategy for high cardinality NON-temporal fields
+        // (Temporal fields are already bucketed in normalizeValue from the start!)
         if (hasHighCardinality) {
-            if (field.toLowerCase().includes('time') || field.toLowerCase().includes('date')) {
-                stats.normalizationStrategy = 'bucket'; // Time bucketing
-            }
-            else if (isNumeric) {
+            if (isNumeric) {
                 stats.normalizationStrategy = 'precision'; // Reduce float precision
             }
             else {
@@ -494,7 +510,7 @@ export class MetadataIndexManager {
      * Generate value chunk filename for scalable storage
      */
     getValueChunkFilename(field, value, chunkIndex = 0) {
-        const normalizedValue = this.normalizeValue(value);
+        const normalizedValue = this.normalizeValue(value, field); // Pass field for bucketing!
         const safeValue = this.makeSafeFilename(normalizedValue);
         return `${field}_${safeValue}_chunk${chunkIndex}`;
     }
@@ -516,19 +532,25 @@ export class MetadataIndexManager {
             return '__NULL__';
         if (typeof value === 'boolean')
             return value ? '__TRUE__' : '__FALSE__';
-        // Apply smart normalization based on field statistics
+        // ALWAYS apply bucketing to temporal fields (prevents pollution from the start!)
+        // This is the key fix: don't wait for cardinality stats, just bucket immediately
+        if (field && typeof value === 'number') {
+            const fieldLower = field.toLowerCase();
+            const isTemporal = fieldLower.includes('time') || fieldLower.includes('date') ||
+                fieldLower.includes('accessed') || fieldLower.includes('modified') ||
+                fieldLower.includes('created') || fieldLower.includes('updated');
+            if (isTemporal) {
+                // Apply time bucketing immediately (no need to wait for stats)
+                const bucketSize = this.TIMESTAMP_PRECISION_MS; // 1 minute buckets
+                const bucketed = Math.floor(value / bucketSize) * bucketSize;
+                return bucketed.toString();
+            }
+        }
+        // Apply smart normalization based on field statistics (for non-temporal fields)
         if (field && this.fieldStats.has(field)) {
             const stats = this.fieldStats.get(field);
             const strategy = stats.normalizationStrategy;
-            if (strategy === 'bucket' && typeof value === 'number') {
-                // Time bucketing for timestamps
-                if (field.toLowerCase().includes('time') || field.toLowerCase().includes('date')) {
-                    const bucketSize = this.TIMESTAMP_PRECISION_MS;
-                    const bucketed = Math.floor(value / bucketSize) * bucketSize;
-                    return bucketed.toString();
-                }
-            }
-            else if (strategy === 'precision' && typeof value === 'number') {
+            if (strategy === 'precision' && typeof value === 'number') {
                 // Reduce float precision for high cardinality numeric fields
                 const rounded = Math.round(value * Math.pow(10, this.FLOAT_PRECISION)) / Math.pow(10, this.FLOAT_PRECISION);
                 return rounded.toString();
@@ -631,7 +653,7 @@ export class MetadataIndexManager {
                 const loadedEntry = await this.loadIndexEntry(key);
                 entry = loadedEntry ?? {
                     field,
-                    value: this.normalizeValue(value),
+                    value: this.normalizeValue(value, field), // Pass field for bucketing!
                     ids: new Set(),
                     lastUpdated: Date.now()
                 };
@@ -704,7 +726,7 @@ export class MetadataIndexManager {
             };
             this.fieldIndexes.set(field, fieldIndex);
         }
-        const normalizedValue = this.normalizeValue(value);
+        const normalizedValue = this.normalizeValue(value, field); // Pass field for bucketing!
         fieldIndex.values[normalizedValue] = (fieldIndex.values[normalizedValue] || 0) + delta;
         // Remove if count drops to 0
         if (fieldIndex.values[normalizedValue] <= 0) {
@@ -1800,7 +1822,7 @@ export class MetadataIndexManager {
         let entityType = null;
         if (field === 'noun') {
             // This is the type definition itself
-            entityType = this.normalizeValue(value);
+            entityType = this.normalizeValue(value, field); // Pass field for bucketing!
         }
         else {
             // Find the noun type for this entity by looking for entries with this entityId

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.40.2",
+  "version": "3.41.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",