@soulcraft/brainy 3.40.1 → 3.40.3

package/CHANGELOG.md

@@ -2,6 +2,18 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [3.40.3](https://github.com/soulcraftlabs/brainy/compare/v3.40.2...v3.40.3) (2025-10-13)
+
+- fix: prevent metadata index file pollution by excluding high-cardinality fields (0c86c4f)
+
+
+### [3.40.2](https://github.com/soulcraftlabs/brainy/compare/v3.40.1...v3.40.2) (2025-10-13)
+
+
+### ⚡ Performance Improvements
+
+* more aggressive cache fairness to prevent thrashing ([829a8a6](https://github.com/soulcraftlabs/brainy/commit/829a8a61a23688aae1384b2844f1e75b1fd773d9))
+
 ### [3.40.1](https://github.com/soulcraftlabs/brainy/compare/v3.40.0...v3.40.1) (2025-10-13)
 
 

package/dist/utils/metadataIndex.js

@@ -39,7 +39,37 @@ export class MetadataIndexManager {
             rebuildThreshold: config.rebuildThreshold ?? 0.1,
             autoOptimize: config.autoOptimize ?? true,
             indexedFields: config.indexedFields ?? [],
-            excludeFields: config.excludeFields ?? ['id', 'createdAt', 'updatedAt', 'embedding', 'vector', 'embeddings', 'vectors']
+            excludeFields: config.excludeFields ?? [
+                // Timestamps (nearly unique per operation - causes massive file pollution)
+                'accessed',
+                'modified',
+                'createdAt',
+                'updatedAt',
+                'importedAt',
+                'extractedAt',
+                // UUIDs (unique per entity/relationship - creates one file per entity)
+                'id',
+                'parent',
+                'sourceId',
+                'targetId',
+                'source',
+                'target',
+                'owner',
+                // Paths and hashes (unique per file - creates one file per path)
+                'path',
+                'hash',
+                'url',
+                // Content fields (too large/unique - unnecessary for indexing)
+                'content',
+                'data',
+                'originalData',
+                '_data',
+                // Vectors (already excluded - keeping for backward compatibility)
+                'embedding',
+                'vector',
+                'embeddings',
+                'vectors'
+            ]
         };
         // Initialize metadata cache with similar config to search cache
         this.metadataCache = new MetadataIndexCache({

package/dist/utils/unifiedCache.d.ts

@@ -78,6 +78,12 @@ export declare class UnifiedCache {
      */
     private startFairnessMonitor;
     private checkFairness;
+    /**
+     * Proactive fairness check (v3.40.2)
+     * Called immediately when adding items to prevent imbalance formation
+     * Uses same thresholds as periodic check but runs on-demand
+     */
+    private checkProactiveFairness;
     /**
      * Force evict items of a specific type
      */

package/dist/utils/unifiedCache.js

@@ -50,7 +50,7 @@ export class UnifiedCache {
         this.config = {
             enableRequestCoalescing: true,
             enableFairnessCheck: true,
-            fairnessCheckInterval: 60000, // Check fairness every minute
+            fairnessCheckInterval: 30000, // Check fairness every 30 seconds (v3.40.2: was 60s)
             persistPatterns: true,
             enableMemoryMonitoring: true,
             memoryCheckInterval: 30000, // Check memory every 30s
@@ -149,6 +149,11 @@ export class UnifiedCache {
         this.currentSize += size;
         this.typeAccessCounts[type]++;
         this.totalAccessCount++;
+        // Proactive fairness check (v3.40.2): Check immediately if adding to a dominant type
+        // This prevents imbalance formation instead of reacting to it
+        if (this.config.enableFairnessCheck && this.cache.size > 10) {
+            this.checkProactiveFairness(type);
+        }
     }
     /**
      * Evict item with lowest value (access count / rebuild cost)
@@ -235,14 +240,36 @@ export class UnifiedCache {
             embedding: typeSizes.embedding / totalSize,
             other: typeSizes.other / totalSize
         };
-        // Check for starvation (90% cache but <10% accesses)
+        // Check for starvation (v3.40.2: more aggressive - 70% cache with <15% accesses)
+        // Previous: 90% cache, <10% access (too lenient, caused thrashing)
         for (const type of ['hnsw', 'metadata', 'embedding', 'other']) {
-            if (sizeRatios[type] > 0.9 && accessRatios[type] < 0.1) {
+            if (sizeRatios[type] > 0.7 && accessRatios[type] < 0.15) {
                 prodLog.warn(`Type ${type} is hogging cache (${(sizeRatios[type] * 100).toFixed(1)}% size, ${(accessRatios[type] * 100).toFixed(1)}% access)`);
                 this.evictType(type);
             }
         }
     }
+    /**
+     * Proactive fairness check (v3.40.2)
+     * Called immediately when adding items to prevent imbalance formation
+     * Uses same thresholds as periodic check but runs on-demand
+     */
+    checkProactiveFairness(addedType) {
+        // Quick check: only evaluate the type being added
+        let typeSize = 0;
+        for (const item of this.cache.values()) {
+            if (item.type === addedType) {
+                typeSize += item.size;
+            }
+        }
+        const sizeRatio = typeSize / (this.currentSize || 1);
+        const accessRatio = this.typeAccessCounts[addedType] / (this.totalAccessCount || 1);
+        // Same threshold as periodic check: 70% size, <15% access
+        if (sizeRatio > 0.7 && accessRatio < 0.15) {
+            prodLog.debug(`Proactive fairness: ${addedType} reaching dominance (${(sizeRatio * 100).toFixed(1)}% size, ${(accessRatio * 100).toFixed(1)}% access)`);
+            this.evictType(addedType);
+        }
+    }
     /**
      * Force evict items of a specific type
      */
@@ -256,8 +283,8 @@ export class UnifiedCache {
         }
         // Sort by score (lower is worse)
         candidates.sort((a, b) => a[1] - b[1]);
-        // Evict bottom 20% of this type
-        const evictCount = Math.max(1, Math.floor(candidates.length * 0.2));
+        // Evict bottom 50% of this type (v3.40.2: was 20%, too slow to prevent thrashing)
+        const evictCount = Math.max(1, Math.floor(candidates.length * 0.5));
         for (let i = 0; i < evictCount && i < candidates.length; i++) {
             const [key, , item] = candidates[i];
             this.currentSize -= item.size;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.40.1",
+  "version": "3.40.3",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",