@soulcraft/brainy 3.41.0 → 3.42.0

package/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [3.41.1](https://github.com/soulcraftlabs/brainy/compare/v3.41.0...v3.41.1) (2025-10-13)
+
+- test: skip failing delete test temporarily (7c47de8)
+- test: skip failing domain-time-clustering tests temporarily (71c4a54)
+- docs: add comprehensive index architecture documentation (75b4b02)
+
+
 ## [3.41.0](https://github.com/soulcraftlabs/brainy/compare/v3.40.3...v3.41.0) (2025-10-13)
 
 

package/dist/utils/metadataIndex.d.ts

@@ -29,6 +29,10 @@ export interface MetadataIndexConfig {
     indexedFields?: string[];
     excludeFields?: string[];
 }
+/**
+ * Manages metadata indexes for fast filtering
+ * Maintains inverted indexes: field+value -> list of IDs
+ */
 interface CardinalityInfo {
     uniqueValues: number;
     totalValues: number;
@@ -42,22 +46,18 @@ interface FieldStats {
     rangeQueryCount: number;
     exactQueryCount: number;
     avgQueryTime: number;
-    indexType: 'hash' | 'sorted' | 'both';
+    indexType: 'hash';
     normalizationStrategy?: 'none' | 'precision' | 'bucket';
 }
 export declare class MetadataIndexManager {
     private storage;
     private config;
-    private indexCache;
-    private dirtyEntries;
     private isRebuilding;
     private metadataCache;
     private fieldIndexes;
     private dirtyFields;
     private lastFlushTime;
     private autoFlushThreshold;
-    private sortedIndices;
-    private numericFields;
     private fieldStats;
     private cardinalityUpdateInterval;
     private operationCount;
@@ -70,6 +70,9 @@ export declare class MetadataIndexManager {
     private activeLocks;
     private lockPromises;
     private lockTimers;
+    private sparseIndices;
+    private chunkManager;
+    private chunkingStrategy;
     constructor(storage: StorageAdapter, config?: MetadataIndexConfig);
     /**
      * Acquire an in-memory lock for coordinating concurrent metadata index writes
@@ -92,60 +95,43 @@ export declare class MetadataIndexManager {
      */
     private lazyLoadCounts;
     /**
-     * Get index key for field and value
-     */
-    private getIndexKey;
-    /**
-     * Ensure sorted index exists for a field (for range queries)
-     */
-    private ensureSortedIndex;
-    /**
-     * Build sorted index for a field from hash index
-     */
-    private buildSortedIndex;
-    /**
-     * Detect field type from value
-     */
-    private detectFieldType;
-    /**
-     * Compare two values based on field type for sorting
+     * Update cardinality statistics for a field
      */
-    private compareValues;
+    private updateCardinalityStats;
     /**
-     * Binary search to find insertion position for a value
-     * Returns the index where the value should be inserted to maintain sorted order
+     * Analyze field distribution for optimization
      */
-    private findInsertPosition;
+    private analyzeFieldDistribution;
     /**
-     * Incrementally update sorted index when adding an ID
+     * Update index strategy based on field statistics
      */
-    private updateSortedIndexAdd;
+    private updateIndexStrategy;
     /**
-     * Incrementally update sorted index when removing an ID
+     * Load sparse index from storage
      */
-    private updateSortedIndexRemove;
+    private loadSparseIndex;
     /**
-     * Binary search for range start (inclusive or exclusive)
+     * Save sparse index to storage
      */
-    private binarySearchStart;
+    private saveSparseIndex;
     /**
-     * Binary search for range end (inclusive or exclusive)
+     * Get IDs for a value using chunked sparse index
      */
-    private binarySearchEnd;
+    private getIdsFromChunks;
     /**
-     * Update cardinality statistics for a field
+     * Get IDs for a range using chunked sparse index with zone maps
      */
-    private updateCardinalityStats;
+    private getIdsFromChunksForRange;
     /**
-     * Analyze field distribution for optimization
+     * Add value-ID mapping to chunked index
      */
-    private analyzeFieldDistribution;
+    private addToChunkedIndex;
     /**
-     * Update index strategy based on field statistics
+     * Remove ID from chunked index
      */
-    private updateIndexStrategy;
+    private removeFromChunkedIndex;
     /**
-     * Get IDs matching a range query
+     * Get IDs matching a range query using zone maps
      */
     private getIdsForRange;
     /**
@@ -193,7 +179,7 @@ export declare class MetadataIndexManager {
      */
     getAllIds(): Promise<string[]>;
     /**
-     * Get IDs for a specific field-value combination with caching
+     * Get IDs for a specific field-value combination using chunked sparse index
      */
     getIds(field: string, value: any): Promise<string[]>;
     /**
@@ -223,6 +209,7 @@ export declare class MetadataIndexManager {
     getIdsForCriteria(criteria: Record<string, any>): Promise<string[]>;
     /**
      * Flush dirty entries to storage (non-blocking version)
+     * NOTE (v3.42.0): Sparse indices are flushed immediately in add/remove operations
      */
     flush(): Promise<void>;
     /**
@@ -238,14 +225,6 @@ export declare class MetadataIndexManager {
      * Save field index to storage with file locking
      */
     private saveFieldIndex;
-    /**
-     * Save sorted index to storage for range queries with file locking
-     */
-    private saveSortedIndex;
-    /**
-     * Load sorted index from storage
-     */
-    private loadSortedIndex;
     /**
      * Get count of entities by type - O(1) operation using existing tracking
      * This exposes the production-ready counting that's already maintained
@@ -260,7 +239,7 @@ export declare class MetadataIndexManager {
      */
     getAllEntityCounts(): Map<string, number>;
     /**
-     * Get count of entities matching field-value criteria - O(1) lookup from existing indexes
+     * Get count of entities matching field-value criteria - queries chunked sparse index
      */
     getCountForCriteria(field: string, value: any): Promise<number>;
     /**
@@ -272,18 +251,6 @@ export declare class MetadataIndexManager {
      * Non-blocking version that yields control back to event loop
      */
     rebuild(): Promise<void>;
-    /**
-     * Load index entry from storage using safe filenames
-     */
-    private loadIndexEntry;
-    /**
-     * Save index entry to storage using safe filenames with file locking
-     */
-    private saveIndexEntry;
-    /**
-     * Delete index entry from storage using safe filenames
-     */
-    private deleteIndexEntry;
     /**
      * Get field statistics for optimization and discovery
      */