@soulcraft/brainy 3.41.1 → 3.42.0

package/dist/utils/metadataIndexChunking.d.ts

@@ -0,0 +1,322 @@
+/**
+ * Metadata Index Chunking System
+ *
+ * Implements Adaptive Chunked Sparse Indexing inspired by ClickHouse MergeTree.
+ * Reduces file count from 560k to ~89 files (630x reduction) while maintaining performance.
+ *
+ * Key Components:
+ * - BloomFilter: Probabilistic membership testing (fast negative lookups)
+ * - SparseIndex: Directory of chunks with zone maps (range query optimization)
+ * - ChunkManager: Chunk lifecycle management (create/split/merge)
+ * - AdaptiveChunkingStrategy: Field-specific optimization strategies
+ *
+ * Architecture:
+ * - Each high-cardinality field gets a sparse index (directory)
+ * - Values are grouped into chunks (~50 values per chunk)
+ * - Each chunk has a bloom filter for fast negative lookups
+ * - Zone maps enable range query optimization
+ * - Backward compatible with existing flat file indexes
+ */
+import { StorageAdapter } from '../coreTypes.js';
+/**
+ * Zone Map for range query optimization
+ * Tracks min/max values in a chunk for fast range filtering
+ */
+export interface ZoneMap {
+    min: any;
+    max: any;
+    count: number;
+    hasNulls: boolean;
+}
+/**
+ * Chunk Descriptor
+ * Metadata about a chunk including its location, zone map, and bloom filter
+ */
+export interface ChunkDescriptor {
+    chunkId: number;
+    field: string;
+    valueCount: number;
+    idCount: number;
+    zoneMap: ZoneMap;
+    bloomFilterPath?: string;
+    lastUpdated: number;
+    splitThreshold: number;
+    mergeThreshold: number;
+}
+/**
+ * Sparse Index Data
+ * Directory structure mapping value ranges to chunks
+ */
+export interface SparseIndexData {
+    field: string;
+    strategy: 'hash' | 'sorted' | 'adaptive';
+    chunks: ChunkDescriptor[];
+    totalValues: number;
+    totalIds: number;
+    lastUpdated: number;
+    chunkSize: number;
+    version: number;
+}
+/**
+ * Chunk Data
+ * Actual storage of field:value -> IDs mappings
+ */
+export interface ChunkData {
+    chunkId: number;
+    field: string;
+    entries: Map<string, Set<string>>;
+    lastUpdated: number;
+}
+/**
+ * Bloom Filter for probabilistic membership testing
+ *
+ * Uses multiple hash functions to achieve ~1% false positive rate.
+ * Memory efficient: ~10 bits per element for 1% FPR.
+ *
+ * Properties:
+ * - Never produces false negatives (if returns false, definitely not in set)
+ * - May produce false positives (~1% with default config)
+ * - Space efficient compared to hash sets
+ * - Fast O(k) lookup where k = number of hash functions
+ */
+export declare class BloomFilter {
+    private bits;
+    private numBits;
+    private numHashFunctions;
+    private itemCount;
+    /**
+     * Create a Bloom filter
+     * @param expectedItems Expected number of items to store
+     * @param falsePositiveRate Target false positive rate (default: 0.01 = 1%)
+     */
+    constructor(expectedItems: number, falsePositiveRate?: number);
+    /**
+     * Add an item to the bloom filter
+     */
+    add(item: string): void;
+    /**
+     * Test if an item might be in the set
+     * @returns false = definitely not in set, true = might be in set
+     */
+    mightContain(item: string): boolean;
+    /**
+     * Get multiple hash positions for an item
+     * Uses double hashing technique: h(i) = (h1 + i*h2) mod m
+     */
+    private getHashPositions;
+    /**
+     * First hash function (FNV-1a variant)
+     */
+    private hash1;
+    /**
+     * Second hash function (DJB2)
+     */
+    private hash2;
+    /**
+     * Set a bit in the bit array
+     */
+    private setBit;
+    /**
+     * Get a bit from the bit array
+     */
+    private getBit;
+    /**
+     * Serialize to JSON for storage
+     */
+    toJSON(): any;
+    /**
+     * Deserialize from JSON
+     */
+    static fromJSON(data: any): BloomFilter;
+    /**
+     * Get estimated false positive rate based on current fill
+     */
+    getEstimatedFPR(): number;
+    /**
+     * Count number of set bits
+     */
+    private countSetBits;
+    /**
+     * Count set bits in a byte (population count)
+     */
+    private popcount;
+}
+/**
+ * Sparse Index manages the directory of chunks for a field
+ *
+ * Inspired by ClickHouse MergeTree sparse primary index:
+ * - Maintains sorted list of chunk descriptors
+ * - Uses zone maps for range query optimization
+ * - Enables fast chunk selection without loading all data
+ *
+ * Query Flow:
+ * 1. Check zone maps to find candidate chunks
+ * 2. Load bloom filters for candidate chunks (fast negative lookup)
+ * 3. Load only the chunks that likely contain the value
+ */
+export declare class SparseIndex {
+    private data;
+    private bloomFilters;
+    constructor(field: string, chunkSize?: number);
+    /**
+     * Find chunks that might contain a specific value
+     */
+    findChunksForValue(value: any): number[];
+    /**
+     * Find chunks that overlap with a value range
+     */
+    findChunksForRange(min?: any, max?: any): number[];
+    /**
+     * Check if a value falls within a zone map's range
+     */
+    private isValueInZoneMap;
+    /**
+     * Check if a range overlaps with a zone map
+     */
+    private doesRangeOverlap;
+    /**
+     * Register a chunk in the sparse index
+     */
+    registerChunk(descriptor: ChunkDescriptor, bloomFilter?: BloomFilter): void;
+    /**
+     * Update a chunk descriptor
+     */
+    updateChunk(chunkId: number, updates: Partial<ChunkDescriptor>): void;
+    /**
+     * Remove a chunk from the sparse index
+     */
+    removeChunk(chunkId: number): void;
+    /**
+     * Get chunk descriptor by ID
+     */
+    getChunk(chunkId: number): ChunkDescriptor | undefined;
+    /**
+     * Get all chunk IDs
+     */
+    getAllChunkIds(): number[];
+    /**
+     * Sort chunks by zone map min value
+     */
+    private sortChunks;
+    /**
+     * Get sparse index statistics
+     */
+    getStats(): {
+        field: string;
+        chunkCount: number;
+        avgValuesPerChunk: number;
+        avgIdsPerChunk: number;
+        totalValues: number;
+        totalIds: number;
+        estimatedFPR: number;
+    };
+    /**
+     * Serialize to JSON for storage
+     */
+    toJSON(): any;
+    /**
+     * Deserialize from JSON
+     */
+    static fromJSON(data: any): SparseIndex;
+}
+/**
+ * ChunkManager handles chunk operations: create, split, merge, compact
+ *
+ * Responsibilities:
+ * - Maintain optimal chunk sizes (~50 values per chunk)
+ * - Split chunks that grow too large (> 80 values)
+ * - Merge chunks that become too small (< 20 values)
+ * - Update zone maps and bloom filters
+ * - Coordinate with storage adapter
+ */
+export declare class ChunkManager {
+    private storage;
+    private chunkCache;
+    private nextChunkId;
+    constructor(storage: StorageAdapter);
+    /**
+     * Create a new chunk for a field
+     */
+    createChunk(field: string, initialEntries?: Map<string, Set<string>>): Promise<ChunkData>;
+    /**
+     * Load a chunk from storage
+     */
+    loadChunk(field: string, chunkId: number): Promise<ChunkData | null>;
+    /**
+     * Save a chunk to storage
+     */
+    saveChunk(chunk: ChunkData): Promise<void>;
+    /**
+     * Add a value-ID mapping to a chunk
+     */
+    addToChunk(chunk: ChunkData, value: string, id: string): Promise<void>;
+    /**
+     * Remove an ID from a chunk
+     */
+    removeFromChunk(chunk: ChunkData, value: string, id: string): Promise<void>;
+    /**
+     * Calculate zone map for a chunk
+     */
+    calculateZoneMap(chunk: ChunkData): ZoneMap;
+    /**
+     * Create bloom filter for a chunk
+     */
+    createBloomFilter(chunk: ChunkData): BloomFilter;
+    /**
+     * Split a chunk if it's too large
+     */
+    splitChunk(chunk: ChunkData, sparseIndex: SparseIndex): Promise<{
+        chunk1: ChunkData;
+        chunk2: ChunkData;
+    }>;
+    /**
+     * Delete a chunk
+     */
+    deleteChunk(field: string, chunkId: number): Promise<void>;
+    /**
+     * Get chunk storage path
+     */
+    private getChunkPath;
+    /**
+     * Get next available chunk ID for a field
+     */
+    private getNextChunkId;
+    /**
+     * Clear chunk cache (for testing/maintenance)
+     */
+    clearCache(): void;
+}
+/**
+ * Determines optimal chunking strategy based on field characteristics
+ */
+export declare class AdaptiveChunkingStrategy {
+    /**
+     * Determine if a field should use chunking
+     */
+    shouldUseChunking(fieldStats: {
+        uniqueValues: number;
+        totalValues: number;
+        distribution: 'uniform' | 'skewed' | 'sparse';
+    }): boolean;
+    /**
+     * Determine optimal chunk size for a field
+     */
+    getOptimalChunkSize(fieldStats: {
+        uniqueValues: number;
+        distribution: 'uniform' | 'skewed' | 'sparse';
+        avgIdsPerValue: number;
+    }): number;
+    /**
+     * Determine if a chunk should be split
+     */
+    shouldSplit(chunk: {
+        valueCount: number;
+        idCount: number;
+    }, threshold: number): boolean;
+    /**
+     * Determine if chunks should be merged
+     */
+    shouldMerge(chunks: Array<{
+        valueCount: number;
+    }>, threshold: number): boolean;
+}