jexidb 2.1.9 → 2.2.1

package/src/managers/IndexManager.mjs

@@ -1,2371 +1,2435 @@
-import { Mutex } from 'async-mutex'
-import { normalizeCriteriaOperators } from '../utils/operatorNormalizer.mjs'
-
-export default class IndexManager {
-  constructor(opts, databaseMutex = null, database = null) {
-    this.opts = Object.assign({}, opts)
-    this.index = Object.assign({data: {}}, this.opts.index)
-    this.totalLines = 0
-    this.rangeThreshold = 10 // Sensible threshold: 10+ consecutive numbers justify ranges
-    this.binarySearchThreshold = 32 // Much higher for better performance
-    this.database = database // Reference to database for term manager access
-    
-    // CRITICAL: Use database mutex to prevent deadlocks
-    // If no database mutex provided, create a local one (for backward compatibility)
-    this.mutex = databaseMutex || new Mutex()
-    
-    this.indexedFields = []
-    this.setIndexesConfig(this.opts.indexes)
-  }
-
-  setTotalLines(total) {
-    this.totalLines = total
-  }
-
-  /**
-   * Update indexes configuration and ensure internal structures stay in sync
-   * @param {Object|Array<string>} indexes
-   */
-  setIndexesConfig(indexes) {
-    if (!indexes) {
-      this.opts.indexes = undefined
-      this.indexedFields = []
-      return
-    }
-
-    if (Array.isArray(indexes)) {
-      const fields = indexes.map(field => String(field))
-      this.indexedFields = fields
-
-      const normalizedConfig = {}
-      for (const field of fields) {
-        const existingConfig = (!Array.isArray(this.opts.indexes) && typeof this.opts.indexes === 'object') ? this.opts.indexes[field] : undefined
-        normalizedConfig[field] = existingConfig ?? 'auto'
-        if (!this.index.data[field]) {
-          this.index.data[field] = {}
-        }
-      }
-      this.opts.indexes = normalizedConfig
-      return
-    }
-
-    if (typeof indexes === 'object') {
-      this.opts.indexes = Object.assign({}, indexes)
-      this.indexedFields = Object.keys(this.opts.indexes)
-
-      for (const field of this.indexedFields) {
-        if (!this.index.data[field]) {
-          this.index.data[field] = {}
-        }
-      }
-    }
-  }
-
-  /**
-   * Check if a field is configured as an index
-   * @param {string} field - Field name
-   * @returns {boolean}
-   */
-  isFieldIndexed(field) {
-    if (!field) return false
-    if (!Array.isArray(this.indexedFields)) {
-      return false
-    }
-    return this.indexedFields.includes(field)
-  }
-
-  /**
-   * Determine whether the index has usable data for a given field
-   * @param {string} field - Field name
-   * @returns {boolean}
-   */
-  hasUsableIndexData(field) {
-    if (!field) return false
-    const fieldData = this.index?.data?.[field]
-    if (!fieldData || typeof fieldData !== 'object') {
-      return false
-    }
-
-    for (const key in fieldData) {
-      if (!Object.prototype.hasOwnProperty.call(fieldData, key)) continue
-      const entry = fieldData[key]
-      if (!entry) continue
-
-      if (entry.set && typeof entry.set.size === 'number' && entry.set.size > 0) {
-        return true
-      }
-
-      if (Array.isArray(entry.ranges) && entry.ranges.length > 0) {
-        const hasRangeData = entry.ranges.some(range => {
-          if (range === null || typeof range === 'undefined') {
-            return false
-          }
-          if (typeof range === 'object') {
-            const count = typeof range.count === 'number' ? range.count : 0
-            return count > 0
-          }
-          // When ranges are stored as individual numbers
-          return true
-        })
-
-        if (hasRangeData) {
-          return true
-        }
-      }
-    }
-
-    return false
-  }
-
-  // Ultra-fast range conversion - only for very large datasets
-  _toRanges(numbers) {
-    if (numbers.length === 0) return []
-    if (numbers.length < this.rangeThreshold) return numbers // Keep as-is for small arrays
-    
-    const sorted = numbers.sort((a, b) => a - b) // Sort in-place
-    const ranges = []
-    let start = sorted[0]
-    let count = 1
-    
-    for (let i = 1; i < sorted.length; i++) {
-      if (sorted[i] === sorted[i-1] + 1) {
-        count++
-      } else {
-        // End of consecutive sequence
-        if (count >= this.rangeThreshold) {
-          ranges.push({start, count})
-        } else {
-          // Add individual numbers for small sequences
-          for (let j = start; j < start + count; j++) {
-            ranges.push(j)
-          }
-        }
-        start = sorted[i]
-        count = 1
-      }
-    }
-    
-    // Handle last sequence
-    if (count >= this.rangeThreshold) {
-      ranges.push({start, count})
-    } else {
-      for (let j = start; j < start + count; j++) {
-        ranges.push(j)
-      }
-    }
-    
-    return ranges
-  }
-
-  // Ultra-fast range expansion
-  _fromRanges(ranges) {
-    if (!ranges || ranges.length === 0) return []
-    
-    const numbers = []
-    for (const item of ranges) {
-      if (typeof item === 'object' && item.start !== undefined) {
-        // It's a range - use direct loop for maximum speed
-        const end = item.start + item.count
-        for (let i = item.start; i < end; i++) {
-          numbers.push(i)
-        }
-      } else {
-        // It's an individual number
-        numbers.push(item)
-      }
-    }
-    return numbers
-  }
-
-  // Ultra-fast lookup - optimized for Set operations
-  _hasLineNumber(hybridData, lineNumber) {
-    if (!hybridData) return false
-    
-    // Check in Set first (O(1)) - most common case
-    if (hybridData.set && hybridData.set.has(lineNumber)) {
-      return true
-    }
-    
-    // Check in ranges only if necessary
-    if (hybridData.ranges && hybridData.ranges.length > 0) {
-      return this._searchInRanges(hybridData.ranges, lineNumber)
-    }
-    
-    return false
-  }
-
-  // Optimized search strategy
-  _searchInRanges(ranges, lineNumber) {
-    if (ranges.length < this.binarySearchThreshold) {
-      // Linear search for small ranges
-      return this._linearSearchRanges(ranges, lineNumber)
-    } else {
-      // Binary search for large ranges
-      return this._binarySearchRanges(ranges, lineNumber)
-    }
-  }
-
-  // Ultra-fast linear search
-  _linearSearchRanges(ranges, lineNumber) {
-    for (const item of ranges) {
-      if (typeof item === 'object' && item.start !== undefined) {
-        // It's a range
-        if (lineNumber >= item.start && lineNumber < item.start + item.count) {
-          return true
-        }
-      } else if (item === lineNumber) {
-        // It's an individual number
-        return true
-      }
-    }
-    return false
-  }
-
-  // Optimized binary search
-  _binarySearchRanges(ranges, lineNumber) {
-    let left = 0
-    let right = ranges.length - 1
-    
-    while (left <= right) {
-      const mid = Math.floor((left + right) / 2)
-      const range = ranges[mid]
-      
-      if (typeof range === 'object' && range.start !== undefined) {
-        // It's a range
-        if (lineNumber >= range.start && lineNumber < range.start + range.count) {
-          return true
-        } else if (lineNumber < range.start) {
-          right = mid - 1
-        } else {
-          left = mid + 1
-        }
-      } else {
-        // It's an individual number
-        if (range === lineNumber) {
-          return true
-        } else if (range < lineNumber) {
-          left = mid + 1
-        } else {
-          right = mid - 1
-        }
-      }
-    }
-    
-    return false
-  }
-
-  // Ultra-fast add operation - minimal overhead
-  _addLineNumber(hybridData, lineNumber) {
-    // Initialize structure if needed
-    if (!hybridData) {
-      hybridData = { set: new Set(), ranges: [] }
-    }
-    
-    // Add to Set directly (fastest path)
-    if (!hybridData.set) {
-      hybridData.set = new Set()
-    }
-    hybridData.set.add(lineNumber)
-    
-    // Optimize to ranges when Set gets reasonably large
-    if (hybridData.set.size >= this.rangeThreshold * 2) { // 20 elements
-      if (this.opts.debugMode) {
-        console.log(`🔧 Triggering range optimization: Set size ${hybridData.set.size} >= threshold ${this.rangeThreshold * 2}`)
-      }
-      this._optimizeToRanges(hybridData)
-    }
-    
-    return hybridData
-  }
-
-  // Ultra-fast remove operation
-  _removeLineNumber(hybridData, lineNumber) {
-    if (!hybridData) {
-      return hybridData
-    }
-    
-    // Remove from Set (fast path)
-    if (hybridData.set) {
-      hybridData.set.delete(lineNumber)
-    }
-    
-    // Remove from ranges (less common)
-    if (hybridData.ranges) {
-      hybridData.ranges = this._removeFromRanges(hybridData.ranges, lineNumber)
-    }
-    
-    return hybridData
-  }
-
-  // Optimized range removal
-  _removeFromRanges(ranges, lineNumber) {
-    if (!ranges || ranges.length === 0) return ranges
-    
-    const newRanges = []
-    
-    for (const item of ranges) {
-      if (typeof item === 'object' && item.start !== undefined) {
-        // It's a range
-        if (lineNumber >= item.start && lineNumber < item.start + item.count) {
-          // Split range if needed
-          if (lineNumber === item.start) {
-            // Remove first element
-            if (item.count > 1) {
-              newRanges.push({ start: item.start + 1, count: item.count - 1 })
-            }
-          } else if (lineNumber === item.start + item.count - 1) {
-            // Remove last element
-            if (item.count > 1) {
-              newRanges.push({ start: item.start, count: item.count - 1 })
-            }
-          } else {
-            // Remove from middle - split into two ranges
-            const beforeCount = lineNumber - item.start
-            const afterCount = item.count - beforeCount - 1
-            
-            if (beforeCount >= this.rangeThreshold) {
-              newRanges.push({ start: item.start, count: beforeCount })
-            } else {
-              // Add individual numbers for small sequences
-              for (let i = item.start; i < lineNumber; i++) {
-                newRanges.push(i)
-              }
-            }
-            
-            if (afterCount >= this.rangeThreshold) {
-              newRanges.push({ start: lineNumber + 1, count: afterCount })
-            } else {
-              // Add individual numbers for small sequences
-              for (let i = lineNumber + 1; i < item.start + item.count; i++) {
-                newRanges.push(i)
-              }
-            }
-          }
-        } else {
-          newRanges.push(item)
-        }
-      } else if (item !== lineNumber) {
-        // It's an individual number
-        newRanges.push(item)
-      }
-    }
-    
-    return newRanges
-  }
-
-  // Ultra-lazy range conversion - only when absolutely necessary
-  _optimizeToRanges(hybridData) {
-    if (!hybridData.set || hybridData.set.size === 0) {
-      return
-    }
-    
-    if (this.opts.debugMode) {
-      console.log(`🔧 Starting range optimization for Set with ${hybridData.set.size} elements`)
-    }
-    
-    // Only convert if we have enough data to make it worthwhile
-    if (hybridData.set.size < this.rangeThreshold) {
-      return
-    }
-    
-    // Convert Set to array and find consecutive sequences
-    const numbers = Array.from(hybridData.set).sort((a, b) => a - b)
-    const ranges = []
-    
-    let start = numbers[0]
-    let count = 1
-    
-    for (let i = 1; i < numbers.length; i++) {
-      if (numbers[i] === numbers[i-1] + 1) {
-        count++
-      } else {
-        // End of consecutive sequence
-        if (count >= this.rangeThreshold) {
-          ranges.push({start, count})
-          // Remove these numbers from Set
-          for (let j = start; j < start + count; j++) {
-            hybridData.set.delete(j)
-          }
-        }
-        start = numbers[i]
-        count = 1
-      }
-    }
-    
-    // Handle last sequence
-    if (count >= this.rangeThreshold) {
-      ranges.push({start, count})
-      for (let j = start; j < start + count; j++) {
-        hybridData.set.delete(j)
-      }
-    }
-    
-    // Add new ranges to existing ranges
-    if (ranges.length > 0) {
-      if (!hybridData.ranges) {
-        hybridData.ranges = []
-      }
-      hybridData.ranges.push(...ranges)
-      // Keep ranges sorted for efficient binary search
-      hybridData.ranges.sort((a, b) => {
-        const aStart = typeof a === 'object' ? a.start : a
-        const bStart = typeof b === 'object' ? b.start : b
-        return aStart - bStart
-      })
-    }
-  }
-
-  // Ultra-fast get all line numbers
-  _getAllLineNumbers(hybridData) {
-    if (!hybridData) return []
-    
-    // Use generator for lazy evaluation and better memory efficiency
-    return Array.from(this._getAllLineNumbersGenerator(hybridData))
-  }
-
-  // OPTIMIZATION: Generator-based approach for better memory efficiency
-  *_getAllLineNumbersGenerator(hybridData) {
-    const normalizeLineNumber = (value) => {
-      if (typeof value === 'number') {
-        return value
-      }
-      if (typeof value === 'string') {
-        const parsed = Number(value)
-        return Number.isNaN(parsed) ? value : parsed
-      }
-      if (typeof value === 'bigint') {
-        const maxSafe = BigInt(Number.MAX_SAFE_INTEGER)
-        return value <= maxSafe ? Number(value) : value
-      }
-      return value
-    }
-
-    // Yield from Set (fastest path)
-    if (hybridData.set) {
-      for (const num of hybridData.set) {
-        yield normalizeLineNumber(num)
-      }
-    }
-    
-    // Yield from ranges (optimized)
-    if (hybridData.ranges) {
-      for (const item of hybridData.ranges) {
-        if (typeof item === 'object' && item.start !== undefined) {
-          // It's a range - use direct loop for better performance
-          const end = item.start + item.count
-          for (let i = item.start; i < end; i++) {
-            yield normalizeLineNumber(i)
-          }
-        } else {
-          // It's an individual number
-          yield normalizeLineNumber(item)
-        }
-      }
-    }
-  }
-
-  // OPTIMIZATION 6: Ultra-fast add operation with incremental index updates
-  async add(row, lineNumber) {
-    if (typeof row !== 'object' || !row) {
-      throw new Error('Invalid \'row\' parameter, it must be an object')
-    }
-    if (typeof lineNumber !== 'number') {
-      throw new Error('Invalid line number')
-    }
-    
-    // OPTIMIZATION 6: Use direct field access with minimal operations
-    const data = this.index.data
-    
-    // OPTIMIZATION 6: Pre-allocate field structures for better performance
-    const fields = Object.keys(this.opts.indexes || {})
-    for (const field of fields) {
-      // PERFORMANCE: Check if this is a term mapping field once
-      const isTermMappingField = this.database?.termManager && 
-        this.database.termManager.termMappingFields && 
-        this.database.termManager.termMappingFields.includes(field)
-      
-      // CRITICAL FIX: For term mapping fields, prefer ${field}Ids if available
-      // Records processed by processTermMapping have term IDs in ${field}Ids
-      // Records loaded from file have term IDs directly in ${field} (after restoreTermIdsAfterDeserialization)
-      let value
-      if (isTermMappingField) {
-        const termIdsField = `${field}Ids`
-        const termIds = row[termIdsField]
-        if (termIds && Array.isArray(termIds) && termIds.length > 0) {
-          // Use term IDs from ${field}Ids (preferred - from processTermMapping)
-          value = termIds
-        } else {
-          // Fallback: use field directly (for records loaded from file that have term IDs in field)
-          value = row[field]
-        }
-      } else {
-        value = row[field]
-      }
-      
-      if (value !== undefined && value !== null) {
-        // OPTIMIZATION 6: Initialize field structure if it doesn't exist
-        if (!data[field]) {
-          data[field] = {}
-        }
-        
-        const values = Array.isArray(value) ? value : [value]
-        for (const val of values) {
-          let key
-          
-          if (isTermMappingField && typeof val === 'number') {
-            // For term mapping fields, values are already term IDs
-            key = String(val)
-          } else if (isTermMappingField && typeof val === 'string') {
-            // Fallback: convert string to term ID
-            // CRITICAL: During indexing (add), we should use getTermId() to create IDs if needed
-            // This is different from queries where we use getTermIdWithoutIncrement() to avoid creating new IDs
-            const termId = this.database.termManager.getTermId(val)
-            key = String(termId)
-          } else {
-            // For non-term-mapping fields (including array:number), use values directly
-            key = String(val)
-          }
-          
-          // OPTIMIZATION 6: Use direct assignment for better performance
-          if (!data[field][key]) {
-            data[field][key] = { set: new Set(), ranges: [] }
-          }
-          
-          // OPTIMIZATION 6: Direct Set operation - fastest possible
-          data[field][key].set.add(lineNumber)
-          
-          // OPTIMIZATION 6: Lazy range optimization - only when beneficial
-          if (data[field][key].set.size >= this.rangeThreshold * 3) {
-            this._optimizeToRanges(data[field][key])
-          }
-        }
-      }
-    }
-  }
-
-  /**
-   * OPTIMIZATION 6: Add multiple records to the index in batch with optimized operations
-   * @param {Array} records - Records to add
-   * @param {number} startLineNumber - Starting line number
-   */
-  async addBatch(records, startLineNumber) {
-    if (!records || !records.length) return
-    
-    // OPTIMIZATION 6: Pre-allocate index structures for better performance
-    const data = this.index.data
-    const fields = Object.keys(this.opts.indexes || {})
-    
-    for (const field of fields) {
-      if (!data[field]) {
-        data[field] = {}
-      }
-    }
-
-    // OPTIMIZATION 6: Use Map for batch processing to reduce lookups
-    const fieldUpdates = new Map()
-    
-    // OPTIMIZATION 6: Process all records in batch with optimized data structures
-    for (let i = 0; i < records.length; i++) {
-      const row = records[i]
-      const lineNumber = startLineNumber + i
-
-      for (const field of fields) {
-        // PERFORMANCE: Check if this is a term mapping field once
-        const isTermMappingField = this.database?.termManager && 
-          this.database.termManager.termMappingFields && 
-          this.database.termManager.termMappingFields.includes(field)
-        
-        // CRITICAL FIX: For term mapping fields, prefer ${field}Ids if available
-        // Records processed by processTermMapping have term IDs in ${field}Ids
-        // Records loaded from file have term IDs directly in ${field} (after restoreTermIdsAfterDeserialization)
-        let value
-        if (isTermMappingField) {
-          const termIdsField = `${field}Ids`
-          const termIds = row[termIdsField]
-          if (termIds && Array.isArray(termIds) && termIds.length > 0) {
-            // Use term IDs from ${field}Ids (preferred - from processTermMapping)
-            value = termIds
-          } else {
-            // Fallback: use field directly (for records loaded from file that have term IDs in field)
-            value = row[field]
-          }
-        } else {
-          value = row[field]
-        }
-        
-        if (value !== undefined && value !== null) {
-          const values = Array.isArray(value) ? value : [value]
-          for (const val of values) {
-            let key
-            
-            if (isTermMappingField && typeof val === 'number') {
-              // For term mapping fields, values are already term IDs
-              key = String(val)
-            } else if (isTermMappingField && typeof val === 'string') {
-              // Fallback: convert string to term ID
-              // CRITICAL: During indexing (addBatch), we should use getTermId() to create IDs if needed
-              // This is different from queries where we use getTermIdWithoutIncrement() to avoid creating new IDs
-              const termId = this.database.termManager.getTermId(val)
-              key = String(termId)
-            } else {
-              // For non-term-mapping fields (including array:number), use values directly
-              key = String(val)
-            }
-            
-            // OPTIMIZATION 6: Use Map for efficient batch updates
-            if (!fieldUpdates.has(field)) {
-              fieldUpdates.set(field, new Map())
-            }
-            
-            const fieldMap = fieldUpdates.get(field)
-            if (!fieldMap.has(key)) {
-              fieldMap.set(key, new Set())
-            }
-            
-            fieldMap.get(key).add(lineNumber)
-          }
-        }
-      }
-    }
-    
-    // OPTIMIZATION 6: Apply all updates in batch for better performance
-    for (const [field, fieldMap] of fieldUpdates) {
-      for (const [key, lineNumbers] of fieldMap) {
-        if (!data[field][key]) {
-          data[field][key] = { set: new Set(), ranges: [] }
-        }
-        
-        // OPTIMIZATION 6: Add all line numbers at once
-        for (const lineNumber of lineNumbers) {
-          data[field][key].set.add(lineNumber)
-        }
-        
-        // OPTIMIZATION 6: Lazy range optimization - only when beneficial
-        if (data[field][key].set.size >= this.rangeThreshold * 3) {
-          this._optimizeToRanges(data[field][key])
-        }
-      }
-    }
-  }
-
-  // Ultra-fast dry remove
-  dryRemove(ln) {
-    const data = this.index.data
-    for (const field in data) {
-      for (const value in data[field]) {
-        // Direct Set operation - fastest possible
-        if (data[field][value].set) {
-          data[field][value].set.delete(ln)
-        }
-        if (data[field][value].ranges) {
-          data[field][value].ranges = this._removeFromRanges(data[field][value].ranges, ln)
-        }
-        // Remove empty entries
-        if ((!data[field][value].set || data[field][value].set.size === 0) &&
-            (!data[field][value].ranges || data[field][value].ranges.length === 0)) {
-          delete data[field][value]
-        }
-      }
-    }
-  }
-
-
-  // Cleanup method to free memory
-  cleanup() {
-    const data = this.index.data
-    for (const field in data) {
-      for (const value in data[field]) {
-        if (data[field][value].set) {
-          if (typeof data[field][value].set.clearAll === 'function') {
-            data[field][value].set.clearAll()
-          } else if (typeof data[field][value].set.clear === 'function') {
-            data[field][value].set.clear()
-          }
-        }
-        if (data[field][value].ranges) {
-          data[field][value].ranges.length = 0
-        }
-      }
-      // Clear the entire field
-      data[field] = {}
-    }
-    // Clear all data
-    this.index.data = {}
-    this.totalLines = 0
-  }
-
-  // Clear all indexes
-  clear() {
-    this.index.data = {}
-    this.totalLines = 0
-  }
-
-  
-
-  // Update a record in the index
-  async update(oldRecord, newRecord, lineNumber = null) {
-    if (!oldRecord || !newRecord) return
-    
-    // Remove old record by ID
-    await this.remove(oldRecord)
-    
-    // Add new record with provided line number or use hash of the ID
-    const actualLineNumber = lineNumber !== null ? lineNumber : this._getIdAsNumber(newRecord.id)
-    await this.add(newRecord, actualLineNumber)
-  }
-
-  // Convert string ID to number for line number
-  _getIdAsNumber(id) {
-    if (typeof id === 'number') return id
-    if (typeof id === 'string') {
-      // Simple hash function to convert string to number
-      let hash = 0
-      for (let i = 0; i < id.length; i++) {
-        const char = id.charCodeAt(i)
-        hash = ((hash << 5) - hash) + char
-        hash = hash & hash // Convert to 32-bit integer
-      }
-      return Math.abs(hash)
-    }
-    return 0
-  }
-
-  // Remove a record from the index
-  async remove(record) {
-    if (!record) return
-    
-    // If record is an array of line numbers, use the original method
-    if (Array.isArray(record)) {
-      return this._removeLineNumbers(record)
-    }
-    
-    // If record is an object, remove by record data
-    if (typeof record === 'object' && record.id) {
-      return await this._removeRecord(record)
-    }
-  }
-
-  // Remove a specific record from the index
-  async _removeRecord(record) {
-    if (!record) return
-    
-    const data = this.index.data
-    const database = this.database
-    const persistedCount = Array.isArray(database?.offsets) ? database.offsets.length : 0
-    const lineMatchCache = new Map()
-
-    const doesLineNumberBelongToRecord = async (lineNumber) => {
-      if (lineMatchCache.has(lineNumber)) {
-        return lineMatchCache.get(lineNumber)
-      }
-
-      let belongs = false
-
-      try {
-        if (lineNumber >= persistedCount) {
-          const writeBufferIndex = lineNumber - persistedCount
-          const candidate = database?.writeBuffer?.[writeBufferIndex]
-          belongs = !!candidate && candidate.id === record.id
-        } else if (lineNumber >= 0) {
-          const range = database?.locate?.(lineNumber)
-          if (range && database.fileHandler && database.serializer) {
-            const [start, end] = range
-            const buffer = await database.fileHandler.readRange(start, end)
-            if (buffer && buffer.length > 0) {
-              let line = buffer.toString('utf8')
-              if (line) {
-                line = line.trim()
-                if (line.length > 0) {
-                  const storedRecord = database.serializer.deserialize(line)
-                  belongs = storedRecord && storedRecord.id === record.id
-                }
-              }
-            }
-          }
-        }
-      } catch (error) {
-        belongs = false
-      }
-
-      lineMatchCache.set(lineNumber, belongs)
-      return belongs
-    }
-
-    for (const field in data) {
-      if (record[field] !== undefined && record[field] !== null) {
-        const values = Array.isArray(record[field]) ? record[field] : [record[field]]
-        for (const val of values) {
-          let key
-          
-          // Check if this is a term mapping field (array:string fields only)
-          const isTermMappingField = this.database?.termManager && 
-            this.database.termManager.termMappingFields && 
-            this.database.termManager.termMappingFields.includes(field)
-          
-          if (isTermMappingField && typeof val === 'number') {
-            // For term mapping fields (array:string), the values are already term IDs
-            key = String(val)
-            if (this.database.opts.debugMode) {
-              console.log(`🔍 IndexManager._removeRecord: Using term ID ${val} directly for field "${field}"`)
-            }
-          } else if (isTermMappingField && typeof val === 'string') {
-            // For term mapping fields (array:string), convert string to term ID
-            const termId = this.database.termManager.getTermIdWithoutIncrement(val)
-            key = String(termId)
-            if (this.database.opts.debugMode) {
-              console.log(`🔍 IndexManager._removeRecord: Using term ID ${termId} for term "${val}"`)
-            }
-          } else {
-            // For non-term-mapping fields (including array:number), use values directly
-            key = String(val)
-            if (this.database?.opts?.debugMode) {
-              console.log(`🔍 IndexManager._removeRecord: Using value "${val}" directly for field "${field}"`)
-            }
-          }
-          
-          // Note: TermManager notification is handled by Database.mjs
-          // to avoid double decrementation during updates
-          
-          const indexEntry = data[field][key]
-          if (indexEntry) {
-            const lineNumbers = this._getAllLineNumbers(indexEntry)
-            const filteredLineNumbers = []
-
-            for (const lineNumber of lineNumbers) {
-              if (!(await doesLineNumberBelongToRecord(lineNumber))) {
-                filteredLineNumbers.push(lineNumber)
-              }
-            }
-            
-            if (filteredLineNumbers.length === 0) {
-              delete data[field][key]
-            } else {
-              // Rebuild the index value with filtered line numbers
-              data[field][key].set = new Set(filteredLineNumbers)
-              data[field][key].ranges = []
-            }
-          }
-        }
-      }
-    }
-  }
-
-  // Ultra-fast remove with batch processing (renamed from remove)
-  _removeLineNumbers(lineNumbers) {
-    if (!lineNumbers || lineNumbers.length === 0) return
-    
-    lineNumbers.sort((a, b) => a - b) // Sort ascending for efficient processing
-    
-    const data = this.index.data
-    for (const field in data) {
-      for (const value in data[field]) {
-        const numbers = this._getAllLineNumbers(data[field][value])
-        const newNumbers = []
-        
-        for (const ln of numbers) {
-          let offset = 0
-          for (const lineNumber of lineNumbers) {
-            if (lineNumber < ln) {
-              offset++
-            } else if (lineNumber === ln) {
-              offset = -1 // Mark for removal
-              break
-            }
-          }  
-          if (offset >= 0) {
-            newNumbers.push(ln - offset) // Update the value
-          }
-        }  
-        
-        if (newNumbers.length > 0) {
-          // Rebuild hybrid structure with new numbers
-          data[field][value] = { set: new Set(), ranges: [] }
-          for (const num of newNumbers) {
-            data[field][value] = this._addLineNumber(data[field][value], num)
-          }
-        } else {
-          delete data[field][value]
-        }
-      }
-    }
-  } 
-
-  // Ultra-fast replace with batch processing
-  replace(map) {
-    if (!map || map.size === 0) return
-    
-    const data = this.index.data
-    for (const field in data) {
-      for (const value in data[field]) {
-        const numbers = this._getAllLineNumbers(data[field][value])
-        const newNumbers = []
-        
-        for (const lineNumber of numbers) {
-          if (map.has(lineNumber)) {
-            newNumbers.push(map.get(lineNumber))
-          } else {
-            newNumbers.push(lineNumber)
-          }
-        }
-        
-        // Rebuild hybrid structure with new numbers
-        data[field][value] = { set: new Set(), ranges: [] }
-        for (const num of newNumbers) {
-          data[field][value] = this._addLineNumber(data[field][value], num)
-        }
-      }
-    }
-  }
-
-  // Ultra-fast query with early exit and smart processing
-  query(criteria, options = {}) {
-    if (typeof options === 'boolean') {
-      options = { matchAny: options };
-    }
-    const { matchAny = false, caseInsensitive = false } = options;
-    
-    if (!criteria) {
-        // Return all line numbers when no criteria provided
-        return new Set(Array.from({ length: this.totalLines || 0 }, (_, i) => i));
-    }
-    
-    // Handle $not operator
-    if (criteria.$not && typeof criteria.$not === 'object') {
-      // Get all possible line numbers from database offsets or totalLines
-      const totalRecords = this.database?.offsets?.length || this.totalLines || 0;
-      const allLines = new Set(Array.from({ length: totalRecords }, (_, i) => i));
-      
-      // Get line numbers matching the $not condition
-      const notLines = this.query(criteria.$not, options);
-      
-      // Return complement (all lines except those matching $not condition)
-      const result = new Set([...allLines].filter(x => !notLines.has(x)));
-      
-      // If there are other conditions besides $not, we need to intersect with them
-      const otherCriteria = { ...criteria };
-      delete otherCriteria.$not;
-      
-      if (Object.keys(otherCriteria).length > 0) {
-        const otherResults = this.query(otherCriteria, options);
-        return new Set([...result].filter(x => otherResults.has(x)));
-      }
-      
-      return result;
-    }
-    
-    // Handle $and queries with parallel processing optimization
-    if (criteria.$and && Array.isArray(criteria.$and)) {
-      // OPTIMIZATION: Process conditions in parallel for better performance
-      if (criteria.$and.length > 1) {
-        // Process all conditions in parallel (synchronous since query is not async)
-        const conditionResults = criteria.$and.map(andCondition => 
-          this.query(andCondition, options)
-        );
-        
-        // Intersect all results for AND logic
-        let result = conditionResults[0];
-        for (let i = 1; i < conditionResults.length; i++) {
-          result = new Set([...result].filter(x => conditionResults[i].has(x)));
-        }
-        
-        // IMPORTANT: Check if there are other fields besides $and at the root level
-        // If so, we need to intersect with them too
-        const otherCriteria = { ...criteria };
-        delete otherCriteria.$and;
-        
-        if (Object.keys(otherCriteria).length > 0) {
-          const otherResults = this.query(otherCriteria, options);
-          result = new Set([...result].filter(x => otherResults.has(x)));
-        }
-        
-        return result || new Set();
-      } else {
-        // Single condition - check for other criteria at root level
-        const andResult = this.query(criteria.$and[0], options);
-        
-        const otherCriteria = { ...criteria };
-        delete otherCriteria.$and;
-        
-        if (Object.keys(otherCriteria).length > 0) {
-          const otherResults = this.query(otherCriteria, options);
-          return new Set([...andResult].filter(x => otherResults.has(x)));
-        }
-        
-        return andResult;
-      }
-    }
-    
-    const fields = Object.keys(criteria);
-    if (!fields.length) {
-        // Return all line numbers when criteria is empty object
-        return new Set(Array.from({ length: this.totalLines || 0 }, (_, i) => i));
-    }
-    
-    let matchingLines = matchAny ? new Set() : null;
-    const data = this.index.data
-  
-    for (const field of fields) {
-      // Skip logical operators - they are handled separately
-      if (field.startsWith('$')) continue;
-      
-      if (typeof data[field] === 'undefined') continue;
-      
-      const originalCriteriaValue = criteria[field];
-      const criteriaValue = normalizeCriteriaOperators(originalCriteriaValue, { target: 'legacy', preserveOriginal: true });
-      let lineNumbersForField = new Set();
-      const isNumericField = this.opts.indexes[field] === 'number';
-  
-      // Handle RegExp values directly (MUST check before object check since RegExp is an object)
-      if (criteriaValue instanceof RegExp) {
-        // RegExp cannot be efficiently queried using indices - fall back to streaming
-        // This will be handled by the QueryManager's streaming strategy
-        continue;
-      }
-
-      if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue) && criteriaValue !== null) {
-        const fieldIndex = data[field];
-
-        // Handle $in operator for array queries
-        if (criteriaValue.$in !== undefined && criteriaValue.$in !== null) {
-          const inValues = Array.isArray(criteriaValue.$in) ? criteriaValue.$in : [criteriaValue.$in];
-          
-          // PERFORMANCE: Cache term mapping field check once
-          const isTermMappingField = this.database?.termManager && 
-            this.database.termManager.termMappingFields && 
-            this.database.termManager.termMappingFields.includes(field)
-          
-          // PERFORMANCE: Track if any term was found and matched
-          let foundAnyMatch = false
-          
-          for (const inValue of inValues) {
-            // SPACE OPTIMIZATION: Convert search term to term ID for lookup
-            let searchTermId
-            
-            if (isTermMappingField && typeof inValue === 'number') {
-              // For term mapping fields (array:string), the search value is already a term ID
-              searchTermId = String(inValue)
-            } else if (isTermMappingField && typeof inValue === 'string') {
-              // For term mapping fields (array:string), convert string to term ID
-              const termId = this.database?.termManager?.getTermIdWithoutIncrement(String(inValue))
-              if (termId === undefined) {
-                // Term not found in termManager - skip this search value
-                // This means the term was never saved to the database
-                if (this.opts?.debugMode) {
-                  console.log(`⚠️  Term "${inValue}" not found in termManager for field "${field}" - skipping`)
-                }
-                continue // Skip this value, no matches possible
-              }
-              searchTermId = String(termId)
-            } else {
-              // For non-term-mapping fields (including array:number), use values directly
-              searchTermId = String(inValue)
-            }
-            
-            // PERFORMANCE: Direct lookup instead of iteration
-            let matched = false
-            if (caseInsensitive && typeof inValue === 'string') {
-              const searchLower = searchTermId.toLowerCase()
-              for (const value in fieldIndex) {
-                if (value.toLowerCase() === searchLower) {
-                  const numbers = this._getAllLineNumbers(fieldIndex[value]);
-                  for (const lineNumber of numbers) {
-                    lineNumbersForField.add(lineNumber);
-                  }
-                  matched = true
-                  foundAnyMatch = true
-                }
-              }
-            } else {
-              const indexData = fieldIndex[searchTermId]
-              if (indexData) {
-                const numbers = this._getAllLineNumbers(indexData);
-                for (const lineNumber of numbers) {
-                  lineNumbersForField.add(lineNumber);
-                }
-                matched = true
-                foundAnyMatch = true
-              }
-            }
-          }
-          
-          // CRITICAL FIX: If no matches found at all (all terms were unknown or not in index),
-          // lineNumbersForField remains empty which is correct (no results for $in)
-          // This is handled correctly by the caller - empty Set means no matches
-        }
-        // Handle $nin operator (not in) - returns complement of $in
-        else if (criteriaValue.$nin !== undefined) {
-          const ninValues = Array.isArray(criteriaValue.$nin) ? criteriaValue.$nin : [criteriaValue.$nin];
-          
-          // Get all possible line numbers
-          const totalRecords = this.database?.offsets?.length || this.totalLines || 0;
-          const allLines = new Set(Array.from({ length: totalRecords }, (_, i) => i));
-          
-          // Get line numbers that match any of the $nin values
-          const matchingLines = new Set();
-          
-          // PERFORMANCE: Cache term mapping field check once
-          const isTermMappingField = this.database?.termManager && 
-            this.database.termManager.termMappingFields && 
-            this.database.termManager.termMappingFields.includes(field)
-          
-          for (const ninValue of ninValues) {
-            // SPACE OPTIMIZATION: Convert search term to term ID for lookup
-            let searchTermId
-            
-            if (isTermMappingField && typeof ninValue === 'number') {
-              // For term mapping fields (array:string), the search value is already a term ID
-              searchTermId = String(ninValue)
-            } else if (isTermMappingField && typeof ninValue === 'string') {
-              // For term mapping fields (array:string), convert string to term ID
-              const termId = this.database?.termManager?.getTermIdWithoutIncrement(String(ninValue))
-              if (termId === undefined) {
-                // Term not found - skip this value (can't exclude what doesn't exist)
-                if (this.opts?.debugMode) {
-                  console.log(`⚠️  Term "${ninValue}" not found in termManager for field "${field}" - skipping`)
-                }
-                continue
-              }
-              searchTermId = String(termId)
-            } else {
-              // For non-term-mapping fields (including array:number), use values directly
-              searchTermId = String(ninValue)
-            }
-            
-            // PERFORMANCE: Direct lookup instead of iteration
-            if (caseInsensitive && typeof ninValue === 'string') {
-              const searchLower = searchTermId.toLowerCase()
-              for (const value in fieldIndex) {
-                if (value.toLowerCase() === searchLower) {
-                  const numbers = this._getAllLineNumbers(fieldIndex[value]);
-                  for (const lineNumber of numbers) {
-                    matchingLines.add(lineNumber);
-                  }
-                }
-              }
-            } else {
-              const indexData = fieldIndex[searchTermId]
-              if (indexData) {
-                const numbers = this._getAllLineNumbers(indexData);
-                for (const lineNumber of numbers) {
-                  matchingLines.add(lineNumber);
-                }
-              }
-            }
-          }
-          
-          // Return complement: all lines EXCEPT those matching $nin values
-          lineNumbersForField = new Set([...allLines].filter(x => !matchingLines.has(x)));
-        }
-        // Handle $contains operator for array queries
-        else if (criteriaValue.$contains !== undefined) {
-          const containsValue = criteriaValue.$contains;
-          // Handle case-insensitive for $contains
-          if (caseInsensitive && typeof containsValue === 'string') {
-            for (const value in fieldIndex) {
-              if (value.toLowerCase() === containsValue.toLowerCase()) {
-                const numbers = this._getAllLineNumbers(fieldIndex[value]);
-                for (const lineNumber of numbers) {
-                  lineNumbersForField.add(lineNumber);
-                }
-              }
-            }
-          } else {
-            if (fieldIndex[containsValue]) {
-              const numbers = this._getAllLineNumbers(fieldIndex[containsValue]);
-              for (const lineNumber of numbers) {
-                lineNumbersForField.add(lineNumber);
-              }
-            }
-          }
-        }
-        // Handle $all operator for array queries - FIXED FOR TERM MAPPING
-        else if (criteriaValue.$all !== undefined) {
-          const allValues = Array.isArray(criteriaValue.$all) ? criteriaValue.$all : [criteriaValue.$all];
-
-          const isTermMappingField = this.database?.termManager && 
-            this.database.termManager.termMappingFields && 
-            this.database.termManager.termMappingFields.includes(field)
-
-          const normalizeValue = (value) => {
-            if (isTermMappingField) {
-              if (typeof value === 'number') {
-                return String(value)
-              }
-              if (typeof value === 'string') {
-                const termId = this.database?.termManager?.getTermIdWithoutIncrement(value)
-                if (termId !== undefined) {
-                  return String(termId)
-                }
-                return null
-              }
-              return null
-            }
-            return String(value)
-          }
-
-          const normalizedValues = []
-          for (const value of allValues) {
-            const normalized = normalizeValue(value)
-            if (normalized === null) {
-              // Term not found in term manager, no matches possible
-              return lineNumbersForField
-            }
-            normalizedValues.push(normalized)
-          }
-          
-          // Early exit optimization
-          if (normalizedValues.length === 0) {
-            // Empty $all matches everything
-            for (const value in fieldIndex) {
-              const numbers = this._getAllLineNumbers(fieldIndex[value]);
-              for (const lineNumber of numbers) {
-                lineNumbersForField.add(lineNumber);
-              }
-            }
-          } else {
-            // For term mapping, we need to find records that contain ALL specified terms
-            // This requires a different approach than simple field matching
-            
-            // First, get all line numbers that contain each individual term
-            const termLineNumbers = new Map();
-            for (const term of normalizedValues) {
-              if (fieldIndex[term]) {
-                termLineNumbers.set(term, new Set(this._getAllLineNumbers(fieldIndex[term])));
-              } else {
-                // If any term doesn't exist, no records can match $all
-                termLineNumbers.set(term, new Set());
-              }
-            }
-            
-            // Find intersection of all term line numbers
-            if (termLineNumbers.size > 0) {
-              const allTermSets = Array.from(termLineNumbers.values());
-              let intersection = allTermSets[0];
-              
-              for (let i = 1; i < allTermSets.length; i++) {
-                intersection = new Set([...intersection].filter(x => allTermSets[i].has(x)));
-              }
-              
-              // Add all line numbers from intersection
-              for (const lineNumber of intersection) {
-                lineNumbersForField.add(lineNumber);
-              }
-            }
-          }
-        }
-        // Handle other operators
-        else {
-          for (const value in fieldIndex) {
-            let includeValue = true;
-            if (isNumericField) {
-              const numericValue = parseFloat(value);
-              if (!isNaN(numericValue)) {
-                if (criteriaValue['>'] !== undefined && numericValue <= criteriaValue['>']) {
-                  includeValue = false;
-                }
-                if (criteriaValue['>='] !== undefined && numericValue < criteriaValue['>=']) {
-                  includeValue = false;
-                }
-                if (criteriaValue['<'] !== undefined && numericValue >= criteriaValue['<']) {
-                  includeValue = false;
-                }
-                if (criteriaValue['<='] !== undefined && numericValue > criteriaValue['<=']) {
-                  includeValue = false;
-                }
-                if (criteriaValue['!='] !== undefined) {
-                  const excludeValues = Array.isArray(criteriaValue['!='])
-                    ? criteriaValue['!=']
-                    : [criteriaValue['!=']];
-                  if (excludeValues.includes(numericValue)) {
-                    includeValue = false;
-                  }
-                }
-              }
-            } else {
-              if (criteriaValue['contains'] !== undefined && typeof value === 'string') {
-                const term = String(criteriaValue['contains']);
-                if (caseInsensitive) {
-                  if (!value.toLowerCase().includes(term.toLowerCase())) {
-                    includeValue = false;
-                  }
-                } else {
-                  if (!value.includes(term)) {
-                    includeValue = false;
-                  }
-                }
-              }
-              if (criteriaValue['regex'] !== undefined) {
-                let regex;
-                if (typeof criteriaValue['regex'] === 'string') {
-                  regex = new RegExp(criteriaValue['regex'], caseInsensitive ? 'i' : '');
-                } else if (criteriaValue['regex'] instanceof RegExp) {
-                  if (caseInsensitive && !criteriaValue['regex'].ignoreCase) {
-                    const flags = criteriaValue['regex'].flags.includes('i')
-                      ? criteriaValue['regex'].flags
-                      : criteriaValue['regex'].flags + 'i';
-                    regex = new RegExp(criteriaValue['regex'].source, flags);
-                  } else {
-                    regex = criteriaValue['regex'];
-                  }
-                }
-                if (regex) {
-                  // For array fields, test regex against each element
-                  if (Array.isArray(value)) {
-                    if (!value.some(element => regex.test(String(element)))) {
-                      includeValue = false;
-                    }
-                  } else {
-                    // For non-array fields, test regex against the value directly
-                    if (!regex.test(String(value))) {
-                      includeValue = false;
-                    }
-                  }
-                }
-              }
-              if (criteriaValue['!='] !== undefined) {
-                const excludeValues = Array.isArray(criteriaValue['!='])
-                  ? criteriaValue['!=']
-                  : [criteriaValue['!=']];
-                if (excludeValues.includes(value)) {
-                  includeValue = false;
-                }
-              }
-            }
-
-            if (includeValue) {
-              const numbers = this._getAllLineNumbers(fieldIndex[value]);
-              for (const lineNumber of numbers) {
-                lineNumbersForField.add(lineNumber);
-              }
-            }
-          }
-        }
-      } else {
-        // Simple equality comparison - handle array queries
-        const values = Array.isArray(criteriaValue) ? criteriaValue : [criteriaValue];
-        const fieldData = data[field];
-        for (const searchValue of values) {
-          // SPACE OPTIMIZATION: Convert search term to term ID for lookup
-          let searchTermId
-          
-          // PERFORMANCE: Cache term mapping field check once per field
-          const isTermMappingField = this.database?.termManager && 
-            this.database.termManager.termMappingFields && 
-            this.database.termManager.termMappingFields.includes(field)
-          
-          if (isTermMappingField && typeof searchValue === 'number') {
-            // For term mapping fields (array:string), the search value is already a term ID
-            searchTermId = String(searchValue)
-          } else if (isTermMappingField && typeof searchValue === 'string') {
-            // For term mapping fields (array:string), convert string to term ID
-            const termId = this.database?.termManager?.getTermIdWithoutIncrement(String(searchValue))
-            if (termId === undefined) {
-              // Term not found - skip this value
-              if (this.opts?.debugMode) {
-                console.log(`⚠️  Term "${searchValue}" not found in termManager for field "${field}" - skipping`)
-              }
-              continue // Skip this value, no matches possible
-            }
-            searchTermId = String(termId)
-          } else {
-            // For non-term-mapping fields (including array:number), use values directly
-            searchTermId = String(searchValue)
-          }
-          
-          for (const key in fieldData) {
-            let match = false;
-            if (isNumericField) {
-              // Convert both parts to number
-              match = Number(key) === Number(searchValue);
-            } else {
-              // SPACE OPTIMIZATION: Compare term IDs instead of full terms
-              if (caseInsensitive) {
-                // For case-insensitive, we need to check if the search term ID matches any key
-                match = key === String(searchTermId);
-              } else {
-                match = key === String(searchTermId);
-              }
-            }
-            if (match) {
-              const numbers = this._getAllLineNumbers(fieldData[key]);
-              for (const lineNumber of numbers) {
-                lineNumbersForField.add(lineNumber);
-              }
-            }
-          }
-        }
-      }
-  
-      // Consolidate results from each field
-      if (matchAny) {
-        matchingLines = new Set([...matchingLines, ...lineNumbersForField]);
-      } else {
-        if (matchingLines === null) {
-          matchingLines = lineNumbersForField;
-        } else {
-          matchingLines = new Set([...matchingLines].filter(n => lineNumbersForField.has(n)));
-        }
-        if (!matchingLines.size) {
-          return new Set();
-        }
-      }
-    }
-    return matchingLines || new Set();
-  } 
-
-  /**
-   * Check if any records exist for given field and terms (index-only, ultra-fast)
-   * Stops at first match for maximum performance - no disk I/O required
-   * 
-   * @param {string} fieldName - Indexed field name (e.g., 'nameTerms', 'groupTerms')
-   * @param {string|Array<string>} terms - Single term or array of terms to check
-   * @param {Object} options - Options: { $all: true/false, caseInsensitive: true/false, excludes: Array<string> }
-   * @returns {boolean} - True if at least one match exists
-   * 
-   * @example
-   * // Check if any record has 'channel' in nameTerms
-   * indexManager.exists('nameTerms', 'channel')
-   * 
-   * @example
-   * // Check if any record has ALL terms ['a', 'e'] in nameTerms ($all)
-   * indexManager.exists('nameTerms', ['a', 'e'], { $all: true })
-   * 
-   * @example
-   * // Check if any record has ANY of the terms ['channel', 'tv'] in nameTerms
-   * indexManager.exists('nameTerms', ['channel', 'tv'], { $all: false })
-   * 
-   * @example
-   * // Check if any record has 'tv' but NOT 'globo' in nameTerms
-   * indexManager.exists('nameTerms', 'tv', { excludes: ['globo'] })
-   * 
-   * @example
-   * // Check if any record has ['tv', 'news'] but NOT 'sports' in nameTerms
-   * indexManager.exists('nameTerms', ['tv', 'news'], { $all: true, excludes: ['sports'] })
-   */
-  exists(fieldName, terms, options = {}) {
-    // Early exit: validate fieldName
-    if (!fieldName || typeof fieldName !== 'string') {
-      return false;
-    }
-    
-    // Early exit: check if field is indexed
-    if (!this.isFieldIndexed(fieldName)) {
-      return false;
-    }
-    
-    const fieldIndex = this.index.data[fieldName];
-    if (!fieldIndex || typeof fieldIndex !== 'object') {
-      return false;
-    }
-    
-    // Normalize terms to array
-    const termsArray = Array.isArray(terms) ? terms : [terms];
-    if (termsArray.length === 0) {
-      return false;
-    }
-    
-    const { $all = false, caseInsensitive = false, excludes = [] } = options;
-    const hasExcludes = Array.isArray(excludes) && excludes.length > 0;
-    const isTermMappingField = this.database?.termManager && 
-      this.database.termManager.termMappingFields && 
-      this.database.termManager.termMappingFields.includes(fieldName);
-    
-    // Helper: check if termData has any line numbers (ULTRA LIGHT - no expansion)
-    const hasData = (termData) => {
-      if (!termData) return false;
-      // Check Set size (O(1))
-      if (termData.set && termData.set.size > 0) {
-        return true;
-      }
-      // Check ranges length (O(1))
-      if (termData.ranges && termData.ranges.length > 0) {
-        return true;
-      }
-      return false;
-    };
-    
-    // Helper: get term key with term mapping and case-insensitive support
-    const getTermKey = (term, useCaseInsensitive = false) => {
-      if (isTermMappingField && typeof term === 'string') {
-        let termId;
-        if (useCaseInsensitive) {
-          // For case-insensitive, search termManager for case-insensitive match
-          const searchLower = String(term).toLowerCase();
-          termId = null;
-          if (this.database?.termManager?.termToId) {
-            for (const [termStr, id] of this.database.termManager.termToId.entries()) {
-              if (termStr.toLowerCase() === searchLower) {
-                termId = id;
-                break;
-              }
-            }
-          }
-        } else {
-          termId = this.database?.termManager?.getTermIdWithoutIncrement(String(term));
-        }
-        
-        if (termId === undefined || termId === null) {
-          return null;
-        }
-        return String(termId);
-      }
-      
-      // For non-term-mapping fields
-      if (useCaseInsensitive && typeof term === 'string') {
-        const searchLower = String(term).toLowerCase();
-        for (const key in fieldIndex) {
-          if (key.toLowerCase() === searchLower) {
-            return key;
-          }
-        }
-        return null;
-      }
-      
-      return String(term);
-    };
-    
-    // Handle $all (all terms must exist and have intersection)
-    if ($all) {
-      // Collect term data for all terms first (with early exit)
-      const termDataArray = [];
-      
-      for (const term of termsArray) {
-        // Get term key (with term mapping if applicable)
-        let termKey;
-        if (isTermMappingField && typeof term === 'string') {
-          let termId;
-          if (caseInsensitive) {
-            // For case-insensitive, search termManager for case-insensitive match
-            const searchLower = String(term).toLowerCase();
-            termId = null;
-            for (const [termStr, id] of this.database.termManager.termToId.entries()) {
-              if (termStr.toLowerCase() === searchLower) {
-                termId = id;
-                break;
-              }
-            }
-          } else {
-            termId = this.database?.termManager?.getTermIdWithoutIncrement(String(term));
-          }
-          
-          if (termId === undefined || termId === null) {
-            return false; // Early exit: term doesn't exist in mapping
-          }
-          termKey = String(termId);
-        } else {
-          termKey = String(term);
-          // For non-term-mapping fields with case-insensitive, search index keys
-          if (caseInsensitive && typeof term === 'string') {
-            const searchLower = termKey.toLowerCase();
-            let foundKey = null;
-            for (const key in fieldIndex) {
-              if (key.toLowerCase() === searchLower) {
-                foundKey = key;
-                break;
-              }
-            }
-            if (foundKey === null) {
-              return false; // Early exit: term doesn't exist
-            }
-            termKey = foundKey;
-          }
-        }
-        
-        // Check if term exists in index
-        const termData = fieldIndex[termKey];
-        if (!termData || !hasData(termData)) {
-          return false; // Early exit: term doesn't exist or has no data
-        }
-        
-        termDataArray.push(termData);
-      }
-      
-      // If we got here, all terms exist and have data
-      // Now check if there's intersection (only if more than one term)
-      if (termDataArray.length === 1) {
-        // Single term - check excludes if any
-        if (!hasExcludes) {
-          return true; // Single term, already verified it has data, no excludes
-        }
-        // Need to check excludes - expand line numbers
-        const lineNumbers = this._getAllLineNumbers(termDataArray[0]);
-        const candidateLines = new Set(lineNumbers);
-        
-        // Remove lines that have exclude terms
-        for (const excludeTerm of excludes) {
-          const excludeKey = getTermKey(excludeTerm, caseInsensitive);
-          if (excludeKey === null) continue;
-          
-          const excludeData = fieldIndex[excludeKey];
-          if (!excludeData) continue;
-          
-          const excludeLines = this._getAllLineNumbers(excludeData);
-          for (const line of excludeLines) {
-            candidateLines.delete(line);
-          }
-          
-          // Early exit if all candidates excluded
-          if (candidateLines.size === 0) {
-            return false;
-          }
-        }
-        
-        return candidateLines.size > 0;
-      }
-      
-      // For multiple terms, we need to check intersection
-      // But we want to do this as lightly as possible
-      // Get line numbers only for intersection check (unavoidable for $all)
-      const termLineNumberSets = [];
-      for (const termData of termDataArray) {
-        const lineNumbers = this._getAllLineNumbers(termData);
-        if (lineNumbers.length === 0) {
-          return false; // Early exit: no line numbers (shouldn't happen, but safety check)
-        }
-        termLineNumberSets.push(new Set(lineNumbers));
-      }
-      
-      // Calculate intersection incrementally with early exit
-      let intersection = termLineNumberSets[0];
-      for (let i = 1; i < termLineNumberSets.length; i++) {
-        // Filter intersection to only include items in current set
-        intersection = new Set([...intersection].filter(x => termLineNumberSets[i].has(x)));
-        if (intersection.size === 0) {
-          return false; // Early exit: intersection is empty
-        }
-      }
-      
-      // Apply excludes if any
-      if (hasExcludes) {
-        for (const excludeTerm of excludes) {
-          const excludeKey = getTermKey(excludeTerm, caseInsensitive);
-          if (excludeKey === null) continue;
-          
-          const excludeData = fieldIndex[excludeKey];
-          if (!excludeData) continue;
-          
-          const excludeLines = this._getAllLineNumbers(excludeData);
-          for (const line of excludeLines) {
-            intersection.delete(line);
-          }
-          
-          // Early exit if all candidates excluded
-          if (intersection.size === 0) {
-            return false;
-          }
-        }
-      }
-      
-      return intersection.size > 0;
-    }
-    
-    // Handle $in behavior (any term exists) - default - ULTRA LIGHT
-    // If no excludes, use ultra-fast path (no expansion needed)
-    if (!hasExcludes) {
-      for (const term of termsArray) {
-        // Handle case-insensitive FIRST (before normal conversion)
-        if (caseInsensitive && typeof term === 'string') {
-          if (isTermMappingField && this.database?.termManager?.termToId) {
-            // For term mapping fields, we need to find the term in termManager first
-            // (case-insensitive), then convert to ID
-            const searchLower = String(term).toLowerCase();
-            let foundTermId = null;
-            
-            // Search termManager for case-insensitive match
-            for (const [termStr, termId] of this.database.termManager.termToId.entries()) {
-              if (termStr.toLowerCase() === searchLower) {
-                foundTermId = termId;
-                break;
-              }
-            }
-            
-            if (foundTermId !== null) {
-              const termData = fieldIndex[String(foundTermId)];
-              if (hasData(termData)) {
-                return true; // Early exit: found a match
-              }
-            }
-            // If not found, continue to next term
-            continue;
-          } else {
-            // For non-term-mapping fields, search index keys directly
-            const searchLower = String(term).toLowerCase();
-            for (const key in fieldIndex) {
-              if (key.toLowerCase() === searchLower) {
-                const termData = fieldIndex[key];
-                if (hasData(termData)) {
-                  return true; // Early exit: found a match
-                }
-              }
-            }
-            // If not found, continue to next term
-            continue;
-          }
-        }
-        
-        // Normal (case-sensitive) lookup
-        const termKey = getTermKey(term, false);
-        if (termKey === null) {
-          continue; // Term not in mapping, try next
-        }
-        
-        // Direct lookup (fastest path) - O(1) hash lookup
-        const termData = fieldIndex[termKey];
-        if (hasData(termData)) {
-          return true; // Early exit: found a match
-        }
-      }
-      
-      return false;
-    }
-    
-    // With excludes, we need to collect candidates and filter
-    const candidateLines = new Set();
-    
-    for (const term of termsArray) {
-      // Handle case-insensitive FIRST (before normal conversion)
-      if (caseInsensitive && typeof term === 'string') {
-        if (isTermMappingField && this.database?.termManager?.termToId) {
-          // For term mapping fields, we need to find the term in termManager first
-          // (case-insensitive), then convert to ID
-          const searchLower = String(term).toLowerCase();
-          let foundTermId = null;
-          
-          // Search termManager for case-insensitive match
-          for (const [termStr, termId] of this.database.termManager.termToId.entries()) {
-            if (termStr.toLowerCase() === searchLower) {
-              foundTermId = termId;
-              break;
-            }
-          }
-          
-          if (foundTermId !== null) {
-            const termData = fieldIndex[String(foundTermId)];
-            if (hasData(termData)) {
-              // Add line numbers to candidates (need to expand for excludes check)
-              const lineNumbers = this._getAllLineNumbers(termData);
-              for (const line of lineNumbers) {
-                candidateLines.add(line);
-              }
-            }
-          }
-          continue;
-        } else {
-          // For non-term-mapping fields, search index keys directly
-          const searchLower = String(term).toLowerCase();
-          for (const key in fieldIndex) {
-            if (key.toLowerCase() === searchLower) {
-              const termData = fieldIndex[key];
-              if (hasData(termData)) {
-                // Add line numbers to candidates
-                const lineNumbers = this._getAllLineNumbers(termData);
-                for (const line of lineNumbers) {
-                  candidateLines.add(line);
-                }
-              }
-            }
-          }
-          continue;
-        }
-      }
-      
-      // Normal (case-sensitive) lookup
-      const termKey = getTermKey(term, false);
-      if (termKey === null) {
-        continue; // Term not in mapping, try next
-      }
-      
-      // Direct lookup
-      const termData = fieldIndex[termKey];
-      if (hasData(termData)) {
-        // Add line numbers to candidates (need to expand for excludes check)
-        const lineNumbers = this._getAllLineNumbers(termData);
-        for (const line of lineNumbers) {
-          candidateLines.add(line);
-        }
-      }
-    }
-    
-    // If no candidates found, return false
-    if (candidateLines.size === 0) {
-      return false;
-    }
-    
-    // Apply excludes
-    for (const excludeTerm of excludes) {
-      const excludeKey = getTermKey(excludeTerm, caseInsensitive);
-      if (excludeKey === null) continue;
-      
-      const excludeData = fieldIndex[excludeKey];
-      if (!excludeData) continue;
-      
-      const excludeLines = this._getAllLineNumbers(excludeData);
-      for (const line of excludeLines) {
-        candidateLines.delete(line);
-      }
-      
-      // Early exit if all candidates excluded
-      if (candidateLines.size === 0) {
-        return false;
-      }
-    }
-    
-    return candidateLines.size > 0;
-  }
-
-  /**
-   * Evaluate many index-only existence checks while sharing the same field data.
-   * Returns a map keyed by the criteria id so callers can reuse a single scan per field.
-   *
-   * @param {string} fieldName - Indexed field name
-   * @param {Array<Object>} criteriaArray - Array of { id, terms, options }
-   * @param {Object} opts - { limit, allowPartial } (limit only applies when allowPartial is true)
-   * @returns {Object<string, boolean>} - Map of criteria id to boolean existence
-   */
-  multiExists(fieldName, criteriaArray, opts = {}) {
-    const results = {}
-    if (!Array.isArray(criteriaArray) || criteriaArray.length === 0) {
-      return results
-    }
-
-    const prepared = []
-    for (let i = 0; i < criteriaArray.length; i++) {
-      const entry = criteriaArray[i] || {}
-      const key = entry.id !== undefined && entry.id !== null ? String(entry.id) : `criteria-${i}`
-      results[key] = false
-      prepared.push({
-        id: key,
-        terms: entry.terms,
-        options: entry.options || {}
-      })
-    }
-
-    if (!fieldName || typeof fieldName !== 'string') {
-      return results
-    }
-
-    const fieldIndex = this.index?.data?.[fieldName]
-    if (!fieldIndex || typeof fieldIndex !== 'object') {
-      return results
-    }
-
-    const termManager = this.database?.termManager
-    const isTermMappingField = Boolean(termManager && termManager.termMappingFields && termManager.termMappingFields.includes(fieldName))
-
-    const normalizedLimit = Number.isFinite(opts.limit) && opts.limit > 0 ? Math.floor(opts.limit) : Infinity
-    const allowPartial = Boolean(opts.allowPartial)
-    const positiveTarget = allowPartial ? Math.max(1, normalizedLimit) : Infinity
-
-    const termCache = new Map()
-    let caseInsensitiveLookup = null
-    let termMappingLookup = null
-
-    const normalizeValues = (value) => {
-      if (value === null || value === undefined) {
-        return []
-      }
-      if (Array.isArray(value)) {
-        return value.slice()
-      }
-      return [value]
-    }
-
-    const buildCaseInsensitiveLookup = () => {
-      const map = new Map()
-      for (const key in fieldIndex) {
-        const lowerKey = key.toLowerCase()
-        if (!map.has(lowerKey)) {
-          map.set(lowerKey, key)
-        }
-      }
-      return map
-    }
-
-    const buildTermMappingLookup = () => {
-      const map = new Map()
-      const termToId = termManager?.termToId
-      if (termToId) {
-        for (const [termStr, id] of termToId.entries()) {
-          const lower = termStr.toLowerCase()
-          if (!map.has(lower)) {
-            map.set(lower, String(id))
-          }
-        }
-      }
-      return map
-    }
-
-    const resolveTermKey = (value, caseInsensitive) => {
-      if (value === null || value === undefined) {
-        return null
-      }
-
-      if (isTermMappingField) {
-        if (typeof value === 'string') {
-          if (caseInsensitive) {
-            if (!termMappingLookup) {
-              termMappingLookup = buildTermMappingLookup()
-            }
-            return termMappingLookup.get(value.toLowerCase()) ?? null
-          }
-          const termId = termManager?.getTermIdWithoutIncrement(String(value))
-          if (termId === undefined || termId === null) {
-            return null
-          }
-          return String(termId)
-        }
-        return String(value)
-      }
-
-      if (caseInsensitive && typeof value === 'string') {
-        if (!caseInsensitiveLookup) {
-          caseInsensitiveLookup = buildCaseInsensitiveLookup()
-        }
-        return caseInsensitiveLookup.get(value.toLowerCase()) ?? null
-      }
-
-      return String(value)
-    }
-
-    const ensureTermEntry = (termKey) => {
-      if (!termCache.has(termKey)) {
-        const data = fieldIndex[termKey]
-        const entry = {
-          data,
-          hasData: Boolean(data && ((data.set && data.set.size > 0) || (data.ranges && data.ranges.length > 0))),
-          lineArray: null
-        }
-        termCache.set(termKey, entry)
-      }
-      return termCache.get(termKey)
-    }
-
-    const getLineArray = (entry) => {
-      if (!entry.hasData || !entry.data) return []
-      if (!entry.lineArray) {
-        entry.lineArray = this._getAllLineNumbers(entry.data)
-      }
-      return entry.lineArray
-    }
-
-    const applyExcludes = (lineSet, excludeKeys) => {
-      if (excludeKeys.length === 0) {
-        return lineSet.size > 0
-      }
-      for (const excludeKey of excludeKeys) {
-        const excludeEntry = ensureTermEntry(excludeKey)
-        if (!excludeEntry.hasData) continue
-        const excludeLines = getLineArray(excludeEntry)
-        for (const line of excludeLines) {
-          lineSet.delete(line)
-        }
-        if (lineSet.size === 0) {
-          return false
-        }
-      }
-      return lineSet.size > 0
-    }
-
-    let successes = 0
-
-    for (const criterion of prepared) {
-      if (successes >= positiveTarget) {
-        break
-      }
-
-      const { id, terms, options } = criterion
-      const { $all = false, caseInsensitive = false, excludes = [] } = options
-
-      const normalizedTerms = normalizeValues(terms)
-      if (normalizedTerms.length === 0) {
-        continue
-      }
-
-      const normalizedExcludes = normalizeValues(excludes)
-
-      const resolvedKeys = []
-      const seenKeys = new Set()
-      let missingRequiredTerm = false
-
-      for (const term of normalizedTerms) {
-        const termKey = resolveTermKey(term, caseInsensitive)
-        if (termKey === null) {
-          if ($all) {
-            missingRequiredTerm = true
-            break
-          }
-          continue
-        }
-        if (!seenKeys.has(termKey)) {
-          seenKeys.add(termKey)
-          resolvedKeys.push(termKey)
-        }
-      }
-
-      if ($all && missingRequiredTerm) {
-        continue
-      }
-
-      if (resolvedKeys.length === 0) {
-        continue
-      }
-
-      const excludeKeySet = []
-      const seenExcludeKeys = new Set()
-      for (const excludeTerm of normalizedExcludes) {
-        const excludeKey = resolveTermKey(excludeTerm, caseInsensitive)
-        if (excludeKey && !seenExcludeKeys.has(excludeKey)) {
-          seenExcludeKeys.add(excludeKey)
-          excludeKeySet.push(excludeKey)
-        }
-      }
-
-      let match = false
-
-      if ($all) {
-        let intersection = null
-        let failed = false
-
-        for (const termKey of resolvedKeys) {
-          const entry = ensureTermEntry(termKey)
-          if (!entry.hasData) {
-            failed = true
-            break
-          }
-          const lines = getLineArray(entry)
-          if (lines.length === 0) {
-            failed = true
-            break
-          }
-
-          const currentSet = new Set(lines)
-          if (intersection === null) {
-            intersection = currentSet
-          } else {
-            intersection = new Set([...intersection].filter(value => currentSet.has(value)))
-            if (intersection.size === 0) {
-              failed = true
-              break
-            }
-          }
-        }
-
-        if (!failed && intersection && intersection.size > 0) {
-          match = applyExcludes(intersection, excludeKeySet)
-        }
-      } else if (excludeKeySet.length === 0) {
-        match = resolvedKeys.some(termKey => ensureTermEntry(termKey).hasData)
-      } else {
-        const candidates = new Set()
-        for (const termKey of resolvedKeys) {
-          const entry = ensureTermEntry(termKey)
-          if (!entry.hasData) continue
-          const lines = getLineArray(entry)
-          for (const line of lines) {
-            candidates.add(line)
-          }
-        }
-        if (candidates.size > 0) {
-          match = applyExcludes(candidates, excludeKeySet)
-        }
-      }
-
-      results[id] = Boolean(match)
-      if (match) {
-        successes += 1
-      }
-    }
-
-    return results
-  }
- 
-  // Ultra-fast load with minimal conversions
-  load(index) {
-    // CRITICAL FIX: Check if index is already loaded by looking for actual data, not just empty field structures
-    if (this.index && this.index.data) {
-      let hasActualData = false
-      for (const field in this.index.data) {
-        const fieldData = this.index.data[field]
-        if (fieldData && Object.keys(fieldData).length > 0) {
-          // Check if any field has actual index entries with data
-          for (const key in fieldData) {
-            const entry = fieldData[key]
-            if (entry && ((entry.set && entry.set.size > 0) || (entry.ranges && entry.ranges.length > 0))) {
-              hasActualData = true
-              break
-            }
-          }
-          if (hasActualData) break
-        }
-      }
-      
-      if (hasActualData) {
-        if (this.opts.debugMode) {
-          console.log('🔍 IndexManager.load: Index already loaded with actual data, skipping')
-        }
-        return
-      }
-    }
-    
-    // CRITICAL FIX: Add comprehensive null/undefined validation
-    if (!index || typeof index !== 'object') {
-      if (this.opts.debugMode) {
-        console.log(`🔍 IndexManager.load: Invalid index data provided (${typeof index}), using defaults`)
-      }
-      return this._initializeDefaults()
-    }
-
-    if (!index.data || typeof index.data !== 'object') {
-      if (this.opts.debugMode) {
-        console.log(`🔍 IndexManager.load: Invalid index.data provided (${typeof index.data}), using defaults`)
-      }
-      return this._initializeDefaults()
-    }
-    
-    // CRITICAL FIX: Only log if there are actual fields to load
-    if (this.opts.debugMode && Object.keys(index.data).length > 0) {
-      console.log(`🔍 IndexManager.load: Loading index with fields: ${Object.keys(index.data).join(', ')}`)
-    }
-    
-    // Create a deep copy to avoid reference issues
-    const processedIndex = {
-      data: {}
-    }
-    
-    // CRITICAL FIX: Add null/undefined checks for field iteration
-    const fields = Object.keys(index.data)
-    for(const field of fields) {
-      if (!field || typeof field !== 'string') {
-        continue // Skip invalid field names
-      }
-
-      const fieldData = index.data[field]
-      if (!fieldData || typeof fieldData !== 'object') {
-        continue // Skip invalid field data
-      }
-
-      processedIndex.data[field] = {}
-      
-      // CRITICAL FIX: Check if this is a term mapping field for conversion
-      const isTermMappingField = this.database?.termManager && 
-        this.database.termManager.termMappingFields && 
-        this.database.termManager.termMappingFields.includes(field)
-      
-      const terms = Object.keys(fieldData)
-      for(const term of terms) {
-        if (!term || typeof term !== 'string') {
-          continue // Skip invalid term names
-        }
-
-        const termData = fieldData[term]
-        
-        // CRITICAL FIX: Convert term strings to term IDs for term mapping fields
-        // If the key is a string term (not a numeric ID), convert it to term ID
-        let termKey = term
-        if (isTermMappingField && typeof term === 'string' && !/^\d+$/.test(term)) {
-          // Key is a term string, convert to term ID
-          const termId = this.database?.termManager?.getTermIdWithoutIncrement(term)
-          if (termId !== undefined) {
-            termKey = String(termId)
-          } else {
-            // Term not found in termManager - skip this key (orphaned term from old index)
-            // This can happen if termMapping wasn't loaded yet or term was removed
-            if (this.opts?.debugMode) {
-              console.log(`⚠️  IndexManager.load: Term "${term}" not found in termManager for field "${field}" - skipping (orphaned from old index)`)
-            }
-            continue
-          }
-        }
-        
-        // Convert various formats to new hybrid format
-        if (Array.isArray(termData)) {
-              // Check if it's the new compact format [setArray, rangesArray]
-              if (termData.length === 2 && Array.isArray(termData[0]) && Array.isArray(termData[1])) {
-                // New compact format: [setArray, rangesArray]
-                // Convert ultra-compact ranges [start, count] back to {start, count}
-                const ranges = termData[1].map(range => {
-                  if (Array.isArray(range) && range.length === 2) {
-                    // Ultra-compact format: [start, count]
-                    return { start: range[0], count: range[1] }
-                  } else {
-                    // Legacy format: {start, count}
-                    return range
-                  }
-                })
-                processedIndex.data[field][termKey] = {
-                  set: new Set(termData[0]),
-                  ranges: ranges
-                }
-              } else {
-                // Legacy array format (just set data)
-                processedIndex.data[field][termKey] = { set: new Set(termData), ranges: [] }
-              }
-        } else if (termData && typeof termData === 'object') {
-          if (termData.set || termData.ranges) {
-            // Legacy hybrid format - convert set array back to Set
-            const hybridData = termData
-            let setObject
-            if (Array.isArray(hybridData.set)) {
-              // Convert array back to Set
-              setObject = new Set(hybridData.set)
-            } else {
-              // Fallback to empty Set
-              setObject = new Set()
-            }
-            processedIndex.data[field][termKey] = { 
-              set: setObject, 
-              ranges: hybridData.ranges || [] 
-            }
-          } else {
-            // Convert from Set format to hybrid
-            const numbers = Array.from(termData || [])
-            processedIndex.data[field][termKey] = { set: new Set(numbers), ranges: [] }
-          }
-        }
-      }
-    }
-    
-    // Preserve initialized fields if no data was loaded
-    if (!processedIndex.data || Object.keys(processedIndex.data).length === 0) {
-      // CRITICAL FIX: Only log if debug mode is enabled and there are actual fields
-      if (this.opts.debugMode && this.index.data && Object.keys(this.index.data).length > 0) {
-        console.log(`🔍 IndexManager.load: No data loaded, preserving initialized fields: ${Object.keys(this.index.data).join(', ')}`)
-      }
-      // Keep the current index with initialized fields
-      return
-    }
-    
-    // Restore totalLines from saved data
-    if (index.totalLines !== undefined) {
-      this.totalLines = index.totalLines
-      if (this.opts.debugMode) {
-        console.log(`🔍 IndexManager.load: Restored totalLines=${this.totalLines}`)
-      }
-    }
-
-    this.index = processedIndex
-  }
-
-  /**
-   * CRITICAL FIX: Initialize default index structure when invalid data is provided
-   * This prevents TypeError when Object.keys() is called on null/undefined
-   */
-  _initializeDefaults() {
-    if (this.opts.debugMode) {
-      console.log(`🔍 IndexManager._initializeDefaults: Initializing default index structure`)
-    }
-    
-    // Initialize empty index structure
-    this.index = { data: {} }
-    
-    // Initialize fields from options if available
-    if (this.opts.indexes && typeof this.opts.indexes === 'object') {
-      const fields = Object.keys(this.opts.indexes)
-      for (const field of fields) {
-        if (field && typeof field === 'string') {
-          this.index.data[field] = {}
-        }
-      }
-    }
-    
-    if (this.opts.debugMode) {
-      console.log(`🔍 IndexManager._initializeDefaults: Initialized with fields: ${Object.keys(this.index.data).join(', ')}`)
-    }
-  }
-  
-  readColumnIndex(column) {
-    return new Set((this.index.data && this.index.data[column]) ? Object.keys(this.index.data[column]) : [])
-  }
-
-  /**
-   * Convert index to JSON-serializable format for debugging and export
-   * This resolves the issue where Sets appear as empty objects in JSON.stringify
-   */
-  toJSON() {
-    const serializable = {
-      data: {},
-      totalLines: this.totalLines
-    }
-    
-    // Check if this is a term mapping field for conversion
-    const isTermMappingField = (field) => {
-      return this.database?.termManager && 
-        this.database.termManager.termMappingFields && 
-        this.database.termManager.termMappingFields.includes(field)
-    }
-    
-    for (const field in this.index.data) {
-      serializable.data[field] = {}
-      const isTermField = isTermMappingField(field)
-      
-      for (const term in this.index.data[field]) {
-        const hybridData = this.index.data[field][term]
-        
-        // CRITICAL FIX: Convert term strings to term IDs for term mapping fields
-        // If the key is a string term (not a numeric ID), convert it to term ID
-        let termKey = term
-        if (isTermField && typeof term === 'string' && !/^\d+$/.test(term)) {
-          // Key is a term string, convert to term ID
-          const termId = this.database?.termManager?.getTermIdWithoutIncrement(term)
-          if (termId !== undefined) {
-            termKey = String(termId)
-          } else {
-            // Term not found in termManager, keep original key
-            // This prevents data loss when term mapping is incomplete
-            termKey = term
-            if (this.opts?.debugMode) {
-              console.log(`⚠️  IndexManager.toJSON: Term "${term}" not found in termManager for field "${field}" - using original key`)
-            }
-          }
-        }
-        
-        // OPTIMIZATION: Create ranges before serialization if beneficial
-        if (hybridData.set && hybridData.set.size >= this.rangeThreshold) {
-          this._optimizeToRanges(hybridData)
-        }
-        
-        // Convert hybrid structure to serializable format
-        let setArray = []
-        if (hybridData.set) {
-          if (typeof hybridData.set.size !== 'undefined') {
-            // Regular Set
-            setArray = Array.from(hybridData.set)
-          }
-        }
-        
-        // Use ultra-compact format: [setArray, rangesArray] to save space
-        const ranges = hybridData.ranges || []
-        if (ranges.length > 0) {
-          // Convert ranges to ultra-compact format: [start, count] instead of {start, count}
-          const compactRanges = ranges.map(range => [range.start, range.count])
-          serializable.data[field][termKey] = [setArray, compactRanges]
-        } else {
-          // CRITICAL FIX: Always use the [setArray, []] format for consistency
-          // This ensures the load() method can properly deserialize the data
-          serializable.data[field][termKey] = [setArray, []]
-        }
-      }
-    }
-    
-    return serializable
-  }
-
-  /**
-   * Get a JSON string representation of the index
-   * This properly handles Sets unlike the default JSON.stringify
-   */
-  toString() {
-    return JSON.stringify(this.toJSON(), null, 2)
-  }
-
-  // Simplified term mapping methods - just basic functionality
-  
-  /**
-   * Rebuild index (stub for compatibility)
-   */
-  async rebuild() {
-    // Stub implementation for compatibility
-    return Promise.resolve()
-  }
-}
-
+import { Mutex } from 'async-mutex'
+import { normalizeCriteriaOperators } from '../utils/operatorNormalizer.mjs'
+
+export default class IndexManager {
+  constructor(opts, databaseMutex = null, database = null) {
+    this.opts = Object.assign({}, opts)
+    this.index = Object.assign({data: {}}, this.opts.index)
+    this.totalLines = 0
+    this.rangeThreshold = 10 // Sensible threshold: 10+ consecutive numbers justify ranges
+    this.binarySearchThreshold = 32 // Much higher for better performance
+    this.database = database // Reference to database for term manager access
+    this.indexLoaded = false
+    this.indexIdleUnloadMs = typeof this.opts.indexIdleUnloadMs === 'number' ? this.opts.indexIdleUnloadMs : 30000
+    this._idleUnloadTimer = null
+    this._indexLoadPromise = null
+    
+    // CRITICAL: Use database mutex to prevent deadlocks
+    // If no database mutex provided, create a local one (for backward compatibility)
+    this.mutex = databaseMutex || new Mutex()
+    
+    this.indexedFields = []
+    this.setIndexesConfig(this.opts.indexes)
+  }
+
+  setTotalLines(total) {
+    this.totalLines = total
+  }
+
+  /**
+   * Update indexes configuration and ensure internal structures stay in sync
+   * @param {Object|Array<string>} indexes
+   */
+  setIndexesConfig(indexes) {
+    if (!indexes) {
+      this.opts.indexes = undefined
+      this.indexedFields = []
+      return
+    }
+
+    if (Array.isArray(indexes)) {
+      const fields = indexes.map(field => String(field))
+      this.indexedFields = fields
+
+      const normalizedConfig = {}
+      for (const field of fields) {
+        const existingConfig = (!Array.isArray(this.opts.indexes) && typeof this.opts.indexes === 'object') ? this.opts.indexes[field] : undefined
+        normalizedConfig[field] = existingConfig ?? 'auto'
+        if (!this.index.data[field]) {
+          this.index.data[field] = {}
+        }
+      }
+      this.opts.indexes = normalizedConfig
+      return
+    }
+
+    if (typeof indexes === 'object') {
+      this.opts.indexes = Object.assign({}, indexes)
+      this.indexedFields = Object.keys(this.opts.indexes)
+
+      for (const field of this.indexedFields) {
+        if (!this.index.data[field]) {
+          this.index.data[field] = {}
+        }
+      }
+    }
+  }
+
+  _clearIdleUnloadTimer() {
+    if (this._idleUnloadTimer) {
+      clearTimeout(this._idleUnloadTimer)
+      this._idleUnloadTimer = null
+    }
+  }
+
+  cancelIdleUnloadTimer() {
+    this._clearIdleUnloadTimer()
+  }
+
+  markIndexUsed() {
+    if (!this.indexLoaded) return
+
+    this._clearIdleUnloadTimer()
+    if (this.indexIdleUnloadMs > 0) {
+      const timer = setTimeout(() => {
+        try {
+          this.unload()
+        } catch (error) {
+          if (this.opts.debugMode) {
+            console.error('⚠️ IndexManager.markIndexUsed: Failed to unload index', error)
+          }
+        }
+      }, this.indexIdleUnloadMs)
+      if (timer && typeof timer.unref === 'function') {
+        timer.unref()
+      }
+      this._idleUnloadTimer = timer
+    }
+  }
+
+  unload() {
+    if (!this.indexLoaded) return
+    if (this.opts.debugMode) {
+      console.log(`🧹 IndexManager.unload: Unloading index data after ${this.indexIdleUnloadMs}ms idle`)
+    }
+
+    // Mark rebuild as needed so score() can trigger lazy reload
+    if (this.database) {
+      this.database._indexRebuildNeeded = true
+    }
+
+    this._clearIdleUnloadTimer()
+    const newData = {}
+    for (const field of this.indexedFields) {
+      newData[field] = {}
+    }
+    this.index.data = newData
+    this.indexLoaded = false
+  }
+
+  /**
+   * Check if a field is configured as an index
+   * @param {string} field - Field name
+   * @returns {boolean}
+   */
+  isFieldIndexed(field) {
+    if (!field) return false
+    if (!Array.isArray(this.indexedFields)) {
+      return false
+    }
+    return this.indexedFields.includes(field)
+  }
+
+  /**
+   * Determine whether the index has usable data for a given field
+   * @param {string} field - Field name
+   * @returns {boolean}
+   */
+  hasUsableIndexData(field) {
+    if (!field) return false
+    const fieldData = this.index?.data?.[field]
+    if (!fieldData || typeof fieldData !== 'object') {
+      return false
+    }
+
+    for (const key in fieldData) {
+      if (!Object.prototype.hasOwnProperty.call(fieldData, key)) continue
+      const entry = fieldData[key]
+      if (!entry) continue
+
+      if (entry.set && typeof entry.set.size === 'number' && entry.set.size > 0) {
+        return true
+      }
+
+      if (Array.isArray(entry.ranges) && entry.ranges.length > 0) {
+        const hasRangeData = entry.ranges.some(range => {
+          if (range === null || typeof range === 'undefined') {
+            return false
+          }
+          if (typeof range === 'object') {
+            const count = typeof range.count === 'number' ? range.count : 0
+            return count > 0
+          }
+          // When ranges are stored as individual numbers
+          return true
+        })
+
+        if (hasRangeData) {
+          return true
+        }
+      }
+    }
+
+    return false
+  }
+
+  // Ultra-fast range conversion - only for very large datasets
+  _toRanges(numbers) {
+    if (numbers.length === 0) return []
+    if (numbers.length < this.rangeThreshold) return numbers // Keep as-is for small arrays
+    
+    const sorted = numbers.sort((a, b) => a - b) // Sort in-place
+    const ranges = []
+    let start = sorted[0]
+    let count = 1
+    
+    for (let i = 1; i < sorted.length; i++) {
+      if (sorted[i] === sorted[i-1] + 1) {
+        count++
+      } else {
+        // End of consecutive sequence
+        if (count >= this.rangeThreshold) {
+          ranges.push({start, count})
+        } else {
+          // Add individual numbers for small sequences
+          for (let j = start; j < start + count; j++) {
+            ranges.push(j)
+          }
+        }
+        start = sorted[i]
+        count = 1
+      }
+    }
+    
+    // Handle last sequence
+    if (count >= this.rangeThreshold) {
+      ranges.push({start, count})
+    } else {
+      for (let j = start; j < start + count; j++) {
+        ranges.push(j)
+      }
+    }
+    
+    return ranges
+  }
+
+  // Ultra-fast range expansion
+  _fromRanges(ranges) {
+    if (!ranges || ranges.length === 0) return []
+    
+    const numbers = []
+    for (const item of ranges) {
+      if (typeof item === 'object' && item.start !== undefined) {
+        // It's a range - use direct loop for maximum speed
+        const end = item.start + item.count
+        for (let i = item.start; i < end; i++) {
+          numbers.push(i)
+        }
+      } else {
+        // It's an individual number
+        numbers.push(item)
+      }
+    }
+    return numbers
+  }
+
+  // Ultra-fast lookup - optimized for Set operations
+  _hasLineNumber(hybridData, lineNumber) {
+    if (!hybridData) return false
+    
+    // Check in Set first (O(1)) - most common case
+    if (hybridData.set && hybridData.set.has(lineNumber)) {
+      return true
+    }
+    
+    // Check in ranges only if necessary
+    if (hybridData.ranges && hybridData.ranges.length > 0) {
+      return this._searchInRanges(hybridData.ranges, lineNumber)
+    }
+    
+    return false
+  }
+
+  // Optimized search strategy
+  _searchInRanges(ranges, lineNumber) {
+    if (ranges.length < this.binarySearchThreshold) {
+      // Linear search for small ranges
+      return this._linearSearchRanges(ranges, lineNumber)
+    } else {
+      // Binary search for large ranges
+      return this._binarySearchRanges(ranges, lineNumber)
+    }
+  }
+
+  // Ultra-fast linear search
+  _linearSearchRanges(ranges, lineNumber) {
+    for (const item of ranges) {
+      if (typeof item === 'object' && item.start !== undefined) {
+        // It's a range
+        if (lineNumber >= item.start && lineNumber < item.start + item.count) {
+          return true
+        }
+      } else if (item === lineNumber) {
+        // It's an individual number
+        return true
+      }
+    }
+    return false
+  }
+
+  // Optimized binary search
+  _binarySearchRanges(ranges, lineNumber) {
+    let left = 0
+    let right = ranges.length - 1
+    
+    while (left <= right) {
+      const mid = Math.floor((left + right) / 2)
+      const range = ranges[mid]
+      
+      if (typeof range === 'object' && range.start !== undefined) {
+        // It's a range
+        if (lineNumber >= range.start && lineNumber < range.start + range.count) {
+          return true
+        } else if (lineNumber < range.start) {
+          right = mid - 1
+        } else {
+          left = mid + 1
+        }
+      } else {
+        // It's an individual number
+        if (range === lineNumber) {
+          return true
+        } else if (range < lineNumber) {
+          left = mid + 1
+        } else {
+          right = mid - 1
+        }
+      }
+    }
+    
+    return false
+  }
+
+  // Ultra-fast add operation - minimal overhead
+  _addLineNumber(hybridData, lineNumber) {
+    // Initialize structure if needed
+    if (!hybridData) {
+      hybridData = { set: new Set(), ranges: [] }
+    }
+    
+    // Add to Set directly (fastest path)
+    if (!hybridData.set) {
+      hybridData.set = new Set()
+    }
+    hybridData.set.add(lineNumber)
+    
+    // Optimize to ranges when Set gets reasonably large
+    if (hybridData.set.size >= this.rangeThreshold * 2) { // 20 elements
+      if (this.opts.debugMode) {
+        console.log(`🔧 Triggering range optimization: Set size ${hybridData.set.size} >= threshold ${this.rangeThreshold * 2}`)
+      }
+      this._optimizeToRanges(hybridData)
+    }
+    
+    return hybridData
+  }
+
+  // Ultra-fast remove operation
+  _removeLineNumber(hybridData, lineNumber) {
+    if (!hybridData) {
+      return hybridData
+    }
+    
+    // Remove from Set (fast path)
+    if (hybridData.set) {
+      hybridData.set.delete(lineNumber)
+    }
+    
+    // Remove from ranges (less common)
+    if (hybridData.ranges) {
+      hybridData.ranges = this._removeFromRanges(hybridData.ranges, lineNumber)
+    }
+    
+    return hybridData
+  }
+
+  // Optimized range removal
+  _removeFromRanges(ranges, lineNumber) {
+    if (!ranges || ranges.length === 0) return ranges
+    
+    const newRanges = []
+    
+    for (const item of ranges) {
+      if (typeof item === 'object' && item.start !== undefined) {
+        // It's a range
+        if (lineNumber >= item.start && lineNumber < item.start + item.count) {
+          // Split range if needed
+          if (lineNumber === item.start) {
+            // Remove first element
+            if (item.count > 1) {
+              newRanges.push({ start: item.start + 1, count: item.count - 1 })
+            }
+          } else if (lineNumber === item.start + item.count - 1) {
+            // Remove last element
+            if (item.count > 1) {
+              newRanges.push({ start: item.start, count: item.count - 1 })
+            }
+          } else {
+            // Remove from middle - split into two ranges
+            const beforeCount = lineNumber - item.start
+            const afterCount = item.count - beforeCount - 1
+            
+            if (beforeCount >= this.rangeThreshold) {
+              newRanges.push({ start: item.start, count: beforeCount })
+            } else {
+              // Add individual numbers for small sequences
+              for (let i = item.start; i < lineNumber; i++) {
+                newRanges.push(i)
+              }
+            }
+            
+            if (afterCount >= this.rangeThreshold) {
+              newRanges.push({ start: lineNumber + 1, count: afterCount })
+            } else {
+              // Add individual numbers for small sequences
+              for (let i = lineNumber + 1; i < item.start + item.count; i++) {
+                newRanges.push(i)
+              }
+            }
+          }
+        } else {
+          newRanges.push(item)
+        }
+      } else if (item !== lineNumber) {
+        // It's an individual number
+        newRanges.push(item)
+      }
+    }
+    
+    return newRanges
+  }
+
+  // Ultra-lazy range conversion - only when absolutely necessary
+  _optimizeToRanges(hybridData) {
+    if (!hybridData.set || hybridData.set.size === 0) {
+      return
+    }
+    
+    if (this.opts.debugMode) {
+      console.log(`🔧 Starting range optimization for Set with ${hybridData.set.size} elements`)
+    }
+    
+    // Only convert if we have enough data to make it worthwhile
+    if (hybridData.set.size < this.rangeThreshold) {
+      return
+    }
+    
+    // Convert Set to array and find consecutive sequences
+    const numbers = Array.from(hybridData.set).sort((a, b) => a - b)
+    const ranges = []
+    
+    let start = numbers[0]
+    let count = 1
+    
+    for (let i = 1; i < numbers.length; i++) {
+      if (numbers[i] === numbers[i-1] + 1) {
+        count++
+      } else {
+        // End of consecutive sequence
+        if (count >= this.rangeThreshold) {
+          ranges.push({start, count})
+          // Remove these numbers from Set
+          for (let j = start; j < start + count; j++) {
+            hybridData.set.delete(j)
+          }
+        }
+        start = numbers[i]
+        count = 1
+      }
+    }
+    
+    // Handle last sequence
+    if (count >= this.rangeThreshold) {
+      ranges.push({start, count})
+      for (let j = start; j < start + count; j++) {
+        hybridData.set.delete(j)
+      }
+    }
+    
+    // Add new ranges to existing ranges
+    if (ranges.length > 0) {
+      if (!hybridData.ranges) {
+        hybridData.ranges = []
+      }
+      hybridData.ranges.push(...ranges)
+      // Keep ranges sorted for efficient binary search
+      hybridData.ranges.sort((a, b) => {
+        const aStart = typeof a === 'object' ? a.start : a
+        const bStart = typeof b === 'object' ? b.start : b
+        return aStart - bStart
+      })
+    }
+  }
+
+  // Ultra-fast get all line numbers
+  _getAllLineNumbers(hybridData) {
+    if (!hybridData) return []
+    
+    // Use generator for lazy evaluation and better memory efficiency
+    return Array.from(this._getAllLineNumbersGenerator(hybridData))
+  }
+
+  // OPTIMIZATION: Generator-based approach for better memory efficiency
+  *_getAllLineNumbersGenerator(hybridData) {
+    const normalizeLineNumber = (value) => {
+      if (typeof value === 'number') {
+        return value
+      }
+      if (typeof value === 'string') {
+        const parsed = Number(value)
+        return Number.isNaN(parsed) ? value : parsed
+      }
+      if (typeof value === 'bigint') {
+        const maxSafe = BigInt(Number.MAX_SAFE_INTEGER)
+        return value <= maxSafe ? Number(value) : value
+      }
+      return value
+    }
+
+    // Yield from Set (fastest path)
+    if (hybridData.set) {
+      for (const num of hybridData.set) {
+        yield normalizeLineNumber(num)
+      }
+    }
+    
+    // Yield from ranges (optimized)
+    if (hybridData.ranges) {
+      for (const item of hybridData.ranges) {
+        if (typeof item === 'object' && item.start !== undefined) {
+          // It's a range - use direct loop for better performance
+          const end = item.start + item.count
+          for (let i = item.start; i < end; i++) {
+            yield normalizeLineNumber(i)
+          }
+        } else {
+          // It's an individual number
+          yield normalizeLineNumber(item)
+        }
+      }
+    }
+  }
+
+  // OPTIMIZATION 6: Ultra-fast add operation with incremental index updates
+  async add(row, lineNumber) {
+    this.markIndexUsed()
+    if (typeof row !== 'object' || !row) {
+      throw new Error('Invalid \'row\' parameter, it must be an object')
+    }
+    if (typeof lineNumber !== 'number') {
+      throw new Error('Invalid line number')
+    }
+    
+    // OPTIMIZATION 6: Use direct field access with minimal operations
+    const data = this.index.data
+    
+    // OPTIMIZATION 6: Pre-allocate field structures for better performance
+    const fields = Object.keys(this.opts.indexes || {})
+    for (const field of fields) {
+      // PERFORMANCE: Check if this is a term mapping field once
+      const isTermMappingField = this.database?.termManager && 
+        this.database.termManager.termMappingFields && 
+        this.database.termManager.termMappingFields.includes(field)
+      
+      // CRITICAL FIX: For term mapping fields, prefer ${field}Ids if available
+      // Records processed by processTermMapping have term IDs in ${field}Ids
+      // Records loaded from file have term IDs directly in ${field} (after restoreTermIdsAfterDeserialization)
+      let value
+      if (isTermMappingField) {
+        const termIdsField = `${field}Ids`
+        const termIds = row[termIdsField]
+        if (termIds && Array.isArray(termIds) && termIds.length > 0) {
+          // Use term IDs from ${field}Ids (preferred - from processTermMapping)
+          value = termIds
+        } else {
+          // Fallback: use field directly (for records loaded from file that have term IDs in field)
+          value = row[field]
+        }
+      } else {
+        value = row[field]
+      }
+      
+      if (value !== undefined && value !== null) {
+        // OPTIMIZATION 6: Initialize field structure if it doesn't exist
+        if (!data[field]) {
+          data[field] = {}
+        }
+        
+        const values = Array.isArray(value) ? value : [value]
+        for (const val of values) {
+          let key
+          
+          if (isTermMappingField && typeof val === 'number') {
+            // For term mapping fields, values are already term IDs
+            key = String(val)
+          } else if (isTermMappingField && typeof val === 'string') {
+            // Fallback: convert string to term ID
+            // CRITICAL: During indexing (add), we should use getTermId() to create IDs if needed
+            // This is different from queries where we use getTermIdWithoutIncrement() to avoid creating new IDs
+            const termId = this.database.termManager.getTermId(val)
+            key = String(termId)
+          } else {
+            // For non-term-mapping fields (including array:number), use values directly
+            key = String(val)
+          }
+          
+          // OPTIMIZATION 6: Use direct assignment for better performance
+          if (!data[field][key]) {
+            data[field][key] = { set: new Set(), ranges: [] }
+          }
+          
+          // OPTIMIZATION 6: Direct Set operation - fastest possible
+          data[field][key].set.add(lineNumber)
+          
+          // OPTIMIZATION 6: Lazy range optimization - only when beneficial
+          if (data[field][key].set.size >= this.rangeThreshold * 3) {
+            this._optimizeToRanges(data[field][key])
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * OPTIMIZATION 6: Add multiple records to the index in batch with optimized operations
+   * @param {Array} records - Records to add
+   * @param {number} startLineNumber - Starting line number
+   */
+  async addBatch(records, startLineNumber) {
+    this.markIndexUsed()
+    if (!records || !records.length) return
+    
+    // OPTIMIZATION 6: Pre-allocate index structures for better performance
+    const data = this.index.data
+    const fields = Object.keys(this.opts.indexes || {})
+    
+    for (const field of fields) {
+      if (!data[field]) {
+        data[field] = {}
+      }
+    }
+
+    // OPTIMIZATION 6: Use Map for batch processing to reduce lookups
+    const fieldUpdates = new Map()
+    
+    // OPTIMIZATION 6: Process all records in batch with optimized data structures
+    for (let i = 0; i < records.length; i++) {
+      const row = records[i]
+      const lineNumber = startLineNumber + i
+
+      for (const field of fields) {
+        // PERFORMANCE: Check if this is a term mapping field once
+        const isTermMappingField = this.database?.termManager && 
+          this.database.termManager.termMappingFields && 
+          this.database.termManager.termMappingFields.includes(field)
+        
+        // CRITICAL FIX: For term mapping fields, prefer ${field}Ids if available
+        // Records processed by processTermMapping have term IDs in ${field}Ids
+        // Records loaded from file have term IDs directly in ${field} (after restoreTermIdsAfterDeserialization)
+        let value
+        if (isTermMappingField) {
+          const termIdsField = `${field}Ids`
+          const termIds = row[termIdsField]
+          if (termIds && Array.isArray(termIds) && termIds.length > 0) {
+            // Use term IDs from ${field}Ids (preferred - from processTermMapping)
+            value = termIds
+          } else {
+            // Fallback: use field directly (for records loaded from file that have term IDs in field)
+            value = row[field]
+          }
+        } else {
+          value = row[field]
+        }
+        
+        if (value !== undefined && value !== null) {
+          const values = Array.isArray(value) ? value : [value]
+          for (const val of values) {
+            let key
+            
+            if (isTermMappingField && typeof val === 'number') {
+              // For term mapping fields, values are already term IDs
+              key = String(val)
+            } else if (isTermMappingField && typeof val === 'string') {
+              // Fallback: convert string to term ID
+              // CRITICAL: During indexing (addBatch), we should use getTermId() to create IDs if needed
+              // This is different from queries where we use getTermIdWithoutIncrement() to avoid creating new IDs
+              const termId = this.database.termManager.getTermId(val)
+              key = String(termId)
+            } else {
+              // For non-term-mapping fields (including array:number), use values directly
+              key = String(val)
+            }
+            
+            // OPTIMIZATION 6: Use Map for efficient batch updates
+            if (!fieldUpdates.has(field)) {
+              fieldUpdates.set(field, new Map())
+            }
+            
+            const fieldMap = fieldUpdates.get(field)
+            if (!fieldMap.has(key)) {
+              fieldMap.set(key, new Set())
+            }
+            
+            fieldMap.get(key).add(lineNumber)
+          }
+        }
+      }
+    }
+    
+    // OPTIMIZATION 6: Apply all updates in batch for better performance
+    for (const [field, fieldMap] of fieldUpdates) {
+      for (const [key, lineNumbers] of fieldMap) {
+        if (!data[field][key]) {
+          data[field][key] = { set: new Set(), ranges: [] }
+        }
+        
+        // OPTIMIZATION 6: Add all line numbers at once
+        for (const lineNumber of lineNumbers) {
+          data[field][key].set.add(lineNumber)
+        }
+        
+        // OPTIMIZATION 6: Lazy range optimization - only when beneficial
+        if (data[field][key].set.size >= this.rangeThreshold * 3) {
+          this._optimizeToRanges(data[field][key])
+        }
+      }
+    }
+  }
+
+  // Ultra-fast dry remove
+  dryRemove(ln) {
+    const data = this.index.data
+    for (const field in data) {
+      for (const value in data[field]) {
+        // Direct Set operation - fastest possible
+        if (data[field][value].set) {
+          data[field][value].set.delete(ln)
+        }
+        if (data[field][value].ranges) {
+          data[field][value].ranges = this._removeFromRanges(data[field][value].ranges, ln)
+        }
+        // Remove empty entries
+        if ((!data[field][value].set || data[field][value].set.size === 0) &&
+            (!data[field][value].ranges || data[field][value].ranges.length === 0)) {
+          delete data[field][value]
+        }
+      }
+    }
+  }
+
+
+  // Cleanup method to free memory
+  cleanup() {
+    this._clearIdleUnloadTimer()
+    const data = this.index.data
+    for (const field in data) {
+      for (const value in data[field]) {
+        if (data[field][value].set) {
+          if (typeof data[field][value].set.clearAll === 'function') {
+            data[field][value].set.clearAll()
+          } else if (typeof data[field][value].set.clear === 'function') {
+            data[field][value].set.clear()
+          }
+        }
+        if (data[field][value].ranges) {
+          data[field][value].ranges.length = 0
+        }
+      }
+      // Clear the entire field
+      data[field] = {}
+    }
+    // Clear all data
+    this.index.data = {}
+    this.totalLines = 0
+  }
+
+  // Clear all indexes
+  clear() {
+    this._clearIdleUnloadTimer()
+    this.index.data = {}
+    this.totalLines = 0
+  }
+
+  
+
+  // Update a record in the index
+  async update(oldRecord, newRecord, lineNumber = null) {
+    this.markIndexUsed()
+    if (!oldRecord || !newRecord) return
+    
+    // Remove old record by ID
+    await this.remove(oldRecord)
+    
+    // Add new record with provided line number or use hash of the ID
+    const actualLineNumber = lineNumber !== null ? lineNumber : this._getIdAsNumber(newRecord.id)
+    await this.add(newRecord, actualLineNumber)
+  }
+
+  // Convert string ID to number for line number
+  _getIdAsNumber(id) {
+    if (typeof id === 'number') return id
+    if (typeof id === 'string') {
+      // Simple hash function to convert string to number
+      let hash = 0
+      for (let i = 0; i < id.length; i++) {
+        const char = id.charCodeAt(i)
+        hash = ((hash << 5) - hash) + char
+        hash = hash & hash // Convert to 32-bit integer
+      }
+      return Math.abs(hash)
+    }
+    return 0
+  }
+
+  // Remove a record from the index
+  async remove(record) {
+    this.markIndexUsed()
+    if (!record) return
+    
+    // If record is an array of line numbers, use the original method
+    if (Array.isArray(record)) {
+      return this._removeLineNumbers(record)
+    }
+    
+    // If record is an object, remove by record data
+    if (typeof record === 'object' && record.id) {
+      return await this._removeRecord(record)
+    }
+  }
+
+  // Remove a specific record from the index
+  async _removeRecord(record) {
+    if (!record) return
+    
+    const data = this.index.data
+    const database = this.database
+    const persistedCount = Array.isArray(database?.offsets) ? database.offsets.length : 0
+    const lineMatchCache = new Map()
+
+    const doesLineNumberBelongToRecord = async (lineNumber) => {
+      if (lineMatchCache.has(lineNumber)) {
+        return lineMatchCache.get(lineNumber)
+      }
+
+      let belongs = false
+
+      try {
+        if (lineNumber >= persistedCount) {
+          const writeBufferIndex = lineNumber - persistedCount
+          const candidate = database?.writeBuffer?.[writeBufferIndex]
+          belongs = !!candidate && candidate.id === record.id
+        } else if (lineNumber >= 0) {
+          const range = database?.locate?.(lineNumber)
+          if (range && database.fileHandler && database.serializer) {
+            const [start, end] = range
+            const buffer = await database.fileHandler.readRange(start, end)
+            if (buffer && buffer.length > 0) {
+              let line = buffer.toString('utf8')
+              if (line) {
+                line = line.trim()
+                if (line.length > 0) {
+                  const storedRecord = database.serializer.deserialize(line)
+                  belongs = storedRecord && storedRecord.id === record.id
+                }
+              }
+            }
+          }
+        }
+      } catch (error) {
+        belongs = false
+      }
+
+      lineMatchCache.set(lineNumber, belongs)
+      return belongs
+    }
+
+    for (const field in data) {
+      if (record[field] !== undefined && record[field] !== null) {
+        const values = Array.isArray(record[field]) ? record[field] : [record[field]]
+        for (const val of values) {
+          let key
+          
+          // Check if this is a term mapping field (array:string fields only)
+          const isTermMappingField = this.database?.termManager && 
+            this.database.termManager.termMappingFields && 
+            this.database.termManager.termMappingFields.includes(field)
+          
+          if (isTermMappingField && typeof val === 'number') {
+            // For term mapping fields (array:string), the values are already term IDs
+            key = String(val)
+            if (this.database.opts.debugMode) {
+              console.log(`🔍 IndexManager._removeRecord: Using term ID ${val} directly for field "${field}"`)
+            }
+          } else if (isTermMappingField && typeof val === 'string') {
+            // For term mapping fields (array:string), convert string to term ID
+            const termId = this.database.termManager.getTermIdWithoutIncrement(val)
+            key = String(termId)
+            if (this.database.opts.debugMode) {
+              console.log(`🔍 IndexManager._removeRecord: Using term ID ${termId} for term "${val}"`)
+            }
+          } else {
+            // For non-term-mapping fields (including array:number), use values directly
+            key = String(val)
+            if (this.database?.opts?.debugMode) {
+              console.log(`🔍 IndexManager._removeRecord: Using value "${val}" directly for field "${field}"`)
+            }
+          }
+          
+          // Note: TermManager notification is handled by Database.mjs
+          // to avoid double decrementation during updates
+          
+          const indexEntry = data[field][key]
+          if (indexEntry) {
+            const lineNumbers = this._getAllLineNumbers(indexEntry)
+            const filteredLineNumbers = []
+
+            for (const lineNumber of lineNumbers) {
+              if (!(await doesLineNumberBelongToRecord(lineNumber))) {
+                filteredLineNumbers.push(lineNumber)
+              }
+            }
+            
+            if (filteredLineNumbers.length === 0) {
+              delete data[field][key]
+            } else {
+              // Rebuild the index value with filtered line numbers
+              data[field][key].set = new Set(filteredLineNumbers)
+              data[field][key].ranges = []
+            }
+          }
+        }
+      }
+    }
+  }
+
+  // Ultra-fast remove with batch processing (renamed from remove)
+  _removeLineNumbers(lineNumbers) {
+    if (!lineNumbers || lineNumbers.length === 0) return
+    
+    lineNumbers.sort((a, b) => a - b) // Sort ascending for efficient processing
+    
+    const data = this.index.data
+    for (const field in data) {
+      for (const value in data[field]) {
+        const numbers = this._getAllLineNumbers(data[field][value])
+        const newNumbers = []
+        
+        for (const ln of numbers) {
+          let offset = 0
+          for (const lineNumber of lineNumbers) {
+            if (lineNumber < ln) {
+              offset++
+            } else if (lineNumber === ln) {
+              offset = -1 // Mark for removal
+              break
+            }
+          }  
+          if (offset >= 0) {
+            newNumbers.push(ln - offset) // Update the value
+          }
+        }  
+        
+        if (newNumbers.length > 0) {
+          // Rebuild hybrid structure with new numbers
+          data[field][value] = { set: new Set(), ranges: [] }
+          for (const num of newNumbers) {
+            data[field][value] = this._addLineNumber(data[field][value], num)
+          }
+        } else {
+          delete data[field][value]
+        }
+      }
+    }
+  } 
+
+  // Ultra-fast replace with batch processing
+  replace(map) {
+    if (!map || map.size === 0) return
+    
+    const data = this.index.data
+    for (const field in data) {
+      for (const value in data[field]) {
+        const numbers = this._getAllLineNumbers(data[field][value])
+        const newNumbers = []
+        
+        for (const lineNumber of numbers) {
+          if (map.has(lineNumber)) {
+            newNumbers.push(map.get(lineNumber))
+          } else {
+            newNumbers.push(lineNumber)
+          }
+        }
+        
+        // Rebuild hybrid structure with new numbers
+        data[field][value] = { set: new Set(), ranges: [] }
+        for (const num of newNumbers) {
+          data[field][value] = this._addLineNumber(data[field][value], num)
+        }
+      }
+    }
+  }
+
+  // Ultra-fast query with early exit and smart processing
+  query(criteria, options = {}) {
+    this.markIndexUsed()
+    if (typeof options === 'boolean') {
+      options = { matchAny: options };
+    }
+    const { matchAny = false, caseInsensitive = false } = options;
+    
+    if (!criteria) {
+        // Return all line numbers when no criteria provided
+        return new Set(Array.from({ length: this.totalLines || 0 }, (_, i) => i));
+    }
+    
+    // Handle $not operator
+    if (criteria.$not && typeof criteria.$not === 'object') {
+      // Get all possible line numbers from database offsets or totalLines
+      const totalRecords = this.database?.offsets?.length || this.totalLines || 0;
+      const allLines = new Set(Array.from({ length: totalRecords }, (_, i) => i));
+      
+      // Get line numbers matching the $not condition
+      const notLines = this.query(criteria.$not, options);
+      
+      // Return complement (all lines except those matching $not condition)
+      const result = new Set([...allLines].filter(x => !notLines.has(x)));
+      
+      // If there are other conditions besides $not, we need to intersect with them
+      const otherCriteria = { ...criteria };
+      delete otherCriteria.$not;
+      
+      if (Object.keys(otherCriteria).length > 0) {
+        const otherResults = this.query(otherCriteria, options);
+        return new Set([...result].filter(x => otherResults.has(x)));
+      }
+      
+      return result;
+    }
+    
+    // Handle $and queries with parallel processing optimization
+    if (criteria.$and && Array.isArray(criteria.$and)) {
+      // OPTIMIZATION: Process conditions in parallel for better performance
+      if (criteria.$and.length > 1) {
+        // Process all conditions in parallel (synchronous since query is not async)
+        const conditionResults = criteria.$and.map(andCondition => 
+          this.query(andCondition, options)
+        );
+        
+        // Intersect all results for AND logic
+        let result = conditionResults[0];
+        for (let i = 1; i < conditionResults.length; i++) {
+          result = new Set([...result].filter(x => conditionResults[i].has(x)));
+        }
+        
+        // IMPORTANT: Check if there are other fields besides $and at the root level
+        // If so, we need to intersect with them too
+        const otherCriteria = { ...criteria };
+        delete otherCriteria.$and;
+        
+        if (Object.keys(otherCriteria).length > 0) {
+          const otherResults = this.query(otherCriteria, options);
+          result = new Set([...result].filter(x => otherResults.has(x)));
+        }
+        
+        return result || new Set();
+      } else {
+        // Single condition - check for other criteria at root level
+        const andResult = this.query(criteria.$and[0], options);
+        
+        const otherCriteria = { ...criteria };
+        delete otherCriteria.$and;
+        
+        if (Object.keys(otherCriteria).length > 0) {
+          const otherResults = this.query(otherCriteria, options);
+          return new Set([...andResult].filter(x => otherResults.has(x)));
+        }
+        
+        return andResult;
+      }
+    }
+    
+    const fields = Object.keys(criteria);
+    if (!fields.length) {
+        // Return all line numbers when criteria is empty object
+        return new Set(Array.from({ length: this.totalLines || 0 }, (_, i) => i));
+    }
+    
+    let matchingLines = matchAny ? new Set() : null;
+    const data = this.index.data
+  
+    for (const field of fields) {
+      // Skip logical operators - they are handled separately
+      if (field.startsWith('$')) continue;
+      
+      if (typeof data[field] === 'undefined') continue;
+      
+      const originalCriteriaValue = criteria[field];
+      const criteriaValue = normalizeCriteriaOperators(originalCriteriaValue, { target: 'legacy', preserveOriginal: true });
+      let lineNumbersForField = new Set();
+      const isNumericField = this.opts.indexes[field] === 'number';
+  
+      // Handle RegExp values directly (MUST check before object check since RegExp is an object)
+      if (criteriaValue instanceof RegExp) {
+        // RegExp cannot be efficiently queried using indices - fall back to streaming
+        // This will be handled by the QueryManager's streaming strategy
+        continue;
+      }
+
+      if (typeof criteriaValue === 'object' && !Array.isArray(criteriaValue) && criteriaValue !== null) {
+        const fieldIndex = data[field];
+
+        // Handle $in operator for array queries
+        if (criteriaValue.$in !== undefined && criteriaValue.$in !== null) {
+          const inValues = Array.isArray(criteriaValue.$in) ? criteriaValue.$in : [criteriaValue.$in];
+          
+          // PERFORMANCE: Cache term mapping field check once
+          const isTermMappingField = this.database?.termManager && 
+            this.database.termManager.termMappingFields && 
+            this.database.termManager.termMappingFields.includes(field)
+          
+          // PERFORMANCE: Track if any term was found and matched
+          let foundAnyMatch = false
+          
+          for (const inValue of inValues) {
+            // SPACE OPTIMIZATION: Convert search term to term ID for lookup
+            let searchTermId
+            
+            if (isTermMappingField && typeof inValue === 'number') {
+              // For term mapping fields (array:string), the search value is already a term ID
+              searchTermId = String(inValue)
+            } else if (isTermMappingField && typeof inValue === 'string') {
+              // For term mapping fields (array:string), convert string to term ID
+              const termId = this.database?.termManager?.getTermIdWithoutIncrement(String(inValue))
+              if (termId === undefined) {
+                // Term not found in termManager - skip this search value
+                // This means the term was never saved to the database
+                if (this.opts?.debugMode) {
+                  console.log(`⚠️  Term "${inValue}" not found in termManager for field "${field}" - skipping`)
+                }
+                continue // Skip this value, no matches possible
+              }
+              searchTermId = String(termId)
+            } else {
+              // For non-term-mapping fields (including array:number), use values directly
+              searchTermId = String(inValue)
+            }
+            
+            // PERFORMANCE: Direct lookup instead of iteration
+            let matched = false
+            if (caseInsensitive && typeof inValue === 'string') {
+              const searchLower = searchTermId.toLowerCase()
+              for (const value in fieldIndex) {
+                if (value.toLowerCase() === searchLower) {
+                  const numbers = this._getAllLineNumbers(fieldIndex[value]);
+                  for (const lineNumber of numbers) {
+                    lineNumbersForField.add(lineNumber);
+                  }
+                  matched = true
+                  foundAnyMatch = true
+                }
+              }
+            } else {
+              const indexData = fieldIndex[searchTermId]
+              if (indexData) {
+                const numbers = this._getAllLineNumbers(indexData);
+                for (const lineNumber of numbers) {
+                  lineNumbersForField.add(lineNumber);
+                }
+                matched = true
+                foundAnyMatch = true
+              }
+            }
+          }
+          
+          // CRITICAL FIX: If no matches found at all (all terms were unknown or not in index),
+          // lineNumbersForField remains empty which is correct (no results for $in)
+          // This is handled correctly by the caller - empty Set means no matches
+        }
+        // Handle $nin operator (not in) - returns complement of $in
+        else if (criteriaValue.$nin !== undefined) {
+          const ninValues = Array.isArray(criteriaValue.$nin) ? criteriaValue.$nin : [criteriaValue.$nin];
+          
+          // Get all possible line numbers
+          const totalRecords = this.database?.offsets?.length || this.totalLines || 0;
+          const allLines = new Set(Array.from({ length: totalRecords }, (_, i) => i));
+          
+          // Get line numbers that match any of the $nin values
+          const matchingLines = new Set();
+          
+          // PERFORMANCE: Cache term mapping field check once
+          const isTermMappingField = this.database?.termManager && 
+            this.database.termManager.termMappingFields && 
+            this.database.termManager.termMappingFields.includes(field)
+          
+          for (const ninValue of ninValues) {
+            // SPACE OPTIMIZATION: Convert search term to term ID for lookup
+            let searchTermId
+            
+            if (isTermMappingField && typeof ninValue === 'number') {
+              // For term mapping fields (array:string), the search value is already a term ID
+              searchTermId = String(ninValue)
+            } else if (isTermMappingField && typeof ninValue === 'string') {
+              // For term mapping fields (array:string), convert string to term ID
+              const termId = this.database?.termManager?.getTermIdWithoutIncrement(String(ninValue))
+              if (termId === undefined) {
+                // Term not found - skip this value (can't exclude what doesn't exist)
+                if (this.opts?.debugMode) {
+                  console.log(`⚠️  Term "${ninValue}" not found in termManager for field "${field}" - skipping`)
+                }
+                continue
+              }
+              searchTermId = String(termId)
+            } else {
+              // For non-term-mapping fields (including array:number), use values directly
+              searchTermId = String(ninValue)
+            }
+            
+            // PERFORMANCE: Direct lookup instead of iteration
+            if (caseInsensitive && typeof ninValue === 'string') {
+              const searchLower = searchTermId.toLowerCase()
+              for (const value in fieldIndex) {
+                if (value.toLowerCase() === searchLower) {
+                  const numbers = this._getAllLineNumbers(fieldIndex[value]);
+                  for (const lineNumber of numbers) {
+                    matchingLines.add(lineNumber);
+                  }
+                }
+              }
+            } else {
+              const indexData = fieldIndex[searchTermId]
+              if (indexData) {
+                const numbers = this._getAllLineNumbers(indexData);
+                for (const lineNumber of numbers) {
+                  matchingLines.add(lineNumber);
+                }
+              }
+            }
+          }
+          
+          // Return complement: all lines EXCEPT those matching $nin values
+          lineNumbersForField = new Set([...allLines].filter(x => !matchingLines.has(x)));
+        }
+        // Handle $contains operator for array queries
+        else if (criteriaValue.$contains !== undefined) {
+          const containsValue = criteriaValue.$contains;
+          // Handle case-insensitive for $contains
+          if (caseInsensitive && typeof containsValue === 'string') {
+            for (const value in fieldIndex) {
+              if (value.toLowerCase() === containsValue.toLowerCase()) {
+                const numbers = this._getAllLineNumbers(fieldIndex[value]);
+                for (const lineNumber of numbers) {
+                  lineNumbersForField.add(lineNumber);
+                }
+              }
+            }
+          } else {
+            if (fieldIndex[containsValue]) {
+              const numbers = this._getAllLineNumbers(fieldIndex[containsValue]);
+              for (const lineNumber of numbers) {
+                lineNumbersForField.add(lineNumber);
+              }
+            }
+          }
+        }
+        // Handle $all operator for array queries - FIXED FOR TERM MAPPING
+        else if (criteriaValue.$all !== undefined) {
+          const allValues = Array.isArray(criteriaValue.$all) ? criteriaValue.$all : [criteriaValue.$all];
+
+          const isTermMappingField = this.database?.termManager && 
+            this.database.termManager.termMappingFields && 
+            this.database.termManager.termMappingFields.includes(field)
+
+          const normalizeValue = (value) => {
+            if (isTermMappingField) {
+              if (typeof value === 'number') {
+                return String(value)
+              }
+              if (typeof value === 'string') {
+                const termId = this.database?.termManager?.getTermIdWithoutIncrement(value)
+                if (termId !== undefined) {
+                  return String(termId)
+                }
+                return null
+              }
+              return null
+            }
+            return String(value)
+          }
+
+          const normalizedValues = []
+          for (const value of allValues) {
+            const normalized = normalizeValue(value)
+            if (normalized === null) {
+              // Term not found in term manager, no matches possible
+              return lineNumbersForField
+            }
+            normalizedValues.push(normalized)
+          }
+          
+          // Early exit optimization
+          if (normalizedValues.length === 0) {
+            // Empty $all matches everything
+            for (const value in fieldIndex) {
+              const numbers = this._getAllLineNumbers(fieldIndex[value]);
+              for (const lineNumber of numbers) {
+                lineNumbersForField.add(lineNumber);
+              }
+            }
+          } else {
+            // For term mapping, we need to find records that contain ALL specified terms
+            // This requires a different approach than simple field matching
+            
+            // First, get all line numbers that contain each individual term
+            const termLineNumbers = new Map();
+            for (const term of normalizedValues) {
+              if (fieldIndex[term]) {
+                termLineNumbers.set(term, new Set(this._getAllLineNumbers(fieldIndex[term])));
+              } else {
+                // If any term doesn't exist, no records can match $all
+                termLineNumbers.set(term, new Set());
+              }
+            }
+            
+            // Find intersection of all term line numbers
+            if (termLineNumbers.size > 0) {
+              const allTermSets = Array.from(termLineNumbers.values());
+              let intersection = allTermSets[0];
+              
+              for (let i = 1; i < allTermSets.length; i++) {
+                intersection = new Set([...intersection].filter(x => allTermSets[i].has(x)));
+              }
+              
+              // Add all line numbers from intersection
+              for (const lineNumber of intersection) {
+                lineNumbersForField.add(lineNumber);
+              }
+            }
+          }
+        }
+        // Handle other operators
+        else {
+          for (const value in fieldIndex) {
+            let includeValue = true;
+            if (isNumericField) {
+              const numericValue = parseFloat(value);
+              if (!isNaN(numericValue)) {
+                if (criteriaValue['>'] !== undefined && numericValue <= criteriaValue['>']) {
+                  includeValue = false;
+                }
+                if (criteriaValue['>='] !== undefined && numericValue < criteriaValue['>=']) {
+                  includeValue = false;
+                }
+                if (criteriaValue['<'] !== undefined && numericValue >= criteriaValue['<']) {
+                  includeValue = false;
+                }
+                if (criteriaValue['<='] !== undefined && numericValue > criteriaValue['<=']) {
+                  includeValue = false;
+                }
+                if (criteriaValue['!='] !== undefined) {
+                  const excludeValues = Array.isArray(criteriaValue['!='])
+                    ? criteriaValue['!=']
+                    : [criteriaValue['!=']];
+                  if (excludeValues.includes(numericValue)) {
+                    includeValue = false;
+                  }
+                }
+              }
+            } else {
+              if (criteriaValue['contains'] !== undefined && typeof value === 'string') {
+                const term = String(criteriaValue['contains']);
+                if (caseInsensitive) {
+                  if (!value.toLowerCase().includes(term.toLowerCase())) {
+                    includeValue = false;
+                  }
+                } else {
+                  if (!value.includes(term)) {
+                    includeValue = false;
+                  }
+                }
+              }
+              if (criteriaValue['regex'] !== undefined) {
+                let regex;
+                if (typeof criteriaValue['regex'] === 'string') {
+                  regex = new RegExp(criteriaValue['regex'], caseInsensitive ? 'i' : '');
+                } else if (criteriaValue['regex'] instanceof RegExp) {
+                  if (caseInsensitive && !criteriaValue['regex'].ignoreCase) {
+                    const flags = criteriaValue['regex'].flags.includes('i')
+                      ? criteriaValue['regex'].flags
+                      : criteriaValue['regex'].flags + 'i';
+                    regex = new RegExp(criteriaValue['regex'].source, flags);
+                  } else {
+                    regex = criteriaValue['regex'];
+                  }
+                }
+                if (regex) {
+                  // For array fields, test regex against each element
+                  if (Array.isArray(value)) {
+                    if (!value.some(element => regex.test(String(element)))) {
+                      includeValue = false;
+                    }
+                  } else {
+                    // For non-array fields, test regex against the value directly
+                    if (!regex.test(String(value))) {
+                      includeValue = false;
+                    }
+                  }
+                }
+              }
+              if (criteriaValue['!='] !== undefined) {
+                const excludeValues = Array.isArray(criteriaValue['!='])
+                  ? criteriaValue['!=']
+                  : [criteriaValue['!=']];
+                if (excludeValues.includes(value)) {
+                  includeValue = false;
+                }
+              }
+            }
+
+            if (includeValue) {
+              const numbers = this._getAllLineNumbers(fieldIndex[value]);
+              for (const lineNumber of numbers) {
+                lineNumbersForField.add(lineNumber);
+              }
+            }
+          }
+        }
+      } else {
+        // Simple equality comparison - handle array queries
+        const values = Array.isArray(criteriaValue) ? criteriaValue : [criteriaValue];
+        const fieldData = data[field];
+        for (const searchValue of values) {
+          // SPACE OPTIMIZATION: Convert search term to term ID for lookup
+          let searchTermId
+          
+          // PERFORMANCE: Cache term mapping field check once per field
+          const isTermMappingField = this.database?.termManager && 
+            this.database.termManager.termMappingFields && 
+            this.database.termManager.termMappingFields.includes(field)
+          
+          if (isTermMappingField && typeof searchValue === 'number') {
+            // For term mapping fields (array:string), the search value is already a term ID
+            searchTermId = String(searchValue)
+          } else if (isTermMappingField && typeof searchValue === 'string') {
+            // For term mapping fields (array:string), convert string to term ID
+            const termId = this.database?.termManager?.getTermIdWithoutIncrement(String(searchValue))
+            if (termId === undefined) {
+              // Term not found - skip this value
+              if (this.opts?.debugMode) {
+                console.log(`⚠️  Term "${searchValue}" not found in termManager for field "${field}" - skipping`)
+              }
+              continue // Skip this value, no matches possible
+            }
+            searchTermId = String(termId)
+          } else {
+            // For non-term-mapping fields (including array:number), use values directly
+            searchTermId = String(searchValue)
+          }
+          
+          for (const key in fieldData) {
+            let match = false;
+            if (isNumericField) {
+              // Convert both parts to number
+              match = Number(key) === Number(searchValue);
+            } else {
+              // SPACE OPTIMIZATION: Compare term IDs instead of full terms
+              if (caseInsensitive) {
+                // For case-insensitive, we need to check if the search term ID matches any key
+                match = key === String(searchTermId);
+              } else {
+                match = key === String(searchTermId);
+              }
+            }
+            if (match) {
+              const numbers = this._getAllLineNumbers(fieldData[key]);
+              for (const lineNumber of numbers) {
+                lineNumbersForField.add(lineNumber);
+              }
+            }
+          }
+        }
+      }
+  
+      // Consolidate results from each field
+      if (matchAny) {
+        matchingLines = new Set([...matchingLines, ...lineNumbersForField]);
+      } else {
+        if (matchingLines === null) {
+          matchingLines = lineNumbersForField;
+        } else {
+          matchingLines = new Set([...matchingLines].filter(n => lineNumbersForField.has(n)));
+        }
+        if (!matchingLines.size) {
+          return new Set();
+        }
+      }
+    }
+    return matchingLines || new Set();
+  } 
+
+  /**
+   * Check if any records exist for given field and terms (index-only, ultra-fast)
+   * Stops at first match for maximum performance - no disk I/O required
+   * 
+   * @param {string} fieldName - Indexed field name (e.g., 'nameTerms', 'groupTerms')
+   * @param {string|Array<string>} terms - Single term or array of terms to check
+   * @param {Object} options - Options: { $all: true/false, caseInsensitive: true/false, excludes: Array<string> }
+   * @returns {boolean} - True if at least one match exists
+   * 
+   * @example
+   * // Check if any record has 'channel' in nameTerms
+   * indexManager.exists('nameTerms', 'channel')
+   * 
+   * @example
+   * // Check if any record has ALL terms ['a', 'e'] in nameTerms ($all)
+   * indexManager.exists('nameTerms', ['a', 'e'], { $all: true })
+   * 
+   * @example
+   * // Check if any record has ANY of the terms ['channel', 'tv'] in nameTerms
+   * indexManager.exists('nameTerms', ['channel', 'tv'], { $all: false })
+   * 
+   * @example
+   * // Check if any record has 'tv' but NOT 'globo' in nameTerms
+   * indexManager.exists('nameTerms', 'tv', { excludes: ['globo'] })
+   * 
+   * @example
+   * // Check if any record has ['tv', 'news'] but NOT 'sports' in nameTerms
+   * indexManager.exists('nameTerms', ['tv', 'news'], { $all: true, excludes: ['sports'] })
+   */
+  exists(fieldName, terms, options = {}) {
+    this.markIndexUsed()
+    // Early exit: validate fieldName
+    if (!fieldName || typeof fieldName !== 'string') {
+      return false;
+    }
+    
+    // Early exit: check if field is indexed
+    if (!this.isFieldIndexed(fieldName)) {
+      return false;
+    }
+    
+    const fieldIndex = this.index.data[fieldName];
+    if (!fieldIndex || typeof fieldIndex !== 'object') {
+      return false;
+    }
+    
+    // Normalize terms to array
+    const termsArray = Array.isArray(terms) ? terms : [terms];
+    if (termsArray.length === 0) {
+      return false;
+    }
+    
+    const { $all = false, caseInsensitive = false, excludes = [] } = options;
+    const hasExcludes = Array.isArray(excludes) && excludes.length > 0;
+    const isTermMappingField = this.database?.termManager && 
+      this.database.termManager.termMappingFields && 
+      this.database.termManager.termMappingFields.includes(fieldName);
+    
+    // Helper: check if termData has any line numbers (ULTRA LIGHT - no expansion)
+    const hasData = (termData) => {
+      if (!termData) return false;
+      // Check Set size (O(1))
+      if (termData.set && termData.set.size > 0) {
+        return true;
+      }
+      // Check ranges length (O(1))
+      if (termData.ranges && termData.ranges.length > 0) {
+        return true;
+      }
+      return false;
+    };
+    
+    // Helper: get term key with term mapping and case-insensitive support
+    const getTermKey = (term, useCaseInsensitive = false) => {
+      if (isTermMappingField && typeof term === 'string') {
+        let termId;
+        if (useCaseInsensitive) {
+          // For case-insensitive, search termManager for case-insensitive match
+          const searchLower = String(term).toLowerCase();
+          termId = null;
+          if (this.database?.termManager?.termToId) {
+            for (const [termStr, id] of this.database.termManager.termToId.entries()) {
+              if (termStr.toLowerCase() === searchLower) {
+                termId = id;
+                break;
+              }
+            }
+          }
+        } else {
+          termId = this.database?.termManager?.getTermIdWithoutIncrement(String(term));
+        }
+        
+        if (termId === undefined || termId === null) {
+          return null;
+        }
+        return String(termId);
+      }
+      
+      // For non-term-mapping fields
+      if (useCaseInsensitive && typeof term === 'string') {
+        const searchLower = String(term).toLowerCase();
+        for (const key in fieldIndex) {
+          if (key.toLowerCase() === searchLower) {
+            return key;
+          }
+        }
+        return null;
+      }
+      
+      return String(term);
+    };
+    
+    // Handle $all (all terms must exist and have intersection)
+    if ($all) {
+      // Collect term data for all terms first (with early exit)
+      const termDataArray = [];
+      
+      for (const term of termsArray) {
+        // Get term key (with term mapping if applicable)
+        let termKey;
+        if (isTermMappingField && typeof term === 'string') {
+          let termId;
+          if (caseInsensitive) {
+            // For case-insensitive, search termManager for case-insensitive match
+            const searchLower = String(term).toLowerCase();
+            termId = null;
+            for (const [termStr, id] of this.database.termManager.termToId.entries()) {
+              if (termStr.toLowerCase() === searchLower) {
+                termId = id;
+                break;
+              }
+            }
+          } else {
+            termId = this.database?.termManager?.getTermIdWithoutIncrement(String(term));
+          }
+          
+          if (termId === undefined || termId === null) {
+            return false; // Early exit: term doesn't exist in mapping
+          }
+          termKey = String(termId);
+        } else {
+          termKey = String(term);
+          // For non-term-mapping fields with case-insensitive, search index keys
+          if (caseInsensitive && typeof term === 'string') {
+            const searchLower = termKey.toLowerCase();
+            let foundKey = null;
+            for (const key in fieldIndex) {
+              if (key.toLowerCase() === searchLower) {
+                foundKey = key;
+                break;
+              }
+            }
+            if (foundKey === null) {
+              return false; // Early exit: term doesn't exist
+            }
+            termKey = foundKey;
+          }
+        }
+        
+        // Check if term exists in index
+        const termData = fieldIndex[termKey];
+        if (!termData || !hasData(termData)) {
+          return false; // Early exit: term doesn't exist or has no data
+        }
+        
+        termDataArray.push(termData);
+      }
+      
+      // If we got here, all terms exist and have data
+      // Now check if there's intersection (only if more than one term)
+      if (termDataArray.length === 1) {
+        // Single term - check excludes if any
+        if (!hasExcludes) {
+          return true; // Single term, already verified it has data, no excludes
+        }
+        // Need to check excludes - expand line numbers
+        const lineNumbers = this._getAllLineNumbers(termDataArray[0]);
+        const candidateLines = new Set(lineNumbers);
+        
+        // Remove lines that have exclude terms
+        for (const excludeTerm of excludes) {
+          const excludeKey = getTermKey(excludeTerm, caseInsensitive);
+          if (excludeKey === null) continue;
+          
+          const excludeData = fieldIndex[excludeKey];
+          if (!excludeData) continue;
+          
+          const excludeLines = this._getAllLineNumbers(excludeData);
+          for (const line of excludeLines) {
+            candidateLines.delete(line);
+          }
+          
+          // Early exit if all candidates excluded
+          if (candidateLines.size === 0) {
+            return false;
+          }
+        }
+        
+        return candidateLines.size > 0;
+      }
+      
+      // For multiple terms, we need to check intersection
+      // But we want to do this as lightly as possible
+      // Get line numbers only for intersection check (unavoidable for $all)
+      const termLineNumberSets = [];
+      for (const termData of termDataArray) {
+        const lineNumbers = this._getAllLineNumbers(termData);
+        if (lineNumbers.length === 0) {
+          return false; // Early exit: no line numbers (shouldn't happen, but safety check)
+        }
+        termLineNumberSets.push(new Set(lineNumbers));
+      }
+      
+      // Calculate intersection incrementally with early exit
+      let intersection = termLineNumberSets[0];
+      for (let i = 1; i < termLineNumberSets.length; i++) {
+        // Filter intersection to only include items in current set
+        intersection = new Set([...intersection].filter(x => termLineNumberSets[i].has(x)));
+        if (intersection.size === 0) {
+          return false; // Early exit: intersection is empty
+        }
+      }
+      
+      // Apply excludes if any
+      if (hasExcludes) {
+        for (const excludeTerm of excludes) {
+          const excludeKey = getTermKey(excludeTerm, caseInsensitive);
+          if (excludeKey === null) continue;
+          
+          const excludeData = fieldIndex[excludeKey];
+          if (!excludeData) continue;
+          
+          const excludeLines = this._getAllLineNumbers(excludeData);
+          for (const line of excludeLines) {
+            intersection.delete(line);
+          }
+          
+          // Early exit if all candidates excluded
+          if (intersection.size === 0) {
+            return false;
+          }
+        }
+      }
+      
+      return intersection.size > 0;
+    }
+    
+    // Handle $in behavior (any term exists) - default - ULTRA LIGHT
+    // If no excludes, use ultra-fast path (no expansion needed)
+    if (!hasExcludes) {
+      for (const term of termsArray) {
+        // Handle case-insensitive FIRST (before normal conversion)
+        if (caseInsensitive && typeof term === 'string') {
+          if (isTermMappingField && this.database?.termManager?.termToId) {
+            // For term mapping fields, we need to find the term in termManager first
+            // (case-insensitive), then convert to ID
+            const searchLower = String(term).toLowerCase();
+            let foundTermId = null;
+            
+            // Search termManager for case-insensitive match
+            for (const [termStr, termId] of this.database.termManager.termToId.entries()) {
+              if (termStr.toLowerCase() === searchLower) {
+                foundTermId = termId;
+                break;
+              }
+            }
+            
+            if (foundTermId !== null) {
+              const termData = fieldIndex[String(foundTermId)];
+              if (hasData(termData)) {
+                return true; // Early exit: found a match
+              }
+            }
+            // If not found, continue to next term
+            continue;
+          } else {
+            // For non-term-mapping fields, search index keys directly
+            const searchLower = String(term).toLowerCase();
+            for (const key in fieldIndex) {
+              if (key.toLowerCase() === searchLower) {
+                const termData = fieldIndex[key];
+                if (hasData(termData)) {
+                  return true; // Early exit: found a match
+                }
+              }
+            }
+            // If not found, continue to next term
+            continue;
+          }
+        }
+        
+        // Normal (case-sensitive) lookup
+        const termKey = getTermKey(term, false);
+        if (termKey === null) {
+          continue; // Term not in mapping, try next
+        }
+        
+        // Direct lookup (fastest path) - O(1) hash lookup
+        const termData = fieldIndex[termKey];
+        if (hasData(termData)) {
+          return true; // Early exit: found a match
+        }
+      }
+      
+      return false;
+    }
+    
+    // With excludes, we need to collect candidates and filter
+    const candidateLines = new Set();
+    
+    for (const term of termsArray) {
+      // Handle case-insensitive FIRST (before normal conversion)
+      if (caseInsensitive && typeof term === 'string') {
+        if (isTermMappingField && this.database?.termManager?.termToId) {
+          // For term mapping fields, we need to find the term in termManager first
+          // (case-insensitive), then convert to ID
+          const searchLower = String(term).toLowerCase();
+          let foundTermId = null;
+          
+          // Search termManager for case-insensitive match
+          for (const [termStr, termId] of this.database.termManager.termToId.entries()) {
+            if (termStr.toLowerCase() === searchLower) {
+              foundTermId = termId;
+              break;
+            }
+          }
+          
+          if (foundTermId !== null) {
+            const termData = fieldIndex[String(foundTermId)];
+            if (hasData(termData)) {
+              // Add line numbers to candidates (need to expand for excludes check)
+              const lineNumbers = this._getAllLineNumbers(termData);
+              for (const line of lineNumbers) {
+                candidateLines.add(line);
+              }
+            }
+          }
+          continue;
+        } else {
+          // For non-term-mapping fields, search index keys directly
+          const searchLower = String(term).toLowerCase();
+          for (const key in fieldIndex) {
+            if (key.toLowerCase() === searchLower) {
+              const termData = fieldIndex[key];
+              if (hasData(termData)) {
+                // Add line numbers to candidates
+                const lineNumbers = this._getAllLineNumbers(termData);
+                for (const line of lineNumbers) {
+                  candidateLines.add(line);
+                }
+              }
+            }
+          }
+          continue;
+        }
+      }
+      
+      // Normal (case-sensitive) lookup
+      const termKey = getTermKey(term, false);
+      if (termKey === null) {
+        continue; // Term not in mapping, try next
+      }
+      
+      // Direct lookup
+      const termData = fieldIndex[termKey];
+      if (hasData(termData)) {
+        // Add line numbers to candidates (need to expand for excludes check)
+        const lineNumbers = this._getAllLineNumbers(termData);
+        for (const line of lineNumbers) {
+          candidateLines.add(line);
+        }
+      }
+    }
+    
+    // If no candidates found, return false
+    if (candidateLines.size === 0) {
+      return false;
+    }
+    
+    // Apply excludes
+    for (const excludeTerm of excludes) {
+      const excludeKey = getTermKey(excludeTerm, caseInsensitive);
+      if (excludeKey === null) continue;
+      
+      const excludeData = fieldIndex[excludeKey];
+      if (!excludeData) continue;
+      
+      const excludeLines = this._getAllLineNumbers(excludeData);
+      for (const line of excludeLines) {
+        candidateLines.delete(line);
+      }
+      
+      // Early exit if all candidates excluded
+      if (candidateLines.size === 0) {
+        return false;
+      }
+    }
+    
+    return candidateLines.size > 0;
+  }
+
+  /**
+   * Evaluate many index-only existence checks while sharing the same field data.
+   * Returns a map keyed by the criteria id so callers can reuse a single scan per field.
+   *
+   * @param {string} fieldName - Indexed field name
+   * @param {Array<Object>} criteriaArray - Array of { id, terms, options }
+   * @param {Object} opts - { limit, allowPartial } (limit only applies when allowPartial is true)
+   * @returns {Object<string, boolean>} - Map of criteria id to boolean existence
+   */
+  multiExists(fieldName, criteriaArray, opts = {}) {
+    this.markIndexUsed()
+    const results = {}
+    if (!Array.isArray(criteriaArray) || criteriaArray.length === 0) {
+      return results
+    }
+
+    const prepared = []
+    for (let i = 0; i < criteriaArray.length; i++) {
+      const entry = criteriaArray[i] || {}
+      const key = entry.id !== undefined && entry.id !== null ? String(entry.id) : `criteria-${i}`
+      results[key] = false
+      prepared.push({
+        id: key,
+        terms: entry.terms,
+        options: entry.options || {}
+      })
+    }
+
+    if (!fieldName || typeof fieldName !== 'string') {
+      return results
+    }
+
+    const fieldIndex = this.index?.data?.[fieldName]
+    if (!fieldIndex || typeof fieldIndex !== 'object') {
+      return results
+    }
+
+    const termManager = this.database?.termManager
+    const isTermMappingField = Boolean(termManager && termManager.termMappingFields && termManager.termMappingFields.includes(fieldName))
+
+    const normalizedLimit = Number.isFinite(opts.limit) && opts.limit > 0 ? Math.floor(opts.limit) : Infinity
+    const allowPartial = Boolean(opts.allowPartial)
+    const positiveTarget = allowPartial ? Math.max(1, normalizedLimit) : Infinity
+
+    const termCache = new Map()
+    let caseInsensitiveLookup = null
+    let termMappingLookup = null
+
+    const normalizeValues = (value) => {
+      if (value === null || value === undefined) {
+        return []
+      }
+      if (Array.isArray(value)) {
+        return value.slice()
+      }
+      return [value]
+    }
+
+    const buildCaseInsensitiveLookup = () => {
+      const map = new Map()
+      for (const key in fieldIndex) {
+        const lowerKey = key.toLowerCase()
+        if (!map.has(lowerKey)) {
+          map.set(lowerKey, key)
+        }
+      }
+      return map
+    }
+
+    const buildTermMappingLookup = () => {
+      const map = new Map()
+      const termToId = termManager?.termToId
+      if (termToId) {
+        for (const [termStr, id] of termToId.entries()) {
+          const lower = termStr.toLowerCase()
+          if (!map.has(lower)) {
+            map.set(lower, String(id))
+          }
+        }
+      }
+      return map
+    }
+
+    const resolveTermKey = (value, caseInsensitive) => {
+      if (value === null || value === undefined) {
+        return null
+      }
+
+      if (isTermMappingField) {
+        if (typeof value === 'string') {
+          if (caseInsensitive) {
+            if (!termMappingLookup) {
+              termMappingLookup = buildTermMappingLookup()
+            }
+            return termMappingLookup.get(value.toLowerCase()) ?? null
+          }
+          const termId = termManager?.getTermIdWithoutIncrement(String(value))
+          if (termId === undefined || termId === null) {
+            return null
+          }
+          return String(termId)
+        }
+        return String(value)
+      }
+
+      if (caseInsensitive && typeof value === 'string') {
+        if (!caseInsensitiveLookup) {
+          caseInsensitiveLookup = buildCaseInsensitiveLookup()
+        }
+        return caseInsensitiveLookup.get(value.toLowerCase()) ?? null
+      }
+
+      return String(value)
+    }
+
+    const ensureTermEntry = (termKey) => {
+      if (!termCache.has(termKey)) {
+        const data = fieldIndex[termKey]
+        const entry = {
+          data,
+          hasData: Boolean(data && ((data.set && data.set.size > 0) || (data.ranges && data.ranges.length > 0))),
+          lineArray: null
+        }
+        termCache.set(termKey, entry)
+      }
+      return termCache.get(termKey)
+    }
+
+    const getLineArray = (entry) => {
+      if (!entry.hasData || !entry.data) return []
+      if (!entry.lineArray) {
+        entry.lineArray = this._getAllLineNumbers(entry.data)
+      }
+      return entry.lineArray
+    }
+
+    const applyExcludes = (lineSet, excludeKeys) => {
+      if (excludeKeys.length === 0) {
+        return lineSet.size > 0
+      }
+      for (const excludeKey of excludeKeys) {
+        const excludeEntry = ensureTermEntry(excludeKey)
+        if (!excludeEntry.hasData) continue
+        const excludeLines = getLineArray(excludeEntry)
+        for (const line of excludeLines) {
+          lineSet.delete(line)
+        }
+        if (lineSet.size === 0) {
+          return false
+        }
+      }
+      return lineSet.size > 0
+    }
+
+    let successes = 0
+
+    for (const criterion of prepared) {
+      if (successes >= positiveTarget) {
+        break
+      }
+
+      const { id, terms, options } = criterion
+      const { $all = false, caseInsensitive = false, excludes = [] } = options
+
+      const normalizedTerms = normalizeValues(terms)
+      if (normalizedTerms.length === 0) {
+        continue
+      }
+
+      const normalizedExcludes = normalizeValues(excludes)
+
+      const resolvedKeys = []
+      const seenKeys = new Set()
+      let missingRequiredTerm = false
+
+      for (const term of normalizedTerms) {
+        const termKey = resolveTermKey(term, caseInsensitive)
+        if (termKey === null) {
+          if ($all) {
+            missingRequiredTerm = true
+            break
+          }
+          continue
+        }
+        if (!seenKeys.has(termKey)) {
+          seenKeys.add(termKey)
+          resolvedKeys.push(termKey)
+        }
+      }
+
+      if ($all && missingRequiredTerm) {
+        continue
+      }
+
+      if (resolvedKeys.length === 0) {
+        continue
+      }
+
+      const excludeKeySet = []
+      const seenExcludeKeys = new Set()
+      for (const excludeTerm of normalizedExcludes) {
+        const excludeKey = resolveTermKey(excludeTerm, caseInsensitive)
+        if (excludeKey && !seenExcludeKeys.has(excludeKey)) {
+          seenExcludeKeys.add(excludeKey)
+          excludeKeySet.push(excludeKey)
+        }
+      }
+
+      let match = false
+
+      if ($all) {
+        let intersection = null
+        let failed = false
+
+        for (const termKey of resolvedKeys) {
+          const entry = ensureTermEntry(termKey)
+          if (!entry.hasData) {
+            failed = true
+            break
+          }
+          const lines = getLineArray(entry)
+          if (lines.length === 0) {
+            failed = true
+            break
+          }
+
+          const currentSet = new Set(lines)
+          if (intersection === null) {
+            intersection = currentSet
+          } else {
+            intersection = new Set([...intersection].filter(value => currentSet.has(value)))
+            if (intersection.size === 0) {
+              failed = true
+              break
+            }
+          }
+        }
+
+        if (!failed && intersection && intersection.size > 0) {
+          match = applyExcludes(intersection, excludeKeySet)
+        }
+      } else if (excludeKeySet.length === 0) {
+        match = resolvedKeys.some(termKey => ensureTermEntry(termKey).hasData)
+      } else {
+        const candidates = new Set()
+        for (const termKey of resolvedKeys) {
+          const entry = ensureTermEntry(termKey)
+          if (!entry.hasData) continue
+          const lines = getLineArray(entry)
+          for (const line of lines) {
+            candidates.add(line)
+          }
+        }
+        if (candidates.size > 0) {
+          match = applyExcludes(candidates, excludeKeySet)
+        }
+      }
+
+      results[id] = Boolean(match)
+      if (match) {
+        successes += 1
+      }
+    }
+
+    return results
+  }
+ 
+  // Ultra-fast load with minimal conversions
+  load(index) {
+    // CRITICAL FIX: Check if index is already loaded by looking for actual data, not just empty field structures
+    if (this.index && this.index.data) {
+      let hasActualData = false
+      for (const field in this.index.data) {
+        const fieldData = this.index.data[field]
+        if (fieldData && Object.keys(fieldData).length > 0) {
+          // Check if any field has actual index entries with data
+          for (const key in fieldData) {
+            const entry = fieldData[key]
+            if (entry && ((entry.set && entry.set.size > 0) || (entry.ranges && entry.ranges.length > 0))) {
+              hasActualData = true
+              break
+            }
+          }
+          if (hasActualData) break
+        }
+      }
+      
+      if (hasActualData) {
+        if (this.opts.debugMode) {
+          console.log('🔍 IndexManager.load: Index already loaded with actual data, skipping')
+        }
+        return
+      }
+    }
+    
+    // CRITICAL FIX: Add comprehensive null/undefined validation
+    if (!index || typeof index !== 'object') {
+      if (this.opts.debugMode) {
+        console.log(`🔍 IndexManager.load: Invalid index data provided (${typeof index}), using defaults`)
+      }
+      return this._initializeDefaults()
+    }
+
+    if (!index.data || typeof index.data !== 'object') {
+      if (this.opts.debugMode) {
+        console.log(`🔍 IndexManager.load: Invalid index.data provided (${typeof index.data}), using defaults`)
+      }
+      return this._initializeDefaults()
+    }
+    
+    // CRITICAL FIX: Only log if there are actual fields to load
+    if (this.opts.debugMode && Object.keys(index.data).length > 0) {
+      console.log(`🔍 IndexManager.load: Loading index with fields: ${Object.keys(index.data).join(', ')}`)
+    }
+    
+    // Create a deep copy to avoid reference issues
+    const processedIndex = {
+      data: {}
+    }
+    
+    // CRITICAL FIX: Add null/undefined checks for field iteration
+    const fields = Object.keys(index.data)
+    for(const field of fields) {
+      if (!field || typeof field !== 'string') {
+        continue // Skip invalid field names
+      }
+
+      const fieldData = index.data[field]
+      if (!fieldData || typeof fieldData !== 'object') {
+        continue // Skip invalid field data
+      }
+
+      processedIndex.data[field] = {}
+      
+      // CRITICAL FIX: Check if this is a term mapping field for conversion
+      const isTermMappingField = this.database?.termManager && 
+        this.database.termManager.termMappingFields && 
+        this.database.termManager.termMappingFields.includes(field)
+      
+      const terms = Object.keys(fieldData)
+      for(const term of terms) {
+        if (!term || typeof term !== 'string') {
+          continue // Skip invalid term names
+        }
+
+        const termData = fieldData[term]
+        
+        // CRITICAL FIX: Convert term strings to term IDs for term mapping fields
+        // If the key is a string term (not a numeric ID), convert it to term ID
+        let termKey = term
+        if (isTermMappingField && typeof term === 'string' && !/^\d+$/.test(term)) {
+          // Key is a term string, convert to term ID
+          const termId = this.database?.termManager?.getTermIdWithoutIncrement(term)
+          if (termId !== undefined) {
+            termKey = String(termId)
+          } else {
+            // Term not found in termManager - skip this key (orphaned term from old index)
+            // This can happen if termMapping wasn't loaded yet or term was removed
+            if (this.opts?.debugMode) {
+              console.log(`⚠️  IndexManager.load: Term "${term}" not found in termManager for field "${field}" - skipping (orphaned from old index)`)
+            }
+            continue
+          }
+        }
+        
+        // Convert various formats to new hybrid format
+        if (Array.isArray(termData)) {
+              // Check if it's the new compact format [setArray, rangesArray]
+              if (termData.length === 2 && Array.isArray(termData[0]) && Array.isArray(termData[1])) {
+                // New compact format: [setArray, rangesArray]
+                // Convert ultra-compact ranges [start, count] back to {start, count}
+                const ranges = termData[1].map(range => {
+                  if (Array.isArray(range) && range.length === 2) {
+                    // Ultra-compact format: [start, count]
+                    return { start: range[0], count: range[1] }
+                  } else {
+                    // Legacy format: {start, count}
+                    return range
+                  }
+                })
+                processedIndex.data[field][termKey] = {
+                  set: new Set(termData[0]),
+                  ranges: ranges
+                }
+              } else {
+                // Legacy array format (just set data)
+                processedIndex.data[field][termKey] = { set: new Set(termData), ranges: [] }
+              }
+        } else if (termData && typeof termData === 'object') {
+          if (termData.set || termData.ranges) {
+            // Legacy hybrid format - convert set array back to Set
+            const hybridData = termData
+            let setObject
+            if (Array.isArray(hybridData.set)) {
+              // Convert array back to Set
+              setObject = new Set(hybridData.set)
+            } else {
+              // Fallback to empty Set
+              setObject = new Set()
+            }
+            processedIndex.data[field][termKey] = { 
+              set: setObject, 
+              ranges: hybridData.ranges || [] 
+            }
+          } else {
+            // Convert from Set format to hybrid
+            const numbers = Array.from(termData || [])
+            processedIndex.data[field][termKey] = { set: new Set(numbers), ranges: [] }
+          }
+        }
+      }
+    }
+    
+    // Preserve initialized fields if no data was loaded
+    if (!processedIndex.data || Object.keys(processedIndex.data).length === 0) {
+      // CRITICAL FIX: Only log if debug mode is enabled and there are actual fields
+      if (this.opts.debugMode && this.index.data && Object.keys(this.index.data).length > 0) {
+        console.log(`🔍 IndexManager.load: No data loaded, preserving initialized fields: ${Object.keys(this.index.data).join(', ')}`)
+      }
+      // Keep the current index with initialized fields
+      this.indexLoaded = true
+      this.markIndexUsed()
+      return
+    }
+    
+    // Restore totalLines from saved data
+    if (index.totalLines !== undefined) {
+      this.totalLines = index.totalLines
+      if (this.opts.debugMode) {
+        console.log(`🔍 IndexManager.load: Restored totalLines=${this.totalLines}`)
+      }
+    }
+
+    this.index = processedIndex
+    this.indexLoaded = true
+    this.markIndexUsed()
+  }
+
+  /**
+   * CRITICAL FIX: Initialize default index structure when invalid data is provided
+   * This prevents TypeError when Object.keys() is called on null/undefined
+   */
+  _initializeDefaults() {
+    if (this.opts.debugMode) {
+      console.log(`🔍 IndexManager._initializeDefaults: Initializing default index structure`)
+    }
+    
+    // Initialize empty index structure
+    this.index = { data: {} }
+    
+    // Initialize fields from options if available
+    if (this.opts.indexes && typeof this.opts.indexes === 'object') {
+      const fields = Object.keys(this.opts.indexes)
+      for (const field of fields) {
+        if (field && typeof field === 'string') {
+          this.index.data[field] = {}
+        }
+      }
+    }
+
+    this.indexLoaded = true
+    this.markIndexUsed()
+  }
+
+  /**
+   * Convert index to JSON-serializable format for debugging and export
+   * This resolves the issue where Sets appear as empty objects in JSON.stringify
+   */
+  toJSON() {
+    const serializable = {
+      data: {},
+      totalLines: this.totalLines
+    }
+    
+    // Check if this is a term mapping field for conversion
+    const isTermMappingField = (field) => {
+      return this.database?.termManager && 
+        this.database.termManager.termMappingFields && 
+        this.database.termManager.termMappingFields.includes(field)
+    }
+    
+    for (const field in this.index.data) {
+      serializable.data[field] = {}
+      const isTermField = isTermMappingField(field)
+      
+      for (const term in this.index.data[field]) {
+        const hybridData = this.index.data[field][term]
+        
+        // CRITICAL FIX: Convert term strings to term IDs for term mapping fields
+        // If the key is a string term (not a numeric ID), convert it to term ID
+        let termKey = term
+        if (isTermField && typeof term === 'string' && !/^\d+$/.test(term)) {
+          // Key is a term string, convert to term ID
+          const termId = this.database?.termManager?.getTermIdWithoutIncrement(term)
+          if (termId !== undefined) {
+            termKey = String(termId)
+          } else {
+            // Term not found in termManager, keep original key
+            // This prevents data loss when term mapping is incomplete
+            termKey = term
+            if (this.opts?.debugMode) {
+              console.log(`⚠️  IndexManager.toJSON: Term "${term}" not found in termManager for field "${field}" - using original key`)
+            }
+          }
+        }
+        
+        // OPTIMIZATION: Create ranges before serialization if beneficial
+        if (hybridData.set && hybridData.set.size >= this.rangeThreshold) {
+          this._optimizeToRanges(hybridData)
+        }
+        
+        // Convert hybrid structure to serializable format
+        let setArray = []
+        if (hybridData.set) {
+          if (typeof hybridData.set.size !== 'undefined') {
+            // Regular Set
+            setArray = Array.from(hybridData.set)
+          }
+        }
+        
+        // Use ultra-compact format: [setArray, rangesArray] to save space
+        const ranges = hybridData.ranges || []
+        if (ranges.length > 0) {
+          // Convert ranges to ultra-compact format: [start, count] instead of {start, count}
+          const compactRanges = ranges.map(range => [range.start, range.count])
+          serializable.data[field][termKey] = [setArray, compactRanges]
+        } else {
+          // CRITICAL FIX: Always use the [setArray, []] format for consistency
+          // This ensures the load() method can properly deserialize the data
+          serializable.data[field][termKey] = [setArray, []]
+        }
+      }
+    }
+    
+    return serializable
+  }
+
+  /**
+   * Get a JSON string representation of the index
+   * This properly handles Sets unlike the default JSON.stringify
+   */
+  toString() {
+    return JSON.stringify(this.toJSON(), null, 2)
+  }
+
+  // Simplified term mapping methods - just basic functionality
+  
+  /**
+   * Rebuild index (stub for compatibility)
+   */
+  async rebuild() {
+    // Stub implementation for compatibility
+    return Promise.resolve()
+  }
+}
+