jexidb 2.1.5 → 2.1.7

package/src/Database.mjs

@@ -4,6 +4,7 @@ import Serializer from './Serializer.mjs'
 import { Mutex } from 'async-mutex'
 import fs from 'fs'
 import readline from 'readline'
+import pRetry from 'p-retry'
 import { OperationQueue } from './OperationQueue.mjs'
 
 /**
@@ -2332,9 +2333,19 @@ class Database extends EventEmitter {
 
     // CRITICAL FIX: Validate state before find operation
     this.validateState()
-    
+
+    // Check for index-only optimization
+    if (options.indexOnly && this._canUseIndexOnlyForExists(criteria)) {
+      if (this.opts.debugMode) {
+        console.log(`⚡ find() using INDEX-ONLY optimization for: ${JSON.stringify(criteria)}`)
+      }
+      // Return records using index-only lookup (but we need to fetch actual records)
+      // For now, this just ensures we use optimized path in QueryManager
+      options._forceIndexOnly = true
+    }
+
     // OPTIMIZATION: Find searches writeBuffer directly
-    
+
     const startTime = Date.now()
     
     if (this.opts.debugMode) {
@@ -2656,6 +2667,8 @@ class Database extends EventEmitter {
     return Array.from(fields)
   }
 
+
+
   /**
    * Update records matching criteria
    */
@@ -3088,121 +3101,184 @@ class Database extends EventEmitter {
       let count = 0
       const startTime = Date.now()
       
-      // Auto-detect schema from first line if not initialized
-      if (!this.serializer.schemaManager.isInitialized) {
-        const fs = await import('fs')
-        const readline = await import('readline')
-        const stream = fs.createReadStream(this.fileHandler.file, {
-          highWaterMark: 64 * 1024,
-          encoding: 'utf8'
-        })
-        const rl = readline.createInterface({
-          input: stream,
-          crlfDelay: Infinity
-        })
-        
-        for await (const line of rl) {
-          if (line && line.trim()) {
-            try {
-              const firstRecord = JSON.parse(line)
-              if (Array.isArray(firstRecord)) {
-                // Try to infer schema from opts.fields if available
-                if (this.opts.fields && typeof this.opts.fields === 'object') {
-                  const fieldNames = Object.keys(this.opts.fields)
-                  if (fieldNames.length >= firstRecord.length) {
-                    // Use first N fields from opts.fields to match array length
-                    const schema = fieldNames.slice(0, firstRecord.length)
-                    this.serializer.initializeSchema(schema)
-                    if (this.opts.debugMode) {
-                      console.log(`🔍 Inferred schema from opts.fields: ${schema.join(', ')}`)
+      // Use retry for the streaming rebuild only if timeout is configured
+      if (this.opts.ioTimeoutMs && this.opts.ioTimeoutMs > 0) {
+        count = await this._rebuildIndexesWithRetry()
+      } else {
+        // Use original logic without retry for backward compatibility
+        count = await this._rebuildIndexesOriginal()
+      }
+      
+      // Update indexManager totalLines
+      if (this.indexManager) {
+        this.indexManager.setTotalLines(this.offsets.length)
+      }
+      
+      this._indexRebuildNeeded = false
+      
+      if (this.opts.debugMode) {
+        console.log(`✅ Index rebuilt from ${count} records in ${Date.now() - startTime}ms`)
+      }
+      
+      // Save the rebuilt index
+      await this._saveIndexDataToFile()
+    } catch (error) {
+      if (this.opts.debugMode) {
+        console.error('❌ Failed to rebuild indexes:', error.message)
+      }
+      // Don't throw - queries will fall back to streaming
+    }
+  }
+
+  /**
+   * Rebuild indexes with retry logic to handle I/O hangs
+   * @private
+   */
+  async _rebuildIndexesWithRetry() {
+    // If no timeout configured, use original implementation without retry
+    if (!this.opts.ioTimeoutMs) {
+      return this._rebuildIndexesOriginal();
+    }
+
+    const timeoutMs = this.opts.ioTimeoutMs || 10000; // Longer timeout for rebuild
+    const maxRetries = this.opts.maxRetries || 3;
+
+    let count = 0;
+
+    await pRetry(async (attempt) => {
+      const controller = new AbortController();
+      const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+      try {
+        // Auto-detect schema from first line if not initialized
+        if (!this.serializer.schemaManager.isInitialized) {
+          const stream = fs.createReadStream(this.fileHandler.file, {
+            highWaterMark: 64 * 1024,
+            encoding: 'utf8'
+          })
+          const rl = readline.createInterface({
+            input: stream,
+            crlfDelay: Infinity
+          })
+
+          // Handle abort
+          controller.signal.addEventListener('abort', () => {
+            stream.destroy(new Error('AbortError'));
+            rl.close();
+          });
+
+          for await (const line of rl) {
+            if (controller.signal.aborted) break;
+            if (line && line.trim()) {
+              try {
+                const firstRecord = JSON.parse(line)
+                if (Array.isArray(firstRecord)) {
+                  // Try to infer schema from opts.fields if available
+                  if (this.opts.fields && typeof this.opts.fields === 'object') {
+                    const fieldNames = Object.keys(this.opts.fields)
+                    if (fieldNames.length >= firstRecord.length) {
+                      // Use first N fields from opts.fields to match array length
+                      const schema = fieldNames.slice(0, firstRecord.length)
+                      this.serializer.initializeSchema(schema)
+                      if (this.opts.debugMode) {
+                        console.log(`🔍 Inferred schema from opts.fields: ${schema.join(', ')}`)
+                      }
+                    } else {
+                      throw new Error(`Cannot rebuild index: array has ${firstRecord.length} elements but opts.fields only defines ${fieldNames.length} fields. Schema must be explicitly provided.`)
                     }
                   } else {
-                    throw new Error(`Cannot rebuild index: array has ${firstRecord.length} elements but opts.fields only defines ${fieldNames.length} fields. Schema must be explicitly provided.`)
+                    throw new Error('Cannot rebuild index: schema missing, file uses array format, and opts.fields not provided. The .idx.jdb file is corrupted.')
                   }
                 } else {
-                  throw new Error('Cannot rebuild index: schema missing, file uses array format, and opts.fields not provided. The .idx.jdb file is corrupted.')
+                  // Object format, initialize from object keys
+                  this.serializer.initializeSchema(firstRecord, true)
+                  if (this.opts.debugMode) {
+                    console.log(`🔍 Auto-detected schema from object: ${Object.keys(firstRecord).join(', ')}`)
+                  }
                 }
-              } else {
-                // Object format, initialize from object keys
-                this.serializer.initializeSchema(firstRecord, true)
+                break
+              } catch (error) {
                 if (this.opts.debugMode) {
-                  console.log(`🔍 Auto-detected schema from object: ${Object.keys(firstRecord).join(', ')}`)
+                  console.error('❌ Failed to auto-detect schema:', error.message)
                 }
+                throw error
               }
-              break
-            } catch (error) {
-              if (this.opts.debugMode) {
-                console.error('❌ Failed to auto-detect schema:', error.message)
-              }
-              throw error
             }
           }
+          stream.destroy()
         }
-        stream.destroy()
-      }
-      
-      // Use streaming to read records without loading everything into memory
-      // Also rebuild offsets while we're at it
-      const fs = await import('fs')
-      const readline = await import('readline')
-      
-      this.offsets = []
-      let currentOffset = 0
-      
-      const stream = fs.createReadStream(this.fileHandler.file, {
-        highWaterMark: 64 * 1024,
-        encoding: 'utf8'
-      })
-      
-      const rl = readline.createInterface({
-        input: stream,
-        crlfDelay: Infinity
-      })
-      
-      try {
+
+        // Use streaming to read records without loading everything into memory
+        // Also rebuild offsets while we're at it
+
+        this.offsets = []
+        let currentOffset = 0
+
+        const stream = fs.createReadStream(this.fileHandler.file, {
+          highWaterMark: 64 * 1024,
+          encoding: 'utf8'
+        })
+
+        const rl = readline.createInterface({
+          input: stream,
+          crlfDelay: Infinity
+        })
+
+        // Handle abort
+        controller.signal.addEventListener('abort', () => {
+          stream.destroy(new Error('AbortError'));
+          rl.close();
+        });
+
+        let localCount = 0;
         for await (const line of rl) {
+          if (controller.signal.aborted) break;
           if (line && line.trim()) {
             try {
               // Record the offset for this line
               this.offsets.push(currentOffset)
-              
+
               const record = this.serializer.deserialize(line)
               const recordWithTerms = this.restoreTermIdsAfterDeserialization(record)
-              await this.indexManager.add(recordWithTerms, count)
-              count++
+              await this.indexManager.add(recordWithTerms, count + localCount)
+              localCount++
             } catch (error) {
               // Skip invalid lines
               if (this.opts.debugMode) {
-                console.log(`⚠️ Rebuild: Failed to deserialize line ${count}:`, error.message)
+                console.log(`⚠️ Rebuild: Failed to deserialize line ${count + localCount}:`, error.message)
               }
             }
           }
           // Update offset for next line (including newline character)
           currentOffset += Buffer.byteLength(line, 'utf8') + 1
         }
-      } finally {
+
+        count += localCount;
         stream.destroy()
+      } catch (error) {
+        if (error.name === 'AbortError' || error.code === 'ETIMEDOUT') {
+          if (this.opts.debugMode) {
+            console.log(`⚠️ Index rebuild attempt ${attempt} timed out, retrying...`);
+          }
+          throw error; // p-retry will retry
+        }
+        // For other errors, don't retry
+        throw new pRetry.AbortError(error);
+      } finally {
+        clearTimeout(timeout);
       }
-      
-      // Update indexManager totalLines
-      if (this.indexManager) {
-        this.indexManager.setTotalLines(this.offsets.length)
-      }
-      
-      this._indexRebuildNeeded = false
-      
-      if (this.opts.debugMode) {
-        console.log(`✅ Index rebuilt from ${count} records in ${Date.now() - startTime}ms`)
-      }
-      
-      // Save the rebuilt index
-      await this._saveIndexDataToFile()
-    } catch (error) {
-      if (this.opts.debugMode) {
-        console.error('❌ Failed to rebuild indexes:', error.message)
+    }, {
+      retries: maxRetries,
+      minTimeout: 200,
+      maxTimeout: 2000,
+      onFailedAttempt: (error) => {
+        if (this.opts.debugMode) {
+          console.log(`Index rebuild failed (attempt ${error.attemptNumber}), ${error.retriesLeft} retries left`);
+        }
       }
-      // Don't throw - queries will fall back to streaming
-    }
+    });
+
+    return count;
   }
 
   /**
@@ -3404,35 +3480,258 @@ class Database extends EventEmitter {
   }
 
   /**
-   * Check if any records exist for given field and terms (index-only, ultra-fast)
-   * Delegates to IndexManager.exists() for maximum performance
-   * 
-   * @param {string} fieldName - Indexed field name
-   * @param {string|Array<string>} terms - Single term or array of terms
+   * Check if any records exist matching the given criteria (ultra-fast when using indexed fields)
+   *
+   * @param {string|object} fieldName - Indexed field name (legacy) OR query criteria object (new)
+   * @param {string|Array<string>} terms - Single term or array of terms (when using legacy syntax)
    * @param {Object} options - Options: { $all: true/false, caseInsensitive: true/false, excludes: Array<string> }
    * @returns {Promise<boolean>} - True if at least one match exists
-   * 
+   *
    * @example
-   * // Check if channel exists
-   * const exists = await db.exists('nameTerms', ['a', 'e'], { $all: true });
-   * 
+   * // Legacy syntax - ultra-fast index-only check
+   * const exists = await db.exists('nameTerms', 'tv');
+   * const existsAll = await db.exists('nameTerms', ['tv', 'globo'], { $all: true });
+   *
    * @example
-   * // Check if 'tv' exists but not 'globo'
-   * const exists = await db.exists('nameTerms', 'tv', { excludes: ['globo'] });
+   * // New syntax - full query criteria support
+   * const exists = await db.exists({ mediaType: 'live', status: 'active' });
+   * const existsOr = await db.exists({ mediaType: ['live', 'vod'] });
    */
-  async exists(fieldName, terms, options = {}) {
+  async exists(fieldNameOrCriteria, terms, options = {}) {
     this._validateInitialization('exists')
-    return this.indexManager.exists(fieldName, terms, options)
+
+    // Detect syntax: new criteria object vs legacy field/terms
+    if (typeof fieldNameOrCriteria === 'object' && fieldNameOrCriteria !== null && !Array.isArray(fieldNameOrCriteria)) {
+      // New syntax: exists(criteria)
+      const criteria = fieldNameOrCriteria
+      return this._existsWithCriteria(criteria)
+    } else if (typeof fieldNameOrCriteria === 'string' || fieldNameOrCriteria === null || Array.isArray(fieldNameOrCriteria)) {
+      // Legacy syntax: exists(fieldName, terms, options)
+      // Also handle invalid inputs (null, array) for backward compatibility
+      const fieldName = fieldNameOrCriteria
+      return this.indexManager.exists(fieldName, terms, options)
+    } else {
+      // Invalid input type
+      throw new Error('First parameter must be a string (fieldName) or object (criteria)')
+    }
+  }
+
+  /**
+   * Check if any records exist using full query criteria
+   * Uses index intersection when possible for maximum performance
+   * @private
+   * @param {object} criteria - Query criteria object
+   * @returns {Promise<boolean>} - True if at least one match exists
+   */
+
+  async _existsWithCriteria(criteria) {
+    if (criteria === null || criteria === undefined || typeof criteria !== 'object' || Array.isArray(criteria)) {
+      throw new Error('Criteria must be a non-null object')
+    }
+
+    // Check if criteria is empty (should match all records)
+    const criteriaFields = Object.keys(criteria)
+    if (criteriaFields.length === 0) {
+      // Empty criteria matches all records - check if any exist
+      try {
+        const result = await this.find({}, { limit: 1 })
+        return result.length > 0
+      } catch (error) {
+        return false
+      }
+    }
+
+    // 🚀 OPTIMIZATION: Try index-only existence check for simple criteria
+    if (this._canUseIndexOnlyForExists(criteria)) {
+      if (this.opts.debugMode) {
+        console.log(`⚡ exists() using INDEX-ONLY optimization for: ${JSON.stringify(criteria)}`)
+      }
+      return this._existsIndexOnly(criteria)
+    }
+
+    // 🎯 FALLBACK: Use the same find() logic for complex criteria or non-indexed fields
+    // This ensures exists() uses identical logic to find() for all criteria processing
+    try {
+      const result = await this.find(criteria, { limit: 1 })
+      return result.length > 0
+    } catch (error) {
+      // If find() fails (e.g., strict mode violations), no records exist
+      return false
+    }
+  }
+
+  /**
+   * Check if criteria can use index-only existence check
+   * @private
+   * @param {object} criteria - Query criteria
+   * @returns {boolean} - True if can use index-only
+   */
+  _canUseIndexOnlyForExists(criteria) {
+    // Must have indexes configured
+    if (!this.opts.indexes) return false
+
+    // All fields in criteria must be indexed
+    const criteriaFields = Object.keys(criteria)
+    const allFieldsIndexed = criteriaFields.every(field => this.opts.indexes[field])
+
+    if (!allFieldsIndexed) return false
+
+    // No complex operators allowed (only simple equality)
+    for (const [field, value] of Object.entries(criteria)) {
+      // Allow simple values (string, number, boolean)
+      if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
+        continue
+      }
+
+      // Allow arrays (OR logic)
+      if (Array.isArray(value)) {
+        continue
+      }
+
+      // Reject any object (complex operators)
+      if (typeof value === 'object') {
+        return false
+      }
+    }
+
+    return true
+  }
+
+  /**
+   * Perform index-only existence check (ultra-fast, no disk I/O)
+   * @private
+   * @param {object} criteria - Simple criteria with only indexed fields
+   * @returns {boolean} - True if any records match the criteria
+   */
+  _existsIndexOnly(criteria) {
+    const criteriaEntries = Object.entries(criteria)
+
+    // For single field criteria, use direct indexManager.exists()
+    if (criteriaEntries.length === 1) {
+      const [field, value] = criteriaEntries[0]
+      return this.indexManager.exists(field, value)
+    }
+
+    // For multiple field criteria, implement index intersection (AND logic)
+    let intersection = null
+
+    for (const [field, value] of criteriaEntries) {
+      // Get line numbers for this field/value combination
+      const fieldLines = this._getIndexLinesForFieldValue(field, value)
+
+      if (intersection === null) {
+        // First field - start with all its lines
+        intersection = new Set(fieldLines)
+      } else {
+        // Intersect with previous results
+        const currentLines = new Set(fieldLines)
+        for (const line of Array.from(intersection)) {
+          if (!currentLines.has(line)) {
+            intersection.delete(line)
+          }
+        }
+      }
+
+      // Early exit if no matches possible
+      if (intersection.size === 0) {
+        return false
+      }
+    }
+
+    return intersection.size > 0
+  }
+
+  /**
+   * Get line numbers for a field/value combination using index-only lookup
+   * @private
+   * @param {string} field - Field name
+   * @param {*} value - Field value (string, number, or array)
+   * @returns {Array<number>} Array of line numbers
+   */
+  _getIndexLinesForFieldValue(field, value) {
+    // For arrays (OR logic), check each value
+    if (Array.isArray(value)) {
+      const allLines = new Set()
+      for (const singleValue of value) {
+        const lines = this._getSingleFieldLines(field, singleValue)
+        lines.forEach(line => allLines.add(line))
+      }
+      return Array.from(allLines)
+    }
+
+    // For single values
+    return this._getSingleFieldLines(field, value)
+  }
+
+  /**
+   * Get line numbers for a single field/value using index lookup
+   * @private
+   * @param {string} field - Field name
+   * @param {*} value - Single field value
+   * @returns {Array<number>} Array of line numbers
+   */
+  _getSingleFieldLines(field, value) {
+    // Use indexManager.exists() logic but return line numbers instead of boolean
+    const fieldIndex = this.indexManager?.index?.data?.[field]
+    if (!fieldIndex) return []
+
+    const fieldType = this.opts.indexes[field]
+    const isTermMapped = this.termManager &&
+      this.termManager.termMappingFields &&
+      this.termManager.termMappingFields.includes(field)
+
+    let searchKey
+
+    if (fieldType === 'array:string') {
+      // For array:string fields, match records that contain this value
+      searchKey = isTermMapped ?
+        this.termManager.getTermIdWithoutIncrement(String(value)) :
+        String(value)
+    } else {
+      // For simple fields, exact match
+      searchKey = isTermMapped ?
+        this.termManager.getTermIdWithoutIncrement(String(value)) :
+        String(value)
+    }
+
+    if (searchKey === null || searchKey === undefined) {
+      return []
+    }
+
+    const termData = fieldIndex[searchKey]
+    if (!termData) return []
+
+    // Extract all line numbers from termData (similar to indexManager._getAllLineNumbers)
+    const lines = new Set()
+
+    // Check Set (most common case)
+    if (termData.set && termData.set.size > 0) {
+      for (const line of termData.set) {
+        lines.add(line)
+      }
+    }
+
+    // Check ranges
+    if (termData.ranges && termData.ranges.length > 0) {
+      for (const range of termData.ranges) {
+        for (let line = range.start; line <= range.end; line++) {
+          lines.add(line)
+        }
+      }
+    }
+
+    return Array.from(lines)
   }
 
   /**
    * Calculate coverage for grouped include/exclude term sets
    * @param {string} fieldName - Name of the indexed field
    * @param {Array<object>} groups - Array of { terms, excludes } objects
+   * @param {object} filterCriteria - Optional filter criteria
    * @param {object} options - Optional settings
    * @returns {Promise<number>} Coverage percentage between 0 and 100
    */
-  async coverage(fieldName, groups, options = {}) {
+  async coverage(fieldName, groups, filterCriteria = null, options = {}) {
     this._validateInitialization('coverage')
 
     if (typeof fieldName !== 'string' || !fieldName.trim()) {
@@ -3462,6 +3761,30 @@ class Database extends EventEmitter {
       return 0
     }
 
+    // Validate filter criteria
+    let filteredLines = null
+    if (filterCriteria && typeof filterCriteria === 'object') {
+      if (Array.isArray(filterCriteria)) {
+        throw new Error('filterCriteria must be an object, not an array')
+      }
+
+      // Get filtered records using QueryManager for consistency
+      try {
+        const filteredRecords = await this.queryManager.find(filterCriteria, {
+          limit: null, // Get all matching records for coverage calculation
+          indexedQueryMode: this.opts.indexedQueryMode,
+          allowNonIndexed: true
+        })
+        filteredLines = new Set(filteredRecords.map(record => record._))
+        if (filteredLines.size === 0) {
+          return 0 // No records match the filter
+        }
+      } catch (error) {
+        // If filtering fails, return 0 (no coverage possible)
+        return 0
+      }
+    }
+
     const isTermMapped = this.termManager &&
       this.termManager.termMappingFields &&
       this.termManager.termMappingFields.includes(fieldName)
@@ -3532,10 +3855,20 @@ class Database extends EventEmitter {
           break
         }
 
+        // Apply filter if specified
+        let validLineNumbers = lineNumbers
+        if (filteredLines) {
+          validLineNumbers = lineNumbers.filter(line => filteredLines.has(line))
+          if (validLineNumbers.length === 0) {
+            groupMatched = false
+            break
+          }
+        }
+
         if (candidateLines === null) {
-          candidateLines = new Set(lineNumbers)
+          candidateLines = new Set(validLineNumbers)
         } else {
-          const termSet = new Set(lineNumbers)
+          const termSet = new Set(validLineNumbers)
           for (const line of Array.from(candidateLines)) {
             if (!termSet.has(line)) {
               candidateLines.delete(line)
@@ -3569,7 +3902,13 @@ class Database extends EventEmitter {
           continue
         }
 
-        for (const line of excludeLines) {
+        // Apply filter to exclude lines if specified
+        let validExcludeLines = excludeLines
+        if (filteredLines) {
+          validExcludeLines = excludeLines.filter(line => filteredLines.has(line))
+        }
+
+        for (const line of validExcludeLines) {
           if (!candidateLines.size) {
             break
           }
@@ -3793,37 +4132,12 @@ class Database extends EventEmitter {
         
         const groupedRanges = await this.fileHandler.groupedRanges(ranges)
         
-        const fs = await import('fs')
         const fd = await fs.promises.open(this.fileHandler.file, 'r')
         
         try {
           for (const groupedRange of groupedRanges) {
-            for await (const row of this.fileHandler.readGroupedRange(groupedRange, fd)) {
-              try {
-                const record = this.serializer.deserialize(row.line)
-                
-                // Get line number from the row, fallback to start offset mapping
-                let lineNumber = row._ !== null && row._ !== undefined ? row._ : (startToLineNumber.get(row.start) ?? 0)
-                
-                // Restore term IDs to terms
-                const recordWithTerms = this.restoreTermIdsAfterDeserialization(record)
-                
-                // Add line number
-                recordWithTerms._ = lineNumber
-                
-                // Add score if includeScore is true (default is true)
-                if (opts.includeScore !== false) {
-                  recordWithTerms.score = scoresByLineNumber.get(lineNumber) || 0
-                }
-                
-                results.push(recordWithTerms)
-              } catch (error) {
-                // Skip invalid lines
-                if (this.opts.debugMode) {
-                  console.error('Error deserializing record in score():', error)
-                }
-              }
-            }
+            const rangeResults = await this._readGroupedRangeWithRetry(groupedRange, fd, startToLineNumber, scoresByLineNumber, opts);
+            results.push(...rangeResults);
           }
         } finally {
           await fd.close()
@@ -3865,6 +4179,234 @@ class Database extends EventEmitter {
     return results
   }
 
+  /**
+   * Read a grouped range with retry logic to handle I/O hangs
+   * @private
+   */
+  async _readGroupedRangeWithRetry(groupedRange, fd, startToLineNumber, scoresByLineNumber, opts) {
+    // If no timeout configured, use original implementation without retry
+    if (!this.opts.ioTimeoutMs) {
+      return this._readGroupedRangeOriginal(groupedRange, fd, startToLineNumber, scoresByLineNumber, opts);
+    }
+
+    const timeoutMs = this.opts.ioTimeoutMs || 3000; // Shorter timeout for range reads
+    const maxRetries = this.opts.maxRetries || 3;
+
+    const results = [];
+
+    await pRetry(async (attempt) => {
+      const controller = new AbortController();
+      const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+      try {
+        // Collect results from the generator
+        const rangeResults = [];
+        const generator = this.fileHandler.readGroupedRange(groupedRange, fd);
+
+        // Handle abort
+        controller.signal.addEventListener('abort', () => {
+          generator.return(); // Close the generator
+        });
+
+        for await (const row of generator) {
+          if (controller.signal.aborted) break;
+
+          try {
+            const record = this.serializer.deserialize(row.line)
+
+            // Get line number from the row, fallback to start offset mapping
+            let lineNumber = row._ !== null && row._ !== undefined ? row._ : (startToLineNumber.get(row.start) ?? 0)
+
+            // Restore term IDs to terms
+            const recordWithTerms = this.restoreTermIdsAfterDeserialization(record)
+
+            // Add line number
+            recordWithTerms._ = lineNumber
+
+            // Add score if includeScore is true (default is true)
+            if (opts.includeScore !== false) {
+              recordWithTerms.score = scoresByLineNumber.get(lineNumber) || 0
+            }
+
+            rangeResults.push(recordWithTerms)
+          } catch (error) {
+            // Skip invalid lines
+            if (this.opts.debugMode) {
+              console.error('Error deserializing record in score():', error)
+            }
+          }
+        }
+
+        results.push(...rangeResults);
+      } catch (error) {
+        if (error.name === 'AbortError' || error.code === 'ETIMEDOUT') {
+          if (this.opts.debugMode) {
+            console.log(`⚠️ Score range read attempt ${attempt} timed out, retrying...`);
+          }
+          throw error; // p-retry will retry
+        }
+        // For other errors, don't retry
+        throw new pRetry.AbortError(error);
+      } finally {
+        clearTimeout(timeout);
+      }
+    }, {
+      retries: maxRetries,
+      minTimeout: 100,
+      maxTimeout: 500,
+      onFailedAttempt: (error) => {
+        if (this.opts.debugMode) {
+          console.log(`Score range read failed (attempt ${error.attemptNumber}), ${error.retriesLeft} retries left`);
+        }
+      }
+    });
+
+    return results;
+  }
+
+  /**
+   * Original read grouped range logic without retry (for backward compatibility)
+   * @private
+   */
+  async _readGroupedRangeOriginal(groupedRange, fd, startToLineNumber, scoresByLineNumber, opts) {
+    const results = [];
+
+    // Collect results from the generator
+    const rangeResults = [];
+    const generator = this.fileHandler.readGroupedRange(groupedRange, fd);
+
+    for await (const row of generator) {
+      try {
+        const record = this.serializer.deserialize(row.line)
+
+        // Get line number from the row, fallback to start offset mapping
+        let lineNumber = row._ !== null && row._ !== undefined ? row._ : (startToLineNumber.get(row.start) ?? 0)
+
+        // Restore term IDs to terms
+        const recordWithTerms = this.restoreTermIdsAfterDeserialization(record)
+
+        // Add line number
+        recordWithTerms._ = lineNumber
+
+        // Add score if includeScore is true (default is true)
+        if (opts.includeScore !== false) {
+          recordWithTerms.score = scoresByLineNumber.get(lineNumber) || 0
+        }
+
+        rangeResults.push(recordWithTerms)
+      } catch (error) {
+        // Skip invalid lines
+        if (this.opts.debugMode) {
+          console.error('Error deserializing record in score():', error)
+        }
+      }
+    }
+
+    results.push(...rangeResults);
+    return results;
+  }
+
+  /**
+   * Original rebuild indexes logic without retry (for backward compatibility)
+   * @private
+   */
+  async _rebuildIndexesOriginal() {
+    let count = 0;
+
+    // Auto-detect schema from first line if not initialized
+    if (!this.serializer.schemaManager.isInitialized) {
+      const stream = fs.createReadStream(this.fileHandler.file, {
+        highWaterMark: 64 * 1024,
+        encoding: 'utf8'
+      })
+      const rl = readline.createInterface({
+        input: stream,
+        crlfDelay: Infinity
+      })
+
+      for await (const line of rl) {
+        if (line && line.trim()) {
+          try {
+            const firstRecord = JSON.parse(line)
+            if (Array.isArray(firstRecord)) {
+              // Try to infer schema from opts.fields if available
+              if (this.opts.fields && typeof this.opts.fields === 'object') {
+                const fieldNames = Object.keys(this.opts.fields)
+                if (fieldNames.length >= firstRecord.length) {
+                  // Use first N fields from opts.fields to match array length
+                  const schema = fieldNames.slice(0, firstRecord.length)
+                  this.serializer.initializeSchema(schema)
+                  if (this.opts.debugMode) {
+                    console.log(`🔍 Inferred schema from opts.fields: ${schema.join(', ')}`)
+                  }
+                } else {
+                  throw new Error(`Cannot rebuild index: array has ${firstRecord.length} elements but opts.fields only defines ${fieldNames.length} fields. Schema must be explicitly provided.`)
+                }
+              } else {
+                throw new Error('Cannot rebuild index: schema missing, file uses array format, and opts.fields not provided. The .idx.jdb file is corrupted.')
+              }
+            } else {
+              // Object format, initialize from object keys
+              this.serializer.initializeSchema(firstRecord, true)
+              if (this.opts.debugMode) {
+                console.log(`🔍 Auto-detected schema from object: ${Object.keys(firstRecord).join(', ')}`)
+              }
+            }
+            break
+          } catch (error) {
+            if (this.opts.debugMode) {
+              console.error('❌ Failed to auto-detect schema:', error.message)
+            }
+            throw error
+          }
+        }
+      }
+      stream.destroy()
+    }
+
+    // Use streaming to read records without loading everything into memory
+    // Also rebuild offsets while we're at it
+    this.offsets = []
+    let currentOffset = 0
+
+    const stream = fs.createReadStream(this.fileHandler.file, {
+      highWaterMark: 64 * 1024,
+      encoding: 'utf8'
+    })
+
+    const rl = readline.createInterface({
+      input: stream,
+      crlfDelay: Infinity
+    })
+
+    try {
+      for await (const line of rl) {
+        if (line && line.trim()) {
+          try {
+            // Record the offset for this line
+            this.offsets.push(currentOffset)
+
+            const record = this.serializer.deserialize(line)
+            const recordWithTerms = this.restoreTermIdsAfterDeserialization(record)
+            await this.indexManager.add(recordWithTerms, count)
+            count++
+          } catch (error) {
+            // Skip invalid lines
+            if (this.opts.debugMode) {
+              console.log(`⚠️ Rebuild: Failed to deserialize line ${count}:`, error.message)
+            }
+          }
+        }
+        // Update offset for next line (including newline character)
+        currentOffset += Buffer.byteLength(line, 'utf8') + 1
+      }
+    } finally {
+      stream.destroy()
+    }
+
+    return count;
+  }
+
   /**
    * Wait for all pending operations to complete
    */
@@ -4187,7 +4729,6 @@ class Database extends EventEmitter {
       
       // Method 1: Try to read the entire file and filter
       if (this.fileHandler.exists()) {
-        const fs = await import('fs')
         const fileContent = await fs.promises.readFile(this.normalizedFile, 'utf8')
         const lines = fileContent.split('\n').filter(line => line.trim())
         
@@ -4894,11 +5435,10 @@ class Database extends EventEmitter {
 
     this._offsetRecoveryInProgress = true
 
-    const fsModule = this._fsModule || (this._fsModule = await import('fs'))
     let fd
 
     try {
-      fd = await fsModule.promises.open(this.fileHandler.file, 'r')
+      fd = await fs.promises.open(this.fileHandler.file, 'r')
     } catch (error) {
       this._offsetRecoveryInProgress = false
       if (this.opts.debugMode) {
@@ -5224,7 +5764,6 @@ class Database extends EventEmitter {
           const ranges = this.getRanges(map)
           const groupedRanges = await this.fileHandler.groupedRanges(ranges)
           
-          const fs = await import('fs')
           const fd = await fs.promises.open(this.fileHandler.file, 'r')
           
           try {
@@ -5770,7 +6309,6 @@ class Database extends EventEmitter {
         // If the .idx.jdb file exists and has data, and we're trying to save empty index,
         // skip the save to prevent corruption
         if (isEmpty && !this.offsets?.length) {
-          const fs = await import('fs')
           if (fs.existsSync(idxPath)) {
             try {
               const existingData = JSON.parse(await fs.promises.readFile(idxPath, 'utf8'))