jexidb 2.1.0 → 2.1.1

package/src/Database.mjs

@@ -76,12 +76,16 @@ class InsertSession {
   constructor(database, sessionOptions = {}) {
     this.database = database
     this.batchSize = sessionOptions.batchSize || 100
+    this.enableAutoSave = sessionOptions.enableAutoSave !== undefined ? sessionOptions.enableAutoSave : true
     this.totalInserted = 0
     this.flushing = false
     this.batches = [] // Array of batches to avoid slice() in flush()
     this.currentBatch = [] // Current batch being filled
     this.sessionId = Math.random().toString(36).substr(2, 9)
     
+    // Track pending auto-flush operations
+    this.pendingAutoFlushes = new Set()
+    
     // Register this session as active
     this.database.activeInsertSessions.add(this)
   }
@@ -103,46 +107,153 @@ class InsertSession {
     this.currentBatch.push(finalRecord)
     this.totalInserted++
     
-    // If batch is full, move it to batches array
+    // If batch is full, move it to batches array and trigger auto-flush
     if (this.currentBatch.length >= this.batchSize) {
       this.batches.push(this.currentBatch)
       this.currentBatch = []
+      
+      // Auto-flush in background (non-blocking)
+      // This ensures batches are flushed automatically without blocking add()
+      this.autoFlush().catch(err => {
+        // Log error but don't throw - we don't want to break the add() flow
+        console.error('Auto-flush error in InsertSession:', err)
+      })
     }
     
     return finalRecord
   }
 
-  async flush() {
-    // Check if there's anything to flush
-    if (this.batches.length === 0 && this.currentBatch.length === 0) return
-
-    // Prevent concurrent flushes
+  async autoFlush() {
+    // Only flush if not already flushing
+    // This method will process all pending batches
     if (this.flushing) return
+    
+    // Create a promise for this auto-flush operation
+    const flushPromise = this._doFlush()
+    this.pendingAutoFlushes.add(flushPromise)
+    
+    // Remove from pending set when complete (success or error)
+    flushPromise
+      .then(() => {
+        this.pendingAutoFlushes.delete(flushPromise)
+      })
+      .catch((err) => {
+        this.pendingAutoFlushes.delete(flushPromise)
+        throw err
+      })
+    
+    return flushPromise
+  }
+
+  async _doFlush() {
+    // Check if database is destroyed or closed before starting
+    if (this.database.destroyed || this.database.closed) {
+      // Clear batches if database is closed/destroyed
+      this.batches = []
+      this.currentBatch = []
+      return
+    }
+
+    // Prevent concurrent flushes - if already flushing, wait for it
+    if (this.flushing) {
+      // Wait for the current flush to complete
+      while (this.flushing) {
+        await new Promise(resolve => setTimeout(resolve, 1))
+      }
+      // After waiting, check if there's anything left to flush
+      // If another flush completed everything, we're done
+      if (this.batches.length === 0 && this.currentBatch.length === 0) return
+      
+      // Check again if database was closed during wait
+      if (this.database.destroyed || this.database.closed) {
+        this.batches = []
+        this.currentBatch = []
+        return
+      }
+    }
+    
     this.flushing = true
 
     try {
-      // Process all complete batches
-      for (const batch of this.batches) {
-        await this.database.insertBatch(batch)
-      }
+      // Process continuously until queue is completely empty
+      // This handles the case where new data is added during the flush
+      while (this.batches.length > 0 || this.currentBatch.length > 0) {
+        // Check if database was closed during processing
+        if (this.database.destroyed || this.database.closed) {
+          // Clear remaining batches
+          this.batches = []
+          this.currentBatch = []
+          return
+        }
 
-      // Process remaining records in current batch
-      if (this.currentBatch.length > 0) {
-        await this.database.insertBatch(this.currentBatch)
-      }
+        // Process all complete batches that exist at this moment
+        // Note: new batches may be added to this.batches during this loop
+        const batchesToProcess = this.batches.length
+        for (let i = 0; i < batchesToProcess; i++) {
+          // Check again before each batch
+          if (this.database.destroyed || this.database.closed) {
+            this.batches = []
+            this.currentBatch = []
+            return
+          }
+          
+          const batch = this.batches.shift() // Remove from front
+          await this.database.insertBatch(batch)
+        }
 
-      // Clear all batches
-      this.batches = []
-      this.currentBatch = []
+        // Process current batch if it has data
+        // Note: new records may be added to currentBatch during processing
+        if (this.currentBatch.length > 0) {
+          // Check if database was closed
+          if (this.database.destroyed || this.database.closed) {
+            this.batches = []
+            this.currentBatch = []
+            return
+          }
+
+          // Check if currentBatch reached batchSize during processing
+          if (this.currentBatch.length >= this.batchSize) {
+            // Move it to batches array and process in next iteration
+            this.batches.push(this.currentBatch)
+            this.currentBatch = []
+            continue
+          }
+          
+          // Process the current batch
+          const batchToProcess = this.currentBatch
+          this.currentBatch = [] // Clear before processing to allow new adds
+          await this.database.insertBatch(batchToProcess)
+        }
+      }
     } finally {
       this.flushing = false
     }
   }
 
+  async flush() {
+    // Wait for any pending auto-flushes to complete first
+    await this.waitForAutoFlushes()
+    
+    // Then do a final flush to ensure everything is processed
+    await this._doFlush()
+  }
+
+  async waitForAutoFlushes() {
+    // Wait for all pending auto-flush operations to complete
+    if (this.pendingAutoFlushes.size > 0) {
+      await Promise.all(Array.from(this.pendingAutoFlushes))
+    }
+  }
+
   async commit() {
     // CRITICAL FIX: Make session auto-reusable by removing committed state
     // Allow multiple commits on the same session
     
+    // First, wait for all pending auto-flushes to complete
+    await this.waitForAutoFlushes()
+    
+    // Then flush any remaining data (including currentBatch)
+    // This ensures everything is inserted before commit returns
     await this.flush()
     
     // Reset session state for next commit cycle
@@ -158,6 +269,9 @@ class InsertSession {
     const startTime = Date.now()
     const hasTimeout = maxWaitTime !== null && maxWaitTime !== undefined
     
+    // Wait for auto-flushes first
+    await this.waitForAutoFlushes()
+    
     while (this.flushing || this.batches.length > 0 || this.currentBatch.length > 0) {
       // Check timeout only if we have one
       if (hasTimeout && (Date.now() - startTime) >= maxWaitTime) {
@@ -174,7 +288,10 @@ class InsertSession {
    * Check if this session has pending operations
    */
   hasPendingOperations() {
-    return this.flushing || this.batches.length > 0 || this.currentBatch.length > 0
+    return this.pendingAutoFlushes.size > 0 || 
+           this.flushing || 
+           this.batches.length > 0 || 
+           this.currentBatch.length > 0
   }
   
   /**
@@ -189,6 +306,7 @@ class InsertSession {
     this.currentBatch = []
     this.totalInserted = 0
     this.flushing = false
+    this.pendingAutoFlushes.clear()
   }
 }
 
@@ -238,6 +356,8 @@ class Database extends EventEmitter {
       streamingThreshold: opts.streamingThreshold || 0.8, // Use streaming when limit > 80% of total records
       // Serialization options
       enableArraySerialization: opts.enableArraySerialization !== false, // Enable array serialization by default
+      // Index rebuild options
+      allowIndexRebuild: opts.allowIndexRebuild === true, // Allow automatic index rebuild when corrupted (default false - throws error)
     }, opts)
     
     // CRITICAL FIX: Initialize AbortController for lifecycle management
@@ -264,6 +384,8 @@ class Database extends EventEmitter {
     this.isSaving = false
     this.lastSaveTime = null
     this.initialized = false
+    this._offsetRecoveryInProgress = false
+    this.writeBufferTotalSize = 0
     
     
     // Initialize managers
@@ -310,10 +432,11 @@ class Database extends EventEmitter {
 
     // Validate indexes array (new format) - but only if we have fields
     if (this.opts.originalIndexes && Array.isArray(this.opts.originalIndexes)) {
-      if (!this.opts.fields) {
-        throw new Error('Index fields array requires fields configuration. Use: { fields: {...}, indexes: [...] }')
+      if (this.opts.fields) {
+        this.validateIndexFields(this.opts.originalIndexes)
+      } else if (this.opts.debugMode) {
+        console.log('⚠️  Skipping index field validation because no fields configuration was provided')
       }
-      this.validateIndexFields(this.opts.originalIndexes)
     }
 
     if (this.opts.debugMode) {
@@ -330,10 +453,14 @@ class Database extends EventEmitter {
    * Validate field types
    */
   validateFieldTypes(fields, configType) {
-    const supportedTypes = ['string', 'number', 'boolean', 'array:string', 'array:number', 'array:boolean', 'array', 'object']
+    const supportedTypes = ['string', 'number', 'boolean', 'array:string', 'array:number', 'array:boolean', 'array', 'object', 'auto']
     const errors = []
 
     for (const [fieldName, fieldType] of Object.entries(fields)) {
+      if (fieldType === 'auto') {
+        continue
+      }
+
       // Check if type is supported
       if (!supportedTypes.includes(fieldType)) {
         errors.push(`Unsupported ${configType} type '${fieldType}' for field '${fieldName}'. Supported types: ${supportedTypes.join(', ')}`)
@@ -383,26 +510,24 @@ class Database extends EventEmitter {
    * Prepare index configuration for IndexManager
    */
   prepareIndexConfiguration() {
-    // Convert new fields/indexes format to legacy format for IndexManager
-    if (this.opts.fields && Array.isArray(this.opts.indexes)) {
-      // New format: { fields: {...}, indexes: [...] }
+    if (Array.isArray(this.opts.indexes)) {
       const indexedFields = {}
-      const originalIndexes = [...this.opts.indexes] // Keep original for validation
-      
+      const originalIndexes = [...this.opts.indexes]
+      const hasFieldConfig = this.opts.fields && typeof this.opts.fields === 'object'
+
       for (const fieldName of this.opts.indexes) {
-        if (this.opts.fields[fieldName]) {
+        if (hasFieldConfig && this.opts.fields[fieldName]) {
           indexedFields[fieldName] = this.opts.fields[fieldName]
+        } else {
+          indexedFields[fieldName] = 'auto'
         }
       }
-      
-      // Store original indexes for validation
+
       this.opts.originalIndexes = originalIndexes
-      
-      // Replace indexes array with object for IndexManager
       this.opts.indexes = indexedFields
-      
+
       if (this.opts.debugMode) {
-        console.log(`🔍 Converted fields/indexes format: ${Object.keys(indexedFields).join(', ')} [${this.instanceId}]`)
+        console.log(`🔍 Normalized indexes array to object: ${Object.keys(indexedFields).join(', ')} [${this.instanceId}]`)
       }
     }
     // Legacy format (indexes as object) is already compatible
@@ -436,6 +561,19 @@ class Database extends EventEmitter {
     this.termManager.termMappingFields = termMappingFields
     this.opts.termMapping = true // Always enable term mapping for optimal performance
     
+    // Validation: Ensure all array:string indexed fields are in term mapping fields
+    if (this.opts.indexes) {
+      const arrayStringFields = []
+      for (const [field, type] of Object.entries(this.opts.indexes)) {
+        if (type === 'array:string' && !termMappingFields.includes(field)) {
+          arrayStringFields.push(field)
+        }
+      }
+      if (arrayStringFields.length > 0) {
+        console.warn(`⚠️  Warning: The following array:string indexed fields were not added to term mapping: ${arrayStringFields.join(', ')}. This may impact performance.`)
+      }
+    }
+    
     if (this.opts.debugMode) {
       if (termMappingFields.length > 0) {
         console.log(`🔍 TermManager initialized for fields: ${termMappingFields.join(', ')} [${this.instanceId}]`)
@@ -462,6 +600,7 @@ class Database extends EventEmitter {
     this.writeBuffer = []
     this.writeBufferOffsets = [] // Track offsets for writeBuffer records
     this.writeBufferSizes = []   // Track sizes for writeBuffer records
+    this.writeBufferTotalSize = 0
     this.isInsideOperationQueue = false // Flag to prevent deadlock in save() calls
     
     // Initialize other managers
@@ -483,8 +622,8 @@ class Database extends EventEmitter {
     const termMappingFields = []
     
     for (const [field, type] of Object.entries(this.opts.indexes)) {
-      // Fields that should use term mapping
-      if (type === 'array:string' || type === 'string') {
+      // Fields that should use term mapping (only array fields)
+      if (type === 'array:string') {
         termMappingFields.push(field)
       }
     }
@@ -704,6 +843,9 @@ class Database extends EventEmitter {
       // Don't load the entire file - just initialize empty state
       // The actual record count will come from loaded offsets
       this.writeBuffer = [] // writeBuffer is only for new unsaved records
+      this.writeBufferOffsets = []
+      this.writeBufferSizes = []
+      this.writeBufferTotalSize = 0
       
       // recordCount will be determined from loaded offsets
       // If no offsets were loaded, we'll count records only if needed
@@ -713,13 +855,55 @@ class Database extends EventEmitter {
         const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb')
         try {
           const idxFileHandler = new FileHandler(idxPath, this.fileMutex, this.opts)
+          
+          // Check if file exists BEFORE trying to read it
+          const fileExists = await idxFileHandler.exists()
+          if (!fileExists) {
+            // File doesn't exist - this will be handled by catch block
+            throw new Error('Index file does not exist')
+          }
+          
           const idxData = await idxFileHandler.readAll()
-          if (idxData && idxData.trim()) {
+          
+          // If file exists but is empty or has no content, treat as corrupted
+          if (!idxData || !idxData.trim()) {
+            // File exists but is empty - treat as corrupted
+            const fileExists = await this.fileHandler.exists()
+            if (fileExists) {
+              const stats = await this.fileHandler.getFileStats()
+              if (stats && stats.size > 0) {
+                // Data file has content but index is empty - corrupted
+                if (!this.opts.allowIndexRebuild) {
+                  throw new Error(
+                    `Index file is corrupted: ${idxPath} exists but contains no index data, ` +
+                    `while the data file has ${stats.size} bytes. ` +
+                    `Set allowIndexRebuild: true to automatically rebuild the index, ` +
+                    `or manually fix/delete the corrupted index file.`
+                  )
+                }
+                // Schedule rebuild if allowed
+                if (this.opts.debugMode) {
+                  console.log(`⚠️ Index file exists but is empty while data file has ${stats.size} bytes - scheduling rebuild`)
+                }
+                this._scheduleIndexRebuild()
+                // Continue execution - rebuild will happen on first query
+                // Don't return - let the code continue to load other things if needed
+              }
+            }
+            // If data file is also empty, just continue (no error needed)
+            // Don't return - let the code continue to load other things if needed
+          } else {
+            // File has content - parse and load it
             const parsedIdxData = JSON.parse(idxData)
             
             // Always load offsets if available (even without indexed fields)
             if (parsedIdxData.offsets && Array.isArray(parsedIdxData.offsets)) {
               this.offsets = parsedIdxData.offsets
+              // CRITICAL FIX: Update IndexManager totalLines to match offsets length
+              // This ensures queries and length property work correctly even if offsets are reset later
+              if (this.indexManager && this.offsets.length > 0) {
+                this.indexManager.setTotalLines(this.offsets.length)
+              }
               if (this.opts.debugMode) {
                 console.log(`📂 Loaded ${this.offsets.length} offsets from ${idxPath}`)
               }
@@ -733,24 +917,8 @@ class Database extends EventEmitter {
               }
             }
             
-            // Load index data only if available and we have indexed fields
-            if (parsedIdxData && parsedIdxData.index && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
-              this.indexManager.load(parsedIdxData.index)
-              
-              // Load term mapping data from .idx file if it exists
-              if (parsedIdxData.termMapping && this.termManager) {
-                await this.termManager.loadTerms(parsedIdxData.termMapping)
-                if (this.opts.debugMode) {
-                  console.log(`📂 Loaded term mapping from ${idxPath}`)
-                }
-              }
-              
-              if (this.opts.debugMode) {
-                console.log(`📂 Loaded index data from ${idxPath}`)
-              }
-            }
-            
             // Load configuration from .idx file if database exists
+            // CRITICAL: Load config FIRST so indexes are available for term mapping detection
             if (parsedIdxData.config) {
               const config = parsedIdxData.config
               
@@ -764,11 +932,94 @@ class Database extends EventEmitter {
               
               if (config.indexes) {
                 this.opts.indexes = config.indexes
+                if (this.indexManager) {
+                  this.indexManager.setIndexesConfig(config.indexes)
+                }
                 if (this.opts.debugMode) {
                   console.log(`📂 Loaded indexes config from ${idxPath}:`, Object.keys(config.indexes))
                 }
               }
               
+              // CRITICAL FIX: Update term mapping fields AFTER loading indexes from config
+              // This ensures termManager knows which fields use term mapping
+              // (getTermMappingFields() was called during init() before indexes were loaded)
+              if (this.termManager && config.indexes) {
+                const termMappingFields = this.getTermMappingFields()
+                this.termManager.termMappingFields = termMappingFields
+                if (this.opts.debugMode && termMappingFields.length > 0) {
+                  console.log(`🔍 Updated term mapping fields after loading indexes: ${termMappingFields.join(', ')}`)
+                }
+              }
+            }
+            
+            // Load term mapping data from .idx file if it exists
+            // CRITICAL: Load termMapping even if index is empty (terms are needed for queries)
+            // NOTE: termMappingFields should already be set above from config.indexes
+            if (parsedIdxData.termMapping && this.termManager && this.termManager.termMappingFields && this.termManager.termMappingFields.length > 0) {
+              await this.termManager.loadTerms(parsedIdxData.termMapping)
+              if (this.opts.debugMode) {
+                console.log(`📂 Loaded term mapping from ${idxPath} (${Object.keys(parsedIdxData.termMapping).length} terms)`)
+              }
+            }
+            
+            // Load index data only if available and we have indexed fields
+            if (parsedIdxData && parsedIdxData.index && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
+              this.indexManager.load(parsedIdxData.index)
+              
+              if (this.opts.debugMode) {
+                console.log(`📂 Loaded index data from ${idxPath}`)
+              }
+              
+              // Check if loaded index is actually empty (corrupted)
+              let hasAnyIndexData = false
+              for (const field of this.indexManager.indexedFields) {
+                if (this.indexManager.hasUsableIndexData(field)) {
+                  hasAnyIndexData = true
+                  break
+                }
+              }
+              
+              if (this.opts.debugMode) {
+                console.log(`📊 Index check: hasAnyIndexData=${hasAnyIndexData}, indexedFields=${this.indexManager.indexedFields.join(',')}`)
+              }
+              
+              // Schedule rebuild if index is empty AND file exists with data
+              if (!hasAnyIndexData) {
+                // Check if the actual .jdb file has data
+                const fileExists = await this.fileHandler.exists()
+                if (this.opts.debugMode) {
+                  console.log(`📊 File check: exists=${fileExists}`)
+                }
+                if (fileExists) {
+                  const stats = await this.fileHandler.getFileStats()
+                  if (this.opts.debugMode) {
+                    console.log(`📊 File stats: size=${stats?.size}`)
+                  }
+                  if (stats && stats.size > 0) {
+                    // File has data but index is empty - corrupted index detected
+                    if (!this.opts.allowIndexRebuild) {
+                      const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb')
+                      throw new Error(
+                        `Index file is corrupted: ${idxPath} exists but contains no index data, ` +
+                        `while the data file has ${stats.size} bytes. ` +
+                        `Set allowIndexRebuild: true to automatically rebuild the index, ` +
+                        `or manually fix/delete the corrupted index file.`
+                      )
+                    }
+                    // Schedule rebuild if allowed
+                    if (this.opts.debugMode) {
+                      console.log(`⚠️ Index loaded but empty while file has ${stats.size} bytes - scheduling rebuild`)
+                    }
+                    this._scheduleIndexRebuild()
+                  }
+                }
+              }
+            }
+            
+            // Continue with remaining config loading
+            if (parsedIdxData.config) {
+              const config = parsedIdxData.config
+              
               if (config.originalIndexes) {
                 this.opts.originalIndexes = config.originalIndexes
                 if (this.opts.debugMode) {
@@ -787,11 +1038,86 @@ class Database extends EventEmitter {
           }
         } catch (idxError) {
           // Index file doesn't exist or is corrupted, rebuild from data
+          // BUT: if error is about rebuild being disabled, re-throw it immediately
+          if (idxError.message && (idxError.message.includes('allowIndexRebuild') || idxError.message.includes('corrupted'))) {
+            throw idxError
+          }
+          
+          // If error is "Index file does not exist", check if we should throw or rebuild
+          if (idxError.message && idxError.message.includes('does not exist')) {
+            // Check if the actual .jdb file has data that needs indexing
+            try {
+              const fileExists = await this.fileHandler.exists()
+              if (fileExists) {
+                const stats = await this.fileHandler.getFileStats()
+                if (stats && stats.size > 0) {
+                  // File has data but index is missing
+                  if (!this.opts.allowIndexRebuild) {
+                    const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb')
+                    throw new Error(
+                      `Index file is missing or corrupted: ${idxPath} does not exist or is invalid, ` +
+                      `while the data file has ${stats.size} bytes. ` +
+                      `Set allowIndexRebuild: true to automatically rebuild the index, ` +
+                      `or manually create/fix the index file.`
+                    )
+                  }
+                  // Schedule rebuild if allowed
+                  if (this.opts.debugMode) {
+                    console.log(`⚠️ .jdb file has ${stats.size} bytes but index missing - scheduling rebuild`)
+                  }
+                  this._scheduleIndexRebuild()
+                  return // Exit early
+                }
+              }
+            } catch (statsError) {
+              if (this.opts.debugMode) {
+                console.log('⚠️ Could not check file stats:', statsError.message)
+              }
+              // Re-throw if it's our error about rebuild being disabled
+              if (statsError.message && statsError.message.includes('allowIndexRebuild')) {
+                throw statsError
+              }
+            }
+            // If no data file or empty, just continue (no error needed)
+            return
+          }
+          
           if (this.opts.debugMode) {
-            console.log('📂 No index file found, rebuilding indexes from data')
+            console.log('📂 No index file found or corrupted, checking if rebuild is needed...')
+          }
+          
+          // Check if the actual .jdb file has data that needs indexing
+          try {
+            const fileExists = await this.fileHandler.exists()
+            if (fileExists) {
+              const stats = await this.fileHandler.getFileStats()
+              if (stats && stats.size > 0) {
+                // File has data but index is missing or corrupted
+                if (!this.opts.allowIndexRebuild) {
+                  const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb')
+                  throw new Error(
+                    `Index file is missing or corrupted: ${idxPath} does not exist or is invalid, ` +
+                    `while the data file has ${stats.size} bytes. ` +
+                    `Set allowIndexRebuild: true to automatically rebuild the index, ` +
+                    `or manually create/fix the index file.`
+                  )
+                }
+                // Schedule rebuild if allowed
+                if (this.opts.debugMode) {
+                  console.log(`⚠️ .jdb file has ${stats.size} bytes but index missing - scheduling rebuild`)
+                }
+                this._scheduleIndexRebuild()
+              }
+            }
+          } catch (statsError) {
+            if (this.opts.debugMode) {
+              console.log('⚠️ Could not check file stats:', statsError.message)
+            }
+            // Re-throw if it's our error about rebuild being disabled
+            if (statsError.message && statsError.message.includes('allowIndexRebuild')) {
+              throw statsError
+            }
           }
-          // We can't rebuild index without violating no-memory-storage rule
-          // Index will be rebuilt as needed during queries
         }
       } else {
         // No indexed fields, no need to rebuild indexes
@@ -820,6 +1146,28 @@ class Database extends EventEmitter {
       console.log(`💾 save() called: writeBuffer.length=${this.writeBuffer.length}, offsets.length=${this.offsets.length}`)
     }
     
+    // CRITICAL FIX: Wait for all active insert sessions to complete their auto-flushes
+    // This prevents race conditions where save() writes data while auto-flushes are still adding to writeBuffer
+    if (this.activeInsertSessions && this.activeInsertSessions.size > 0) {
+      if (this.opts.debugMode) {
+        console.log(`⏳ save(): Waiting for ${this.activeInsertSessions.size} active insert sessions to complete auto-flushes`)
+      }
+      
+      const sessionPromises = Array.from(this.activeInsertSessions).map(session => 
+        session.waitForAutoFlushes().catch(err => {
+          if (this.opts.debugMode) {
+            console.warn(`⚠️ save(): Error waiting for insert session: ${err.message}`)
+          }
+        })
+      )
+      
+      await Promise.all(sessionPromises)
+      
+      if (this.opts.debugMode) {
+        console.log(`✅ save(): All insert sessions completed auto-flushes`)
+      }
+    }
+    
     // Auto-save removed - no need to pause anything
     
     try {
@@ -1210,8 +1558,34 @@ class Database extends EventEmitter {
             }
             
             // Rebuild index from the saved records
+            // CRITICAL: Process term mapping for records loaded from file to ensure ${field}Ids are available
             for (let i = 0; i < allData.length; i++) {
-              const record = allData[i]
+              let record = allData[i]
+              
+              // CRITICAL FIX: Ensure records have ${field}Ids for term mapping fields
+              // Records from writeBuffer already have ${field}Ids from processTermMapping
+              // Records from file need to be processed to restore ${field}Ids
+              const termMappingFields = this.getTermMappingFields()
+              if (termMappingFields.length > 0 && this.termManager) {
+                for (const field of termMappingFields) {
+                  if (record[field] && Array.isArray(record[field])) {
+                    // Check if field contains term IDs (numbers) or terms (strings)
+                    const firstValue = record[field][0]
+                    if (typeof firstValue === 'number') {
+                      // Already term IDs, create ${field}Ids
+                      record[`${field}Ids`] = record[field]
+                    } else if (typeof firstValue === 'string') {
+                      // Terms, need to convert to term IDs
+                      const termIds = record[field].map(term => {
+                        const termId = this.termManager.getTermIdWithoutIncrement(term)
+                        return termId !== undefined ? termId : this.termManager.getTermId(term)
+                      })
+                      record[`${field}Ids`] = termIds
+                    }
+                  }
+                }
+              }
+              
               await this.indexManager.add(record, i)
             }
           }
@@ -1247,6 +1621,8 @@ class Database extends EventEmitter {
           this.writeBuffer = []
           this.writeBufferOffsets = []
           this.writeBufferSizes = []
+          this.writeBufferTotalSize = 0
+          this.writeBufferTotalSize = 0
         }
         
         // indexOffset already set correctly to currentOffset (total file size) above
@@ -1465,18 +1841,16 @@ class Database extends EventEmitter {
     }
 
     // OPTIMIZATION: Process records using pre-computed term IDs
-    return records.map(record => {
-      const processedRecord = { ...record }
-      
+    for (const record of records) {
       for (const field of termMappingFields) {
         if (record[field] && Array.isArray(record[field])) {
           const termIds = record[field].map(term => termIdMap.get(term))
-          processedRecord[`${field}Ids`] = termIds
+          record[`${field}Ids`] = termIds
         }
       }
-      
-      return processedRecord
-    })
+    }
+    
+    return records
   }
 
 
@@ -1570,18 +1944,19 @@ class Database extends EventEmitter {
       // OPTIMIZATION: Calculate and store offset and size for writeBuffer record
       // SPACE OPTIMIZATION: Remove term IDs before serialization
       const cleanRecord = this.removeTermIdsForSerialization(record)
-      const recordJson = this.serializer.serialize(cleanRecord).toString('utf8')
-      const recordSize = Buffer.byteLength(recordJson, 'utf8')
+      const recordBuffer = this.serializer.serialize(cleanRecord)
+      const recordSize = recordBuffer.length
       
       // Calculate offset based on end of file + previous writeBuffer sizes
-      const previousWriteBufferSize = this.writeBufferSizes.reduce((sum, size) => sum + size, 0)
+      const previousWriteBufferSize = this.writeBufferTotalSize
       const recordOffset = this.indexOffset + previousWriteBufferSize
       
       this.writeBufferOffsets.push(recordOffset)
       this.writeBufferSizes.push(recordSize)
+      this.writeBufferTotalSize += recordSize
       
-      // OPTIMIZATION: Use the current writeBuffer size as the line number (0-based index)
-      const lineNumber = this.writeBuffer.length - 1
+      // OPTIMIZATION: Use the absolute line number (persisted records + writeBuffer index)
+      const lineNumber = this._getAbsoluteLineNumber(this.writeBuffer.length - 1)
       
       // OPTIMIZATION: Defer index updates to batch processing
       // Store the record for batch index processing
@@ -1652,7 +2027,7 @@ class Database extends EventEmitter {
       console.log(`💾 _insertBatchInternal: processing size=${dataArray.length}, startWriteBuffer=${this.writeBuffer.length}`)
     }
     const records = []
-    const startLineNumber = this.writeBuffer.length
+    const existingWriteBufferLength = this.writeBuffer.length
     
     // Initialize schema if not already done (auto-detect from first record)
     if (this.serializer && !this.serializer.schemaManager.isInitialized && dataArray.length > 0) {
@@ -1684,13 +2059,13 @@ class Database extends EventEmitter {
     this.writeBuffer.push(...schemaEnforcedRecords)
     
     // OPTIMIZATION: Calculate offsets and sizes in batch (O(n))
-    let runningTotalSize = this.writeBufferSizes.reduce((sum, size) => sum + size, 0)
+    let runningTotalSize = this.writeBufferTotalSize
     for (let i = 0; i < processedRecords.length; i++) {
       const record = processedRecords[i]
       // SPACE OPTIMIZATION: Remove term IDs before serialization
       const cleanRecord = this.removeTermIdsForSerialization(record)
-      const recordJson = this.serializer.serialize(cleanRecord).toString('utf8')
-      const recordSize = Buffer.byteLength(recordJson, 'utf8')
+      const recordBuffer = this.serializer.serialize(cleanRecord)
+      const recordSize = recordBuffer.length
       
       const recordOffset = this.indexOffset + runningTotalSize
       runningTotalSize += recordSize
@@ -1698,6 +2073,7 @@ class Database extends EventEmitter {
       this.writeBufferOffsets.push(recordOffset)
       this.writeBufferSizes.push(recordSize)
     }
+    this.writeBufferTotalSize = runningTotalSize
     
     // OPTIMIZATION: Batch process index updates
     if (!this.pendingIndexUpdates) {
@@ -1705,7 +2081,7 @@ class Database extends EventEmitter {
     }
     
     for (let i = 0; i < processedRecords.length; i++) {
-      const lineNumber = startLineNumber + i
+      const lineNumber = this._getAbsoluteLineNumber(existingWriteBufferLength + i)
       this.pendingIndexUpdates.push({ record: processedRecords[i], lineNumber })
     }
     
@@ -1745,7 +2121,7 @@ class Database extends EventEmitter {
     try {
       // Validate indexed query mode if enabled
       if (this.opts.indexedQueryMode === 'strict') {
-        this._validateIndexedQuery(criteria)
+        this._validateIndexedQuery(criteria, options)
       }
       
       // Get results from file (QueryManager already handles term ID restoration)
@@ -1820,8 +2196,15 @@ class Database extends EventEmitter {
   /**
    * Validate indexed query mode for strict mode
    * @private
+   * @param {Object} criteria - Query criteria
+   * @param {Object} options - Query options
    */
-  _validateIndexedQuery(criteria) {
+  _validateIndexedQuery(criteria, options = {}) {
+    // Allow bypassing strict mode validation with allowNonIndexed option
+    if (options.allowNonIndexed === true) {
+      return; // Skip validation for this query
+    }
+
     if (!criteria || typeof criteria !== 'object') {
       return // Allow null/undefined criteria
     }
@@ -2100,7 +2483,7 @@ class Database extends EventEmitter {
         if (index !== -1) {
           // Record is already in writeBuffer, update it
           this.writeBuffer[index] = updated
-          lineNumber = index
+          lineNumber = this._getAbsoluteLineNumber(index)
           if (this.opts.debugMode) {
             console.log(`🔄 UPDATE: Updated existing writeBuffer record at index ${index}`)
           }
@@ -2108,7 +2491,7 @@ class Database extends EventEmitter {
           // Record is in file, add updated version to writeBuffer
           // This will ensure the updated record is saved and replaces the file version
           this.writeBuffer.push(updated)
-          lineNumber = this.writeBuffer.length - 1
+          lineNumber = this._getAbsoluteLineNumber(this.writeBuffer.length - 1)
           if (this.opts.debugMode) {
             console.log(`🔄 UPDATE: Added new record to writeBuffer at index ${lineNumber}`)
           }
@@ -2294,6 +2677,21 @@ class Database extends EventEmitter {
     const savedRecords = this.offsets.length
     const writeBufferRecords = this.writeBuffer.length
     
+    // CRITICAL FIX: If offsets are empty but indexOffset exists, use fallback calculation
+    // This handles cases where offsets weren't loaded or were reset
+    if (savedRecords === 0 && this.indexOffset > 0 && this.initialized) {
+      // Try to use IndexManager totalLines if available
+      if (this.indexManager && this.indexManager.totalLines > 0) {
+        return this.indexManager.totalLines + writeBufferRecords
+      }
+      
+      // Fallback: estimate from indexOffset (less accurate but better than 0)
+      // This is a defensive fix for cases where offsets are missing but file has data
+      if (this.opts.debugMode) {
+        console.log(`⚠️  LENGTH: offsets array is empty but indexOffset=${this.indexOffset}, using IndexManager.totalLines or estimation`)
+      }
+    }
+    
     // CRITICAL FIX: Validate that offsets array is consistent with actual data
     // This prevents the bug where database reassignment causes desynchronization
     if (this.initialized && savedRecords > 0) {
@@ -2339,22 +2737,7 @@ class Database extends EventEmitter {
    * Calculate current writeBuffer size in bytes (similar to published v1.1.0)
    */
   currentWriteBufferSize() {
-    if (!this.writeBuffer || this.writeBuffer.length === 0) {
-      return 0
-    }
-    
-    // Calculate total size of all records in writeBuffer
-    let totalSize = 0
-    for (const record of this.writeBuffer) {
-      if (record) {
-        // SPACE OPTIMIZATION: Remove term IDs before size calculation
-        const cleanRecord = this.removeTermIdsForSerialization(record)
-        const recordJson = JSON.stringify(cleanRecord) + '\n'
-        totalSize += Buffer.byteLength(recordJson, 'utf8')
-      }
-    }
-    
-    return totalSize
+    return this.writeBufferTotalSize || 0
   }
 
   /**
@@ -2387,62 +2770,236 @@ class Database extends EventEmitter {
   }
 
   /**
-   * Destroy database - DESTRUCTIVE MODE
-   * Assumes save() has already been called by user
-   * If anything is still active, it indicates a bug - log error and force cleanup
+   * Schedule index rebuild when index data is missing or corrupted
+   * @private
    */
-  async destroy() {
-    if (this.destroyed) return
+  _scheduleIndexRebuild() {
+    // Mark that rebuild is needed
+    this._indexRebuildNeeded = true
     
-    // Mark as destroying immediately to prevent new operations
-    this.destroying = true
+    // Rebuild will happen lazily on first query if index is empty
+    // This avoids blocking init() but ensures index is available when needed
+  }
+
+  /**
+   * Rebuild indexes from data file if needed
+   * @private
+   */
+  async _rebuildIndexesIfNeeded() {
+    if (this.opts.debugMode) {
+      console.log(`🔍 _rebuildIndexesIfNeeded called: _indexRebuildNeeded=${this._indexRebuildNeeded}`)
+    }
+    if (!this._indexRebuildNeeded) return
+    if (!this.indexManager || !this.indexManager.indexedFields || this.indexManager.indexedFields.length === 0) return
     
-    // Wait for all active insert sessions to complete before destroying
-    if (this.activeInsertSessions.size > 0) {
-      if (this.opts.debugMode) {
-        console.log(`⏳ destroy: Waiting for ${this.activeInsertSessions.size} active insert sessions`)
+    // Check if index actually needs rebuilding
+    let needsRebuild = false
+    for (const field of this.indexManager.indexedFields) {
+      if (!this.indexManager.hasUsableIndexData(field)) {
+        needsRebuild = true
+        break
       }
-      
-      const sessionPromises = Array.from(this.activeInsertSessions).map(session => 
-        session.waitForOperations(null) // Wait indefinitely for sessions to complete
+    }
+    
+    if (!needsRebuild) {
+      this._indexRebuildNeeded = false
+      return
+    }
+    
+    // Check if rebuild is allowed
+    if (!this.opts.allowIndexRebuild) {
+      const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb')
+      throw new Error(
+        `Index rebuild required but disabled: Index file ${idxPath} is corrupted or missing, ` +
+        `and allowIndexRebuild is set to false. ` +
+        `Set allowIndexRebuild: true to automatically rebuild the index, ` +
+        `or manually fix/delete the corrupted index file.`
       )
-      
-      try {
-        await Promise.all(sessionPromises)
-      } catch (error) {
-        if (this.opts.debugMode) {
-          console.log(`⚠️ destroy: Error waiting for sessions: ${error.message}`)
-        }
-        // Continue with destruction even if sessions have issues
-      }
-      
-      // Destroy all active sessions
-      for (const session of this.activeInsertSessions) {
-        session.destroy()
-      }
-      this.activeInsertSessions.clear()
     }
     
-    // CRITICAL FIX: Add timeout protection to prevent destroy() from hanging
-    const destroyPromise = this._performDestroy()
-    let timeoutHandle = null
-    const timeoutPromise = new Promise((_, reject) => {
-      timeoutHandle = setTimeout(() => {
-        reject(new Error('Destroy operation timed out after 5 seconds'))
-      }, 5000)
-    })
+    if (this.opts.debugMode) {
+      console.log('🔨 Rebuilding indexes from data file...')
+    }
     
     try {
-      await Promise.race([destroyPromise, timeoutPromise])
-    } catch (error) {
-      if (error.message === 'Destroy operation timed out after 5 seconds') {
-        console.error('🚨 DESTROY TIMEOUT: Force destroying database after timeout')
-        // Force mark as destroyed even if cleanup failed
-        this.destroyed = true
-        this.destroying = false
-        return
-      }
-      throw error
+      // Read all records and rebuild index
+      let count = 0
+      const startTime = Date.now()
+      
+      // Auto-detect schema from first line if not initialized
+      if (!this.serializer.schemaManager.isInitialized) {
+        const fs = await import('fs')
+        const readline = await import('readline')
+        const stream = fs.createReadStream(this.fileHandler.file, {
+          highWaterMark: 64 * 1024,
+          encoding: 'utf8'
+        })
+        const rl = readline.createInterface({
+          input: stream,
+          crlfDelay: Infinity
+        })
+        
+        for await (const line of rl) {
+          if (line && line.trim()) {
+            try {
+              const firstRecord = JSON.parse(line)
+              if (Array.isArray(firstRecord)) {
+                // Try to infer schema from opts.fields if available
+                if (this.opts.fields && typeof this.opts.fields === 'object') {
+                  const fieldNames = Object.keys(this.opts.fields)
+                  if (fieldNames.length >= firstRecord.length) {
+                    // Use first N fields from opts.fields to match array length
+                    const schema = fieldNames.slice(0, firstRecord.length)
+                    this.serializer.initializeSchema(schema)
+                    if (this.opts.debugMode) {
+                      console.log(`🔍 Inferred schema from opts.fields: ${schema.join(', ')}`)
+                    }
+                  } else {
+                    throw new Error(`Cannot rebuild index: array has ${firstRecord.length} elements but opts.fields only defines ${fieldNames.length} fields. Schema must be explicitly provided.`)
+                  }
+                } else {
+                  throw new Error('Cannot rebuild index: schema missing, file uses array format, and opts.fields not provided. The .idx.jdb file is corrupted.')
+                }
+              } else {
+                // Object format, initialize from object keys
+                this.serializer.initializeSchema(firstRecord, true)
+                if (this.opts.debugMode) {
+                  console.log(`🔍 Auto-detected schema from object: ${Object.keys(firstRecord).join(', ')}`)
+                }
+              }
+              break
+            } catch (error) {
+              if (this.opts.debugMode) {
+                console.error('❌ Failed to auto-detect schema:', error.message)
+              }
+              throw error
+            }
+          }
+        }
+        stream.destroy()
+      }
+      
+      // Use streaming to read records without loading everything into memory
+      // Also rebuild offsets while we're at it
+      const fs = await import('fs')
+      const readline = await import('readline')
+      
+      this.offsets = []
+      let currentOffset = 0
+      
+      const stream = fs.createReadStream(this.fileHandler.file, {
+        highWaterMark: 64 * 1024,
+        encoding: 'utf8'
+      })
+      
+      const rl = readline.createInterface({
+        input: stream,
+        crlfDelay: Infinity
+      })
+      
+      try {
+        for await (const line of rl) {
+          if (line && line.trim()) {
+            try {
+              // Record the offset for this line
+              this.offsets.push(currentOffset)
+              
+              const record = this.serializer.deserialize(line)
+              const recordWithTerms = this.restoreTermIdsAfterDeserialization(record)
+              await this.indexManager.add(recordWithTerms, count)
+              count++
+            } catch (error) {
+              // Skip invalid lines
+              if (this.opts.debugMode) {
+                console.log(`⚠️ Rebuild: Failed to deserialize line ${count}:`, error.message)
+              }
+            }
+          }
+          // Update offset for next line (including newline character)
+          currentOffset += Buffer.byteLength(line, 'utf8') + 1
+        }
+      } finally {
+        stream.destroy()
+      }
+      
+      // Update indexManager totalLines
+      if (this.indexManager) {
+        this.indexManager.setTotalLines(this.offsets.length)
+      }
+      
+      this._indexRebuildNeeded = false
+      
+      if (this.opts.debugMode) {
+        console.log(`✅ Index rebuilt from ${count} records in ${Date.now() - startTime}ms`)
+      }
+      
+      // Save the rebuilt index
+      await this._saveIndexDataToFile()
+    } catch (error) {
+      if (this.opts.debugMode) {
+        console.error('❌ Failed to rebuild indexes:', error.message)
+      }
+      // Don't throw - queries will fall back to streaming
+    }
+  }
+
+  /**
+   * Destroy database - DESTRUCTIVE MODE
+   * Assumes save() has already been called by user
+   * If anything is still active, it indicates a bug - log error and force cleanup
+   */
+  async destroy() {
+    if (this.destroyed) return
+    
+    // Mark as destroying immediately to prevent new operations
+    this.destroying = true
+    
+    // Wait for all active insert sessions to complete before destroying
+    if (this.activeInsertSessions.size > 0) {
+      if (this.opts.debugMode) {
+        console.log(`⏳ destroy: Waiting for ${this.activeInsertSessions.size} active insert sessions`)
+      }
+      
+      const sessionPromises = Array.from(this.activeInsertSessions).map(session => 
+        session.waitForOperations(null) // Wait indefinitely for sessions to complete
+      )
+      
+      try {
+        await Promise.all(sessionPromises)
+      } catch (error) {
+        if (this.opts.debugMode) {
+          console.log(`⚠️ destroy: Error waiting for sessions: ${error.message}`)
+        }
+        // Continue with destruction even if sessions have issues
+      }
+      
+      // Destroy all active sessions
+      for (const session of this.activeInsertSessions) {
+        session.destroy()
+      }
+      this.activeInsertSessions.clear()
+    }
+    
+    // CRITICAL FIX: Add timeout protection to prevent destroy() from hanging
+    const destroyPromise = this._performDestroy()
+    let timeoutHandle = null
+    const timeoutPromise = new Promise((_, reject) => {
+      timeoutHandle = setTimeout(() => {
+        reject(new Error('Destroy operation timed out after 5 seconds'))
+      }, 5000)
+    })
+    
+    try {
+      await Promise.race([destroyPromise, timeoutPromise])
+    } catch (error) {
+      if (error.message === 'Destroy operation timed out after 5 seconds') {
+        console.error('🚨 DESTROY TIMEOUT: Force destroying database after timeout')
+        // Force mark as destroyed even if cleanup failed
+        this.destroyed = true
+        this.destroying = false
+        return
+      }
+      throw error
     } finally {
       // Clear the timeout to prevent Jest open handle warning
       if (timeoutHandle) {
@@ -2503,6 +3060,8 @@ class Database extends EventEmitter {
       this.writeBuffer = []
       this.writeBufferOffsets = []
       this.writeBufferSizes = []
+      this.writeBufferTotalSize = 0
+      this.writeBufferTotalSize = 0
       this.deletedIds.clear()
       this.pendingOperations.clear()
       this.pendingIndexUpdates = []
@@ -2570,8 +3129,211 @@ class Database extends EventEmitter {
   async count(criteria = {}, options = {}) {
     this._validateInitialization('count')
     
-    const results = await this.find(criteria, options)
-    return results.length
+    // OPTIMIZATION: Use queryManager.count() instead of find() for better performance
+    // This is especially faster for indexed queries which can use indexManager.query().size
+    const fileCount = await this.queryManager.count(criteria, options)
+    
+    // Count matching records in writeBuffer
+    const writeBufferCount = this.writeBuffer.filter(record => 
+      this.queryManager.matchesCriteria(record, criteria, options)
+    ).length
+    
+    return fileCount + writeBufferCount
+  }
+
+  /**
+   * Check if any records exist for given field and terms (index-only, ultra-fast)
+   * Delegates to IndexManager.exists() for maximum performance
+   * 
+   * @param {string} fieldName - Indexed field name
+   * @param {string|Array<string>} terms - Single term or array of terms
+   * @param {Object} options - Options: { $all: true/false, caseInsensitive: true/false, excludes: Array<string> }
+   * @returns {Promise<boolean>} - True if at least one match exists
+   * 
+   * @example
+   * // Check if channel exists
+   * const exists = await db.exists('nameTerms', ['a', 'e'], { $all: true });
+   * 
+   * @example
+   * // Check if 'tv' exists but not 'globo'
+   * const exists = await db.exists('nameTerms', 'tv', { excludes: ['globo'] });
+   */
+  async exists(fieldName, terms, options = {}) {
+    this._validateInitialization('exists')
+    return this.indexManager.exists(fieldName, terms, options)
+  }
+
+  /**
+   * Calculate coverage for grouped include/exclude term sets
+   * @param {string} fieldName - Name of the indexed field
+   * @param {Array<object>} groups - Array of { terms, excludes } objects
+   * @param {object} options - Optional settings
+   * @returns {Promise<number>} Coverage percentage between 0 and 100
+   */
+  async coverage(fieldName, groups, options = {}) {
+    this._validateInitialization('coverage')
+
+    if (typeof fieldName !== 'string' || !fieldName.trim()) {
+      throw new Error('fieldName must be a non-empty string')
+    }
+
+    if (!Array.isArray(groups)) {
+      throw new Error('groups must be an array')
+    }
+
+    if (groups.length === 0) {
+      return 0
+    }
+
+    if (!this.opts.indexes || !this.opts.indexes[fieldName]) {
+      throw new Error(`Field "${fieldName}" is not indexed`)
+    }
+
+    const fieldType = this.opts.indexes[fieldName]
+    const supportedTypes = ['array:string', 'string']
+    if (!supportedTypes.includes(fieldType)) {
+      throw new Error(`coverage() only supports fields of type ${supportedTypes.join(', ')} (found: ${fieldType})`)
+    }
+
+    const fieldIndex = this.indexManager?.index?.data?.[fieldName]
+    if (!fieldIndex) {
+      return 0
+    }
+
+    const isTermMapped = this.termManager &&
+      this.termManager.termMappingFields &&
+      this.termManager.termMappingFields.includes(fieldName)
+
+    const normalizeTerm = (term) => {
+      if (term === undefined || term === null) {
+        return ''
+      }
+      return String(term).trim()
+    }
+
+    const resolveKey = (term) => {
+      if (isTermMapped) {
+        const termId = this.termManager.getTermIdWithoutIncrement(term)
+        if (termId === null || termId === undefined) {
+          return null
+        }
+        return String(termId)
+      }
+      return String(term)
+    }
+
+    let matchedGroups = 0
+
+    for (const group of groups) {
+      if (!group || typeof group !== 'object') {
+        throw new Error('Each coverage group must be an object')
+      }
+
+      const includeTermsRaw = Array.isArray(group.terms) ? group.terms : []
+      const excludeTermsRaw = Array.isArray(group.excludes) ? group.excludes : []
+
+      const includeTerms = Array.from(new Set(
+        includeTermsRaw
+          .map(normalizeTerm)
+          .filter(term => term.length > 0)
+      ))
+
+      if (includeTerms.length === 0) {
+        throw new Error('Each coverage group must define at least one term')
+      }
+
+      const excludeTerms = Array.from(new Set(
+        excludeTermsRaw
+          .map(normalizeTerm)
+          .filter(term => term.length > 0)
+      ))
+
+      let candidateLines = null
+      let groupMatched = true
+
+      for (const term of includeTerms) {
+        const key = resolveKey(term)
+        if (key === null) {
+          groupMatched = false
+          break
+        }
+
+        const termData = fieldIndex[key]
+        if (!termData) {
+          groupMatched = false
+          break
+        }
+
+        const lineNumbers = this.indexManager._getAllLineNumbers(termData)
+        if (!lineNumbers || lineNumbers.length === 0) {
+          groupMatched = false
+          break
+        }
+
+        if (candidateLines === null) {
+          candidateLines = new Set(lineNumbers)
+        } else {
+          const termSet = new Set(lineNumbers)
+          for (const line of Array.from(candidateLines)) {
+            if (!termSet.has(line)) {
+              candidateLines.delete(line)
+            }
+          }
+        }
+
+        if (!candidateLines || candidateLines.size === 0) {
+          groupMatched = false
+          break
+        }
+      }
+
+      if (!groupMatched || !candidateLines || candidateLines.size === 0) {
+        continue
+      }
+
+      for (const term of excludeTerms) {
+        const key = resolveKey(term)
+        if (key === null) {
+          continue
+        }
+
+        const termData = fieldIndex[key]
+        if (!termData) {
+          continue
+        }
+
+        const excludeLines = this.indexManager._getAllLineNumbers(termData)
+        if (!excludeLines || excludeLines.length === 0) {
+          continue
+        }
+
+        for (const line of excludeLines) {
+          if (!candidateLines.size) {
+            break
+          }
+          candidateLines.delete(line)
+        }
+
+        if (!candidateLines.size) {
+          break
+        }
+      }
+
+      if (candidateLines && candidateLines.size > 0) {
+        matchedGroups++
+      }
+    }
+
+    if (matchedGroups === 0) {
+      return 0
+    }
+
+    const precision = typeof options.precision === 'number' && options.precision >= 0
+      ? options.precision
+      : 2
+
+    const coverageValue = (matchedGroups / groups.length) * 100
+    return Number(coverageValue.toFixed(precision))
   }
 
   /**
@@ -2589,7 +3351,8 @@ class Database extends EventEmitter {
     const opts = {
       limit: options.limit ?? 100,
       sort: options.sort ?? 'desc',
-      includeScore: options.includeScore !== false
+      includeScore: options.includeScore !== false,
+      mode: options.mode ?? 'sum'
     }
     
     // Validate fieldName
@@ -2613,6 +3376,12 @@ class Database extends EventEmitter {
         throw new Error(`Score value for term "${term}" must be a number`)
       }
     }
+
+    // Validate mode
+    const allowedModes = new Set(['sum', 'max', 'avg', 'first'])
+    if (!allowedModes.has(opts.mode)) {
+      throw new Error(`Invalid score mode "${opts.mode}". Must be one of: ${Array.from(allowedModes).join(', ')}`)
+    }
     
     // Check if field is indexed and is array:string type
     if (!this.opts.indexes || !this.opts.indexes[fieldName]) {
@@ -2637,6 +3406,7 @@ class Database extends EventEmitter {
     
     // Accumulate scores for each line number
     const scoreMap = new Map()
+    const countMap = opts.mode === 'avg' ? new Map() : null
     
     // Iterate through each term in the scores object
     for (const [term, weight] of Object.entries(scores)) {
@@ -2666,8 +3436,44 @@ class Database extends EventEmitter {
       
       // Add weight to score for each line number
       for (const lineNumber of lineNumbers) {
-        const currentScore = scoreMap.get(lineNumber) || 0
-        scoreMap.set(lineNumber, currentScore + weight)
+        const currentScore = scoreMap.get(lineNumber)
+        
+        switch (opts.mode) {
+          case 'sum': {
+            const nextScore = (currentScore || 0) + weight
+            scoreMap.set(lineNumber, nextScore)
+            break
+          }
+          case 'max': {
+            if (currentScore === undefined) {
+              scoreMap.set(lineNumber, weight)
+            } else {
+              scoreMap.set(lineNumber, Math.max(currentScore, weight))
+            }
+            break
+          }
+          case 'avg': {
+            const previous = currentScore || 0
+            scoreMap.set(lineNumber, previous + weight)
+            const count = (countMap.get(lineNumber) || 0) + 1
+            countMap.set(lineNumber, count)
+            break
+          }
+          case 'first': {
+            if (currentScore === undefined) {
+              scoreMap.set(lineNumber, weight)
+            }
+            break
+          }
+        }
+      }
+    }
+
+    // For average mode, divide total by count
+    if (opts.mode === 'avg') {
+      for (const [lineNumber, totalScore] of scoreMap.entries()) {
+        const count = countMap.get(lineNumber) || 1
+        scoreMap.set(lineNumber, totalScore / count)
       }
     }
     
@@ -2944,11 +3750,52 @@ class Database extends EventEmitter {
     }
     
     // CRITICAL FIX: Only remove processed items from writeBuffer after all async operations complete
-    // OPTIMIZATION: Use Set.has() for O(1) lookup - same Set used for processing
     const beforeLength = this.writeBuffer.length
-    this.writeBuffer = this.writeBuffer.filter(item => !itemsToProcess.has(item))
+    if (beforeLength > 0) {
+      const originalRecords = this.writeBuffer
+      const originalOffsets = this.writeBufferOffsets
+      const originalSizes = this.writeBufferSizes
+      const retainedRecords = []
+      const retainedOffsets = []
+      const retainedSizes = []
+      let retainedTotal = 0
+      let removedCount = 0
+      
+      for (let i = 0; i < originalRecords.length; i++) {
+        const record = originalRecords[i]
+        if (itemsToProcess.has(record)) {
+          removedCount++
+          continue
+        }
+        
+        retainedRecords.push(record)
+        if (originalOffsets && i < originalOffsets.length) {
+          retainedOffsets.push(originalOffsets[i])
+        }
+        if (originalSizes && i < originalSizes.length) {
+          const size = originalSizes[i]
+          if (size !== undefined) {
+            retainedSizes.push(size)
+            retainedTotal += size
+          }
+        }
+      }
+      
+      if (removedCount > 0) {
+        this.writeBuffer = retainedRecords
+        this.writeBufferOffsets = retainedOffsets
+        this.writeBufferSizes = retainedSizes
+        this.writeBufferTotalSize = retainedTotal
+      }
+    }
     const afterLength = this.writeBuffer.length
     
+    if (afterLength === 0) {
+      this.writeBufferOffsets = []
+      this.writeBufferSizes = []
+      this.writeBufferTotalSize = 0
+    }
+    
     if (this.opts.debugMode && beforeLength !== afterLength) {
       console.log(`💾 _processWriteBuffer: Removed ${beforeLength - afterLength} items from writeBuffer (${beforeLength} -> ${afterLength})`)
     }
@@ -3534,6 +4381,240 @@ class Database extends EventEmitter {
     }).filter(n => n !== undefined)
   }
 
+  /**
+   * Get the base line number for writeBuffer entries (number of persisted records)
+   * @private
+   */
+  _getWriteBufferBaseLineNumber() {
+    return Array.isArray(this.offsets) ? this.offsets.length : 0
+  }
+
+  /**
+   * Convert a writeBuffer index into an absolute line number
+   * @param {number} writeBufferIndex - Index inside writeBuffer (0-based)
+   * @returns {number} Absolute line number (0-based)
+   * @private
+   */
+  _getAbsoluteLineNumber(writeBufferIndex) {
+    if (typeof writeBufferIndex !== 'number' || writeBufferIndex < 0) {
+      throw new Error('Invalid writeBuffer index')
+    }
+    return this._getWriteBufferBaseLineNumber() + writeBufferIndex
+  }
+
+  async *_streamingRecoveryGenerator(criteria, options, alreadyYielded = 0, map = null, remainingSkipValue = 0) {
+    if (this._offsetRecoveryInProgress) {
+      return
+    }
+
+    if (!this.fileHandler || !this.fileHandler.file) {
+      return
+    }
+
+    this._offsetRecoveryInProgress = true
+
+    const fsModule = this._fsModule || (this._fsModule = await import('fs'))
+    let fd
+
+    try {
+      fd = await fsModule.promises.open(this.fileHandler.file, 'r')
+    } catch (error) {
+      this._offsetRecoveryInProgress = false
+      if (this.opts.debugMode) {
+        console.warn(`⚠️ Offset recovery skipped: ${error.message}`)
+      }
+      return
+    }
+
+    const chunkSize = this.opts.offsetRecoveryChunkSize || 64 * 1024
+    let buffer = Buffer.alloc(0)
+    let readOffset = 0
+    const originalOffsets = Array.isArray(this.offsets) ? [...this.offsets] : []
+    const newOffsets = []
+    let offsetAdjusted = false
+    let limitReached = false
+    let lineIndex = 0
+    let lastLineEnd = 0
+    let producedTotal = alreadyYielded || 0
+    let remainingSkip = remainingSkipValue || 0
+    let remainingAlreadyYielded = alreadyYielded || 0
+    const limit = typeof options?.limit === 'number' ? options.limit : null
+    const includeOffsets = options?.includeOffsets === true
+    const includeLinePosition = this.opts.includeLinePosition
+    const mapSet = map instanceof Set ? new Set(map) : (Array.isArray(map) ? new Set(map) : null)
+    const criteriaIsObject = criteria && typeof criteria === 'object' && !Array.isArray(criteria) && !(criteria instanceof Set)
+    const hasCriteria = criteriaIsObject && Object.keys(criteria).length > 0
+
+    const decodeLineBuffer = (lineBuffer) => {
+      let trimmed = lineBuffer
+      if (trimmed.length > 0 && trimmed[trimmed.length - 1] === 0x0A) {
+        trimmed = trimmed.subarray(0, trimmed.length - 1)
+      }
+      if (trimmed.length > 0 && trimmed[trimmed.length - 1] === 0x0D) {
+        trimmed = trimmed.subarray(0, trimmed.length - 1)
+      }
+      return trimmed
+    }
+
+    const processLine = async (lineBuffer, lineStart) => {
+      const lineLength = lineBuffer.length
+      newOffsets[lineIndex] = lineStart
+      const expected = originalOffsets[lineIndex]
+      if (expected !== undefined && expected !== lineStart) {
+        offsetAdjusted = true
+        if (this.opts.debugMode) {
+          console.warn(`⚠️ Offset mismatch detected at line ${lineIndex}: expected ${expected}, actual ${lineStart}`)
+        }
+      } else if (expected === undefined) {
+        offsetAdjusted = true
+      }
+
+      lastLineEnd = Math.max(lastLineEnd, lineStart + lineLength)
+
+      let entryWithTerms = null
+      let shouldYield = false
+
+      const decodedBuffer = decodeLineBuffer(lineBuffer)
+      if (decodedBuffer.length > 0) {
+        let lineString
+        try {
+          lineString = decodedBuffer.toString('utf8')
+        } catch (error) {
+          lineString = decodedBuffer.toString('utf8', { replacement: '?' })
+        }
+
+        try {
+          const record = await this.serializer.deserialize(lineString)
+          if (record && typeof record === 'object') {
+            entryWithTerms = this.restoreTermIdsAfterDeserialization(record)
+            if (includeLinePosition) {
+              entryWithTerms._ = lineIndex
+            }
+
+            if (mapSet) {
+              shouldYield = mapSet.has(lineIndex)
+              if (shouldYield) {
+                mapSet.delete(lineIndex)
+              }
+            } else if (hasCriteria) {
+              shouldYield = this.queryManager.matchesCriteria(entryWithTerms, criteria, options)
+            } else {
+              shouldYield = true
+            }
+          }
+        } catch (error) {
+          if (this.opts.debugMode) {
+            console.warn(`⚠️ Offset recovery failed to deserialize line ${lineIndex} at ${lineStart}: ${error.message}`)
+          }
+        }
+      }
+
+      let yieldedEntry = null
+
+      if (shouldYield && entryWithTerms) {
+        if (remainingSkip > 0) {
+          remainingSkip--
+        } else if (remainingAlreadyYielded > 0) {
+          remainingAlreadyYielded--
+        } else if (!limit || producedTotal < limit) {
+          producedTotal++
+          yieldedEntry = includeOffsets
+            ? { entry: entryWithTerms, start: lineStart, _: lineIndex }
+            : entryWithTerms
+        } else {
+          limitReached = true
+        }
+      }
+
+      lineIndex++
+
+      if (yieldedEntry) {
+        return yieldedEntry
+      }
+      return null
+    }
+
+    let recoveryFailed = false
+
+    try {
+      while (true) {
+        const tempBuffer = Buffer.allocUnsafe(chunkSize)
+        const { bytesRead } = await fd.read(tempBuffer, 0, chunkSize, readOffset)
+
+        if (bytesRead === 0) {
+          if (buffer.length > 0) {
+            const lineStart = readOffset - buffer.length
+            const yieldedEntry = await processLine(buffer, lineStart)
+            if (yieldedEntry) {
+              yield yieldedEntry
+            }
+          }
+          break
+        }
+
+        readOffset += bytesRead
+        let chunk = buffer.length > 0
+          ? Buffer.concat([buffer, tempBuffer.subarray(0, bytesRead)])
+          : tempBuffer.subarray(0, bytesRead)
+
+        let processedUpTo = 0
+        const chunkBaseOffset = readOffset - chunk.length
+
+        while (true) {
+          const newlineIndex = chunk.indexOf(0x0A, processedUpTo)
+          if (newlineIndex === -1) {
+            break
+          }
+
+          const lineBuffer = chunk.subarray(processedUpTo, newlineIndex + 1)
+          const lineStart = chunkBaseOffset + processedUpTo
+          const yieldedEntry = await processLine(lineBuffer, lineStart)
+          processedUpTo = newlineIndex + 1
+
+          if (yieldedEntry) {
+            yield yieldedEntry
+          }
+        }
+
+        buffer = chunk.subarray(processedUpTo)
+      }
+    } catch (error) {
+      recoveryFailed = true
+      if (this.opts.debugMode) {
+        console.warn(`⚠️ Offset recovery aborted: ${error.message}`)
+      }
+    } finally {
+      await fd.close().catch(() => {})
+      this._offsetRecoveryInProgress = false
+
+      if (recoveryFailed) {
+        return
+      }
+
+      this.offsets = newOffsets
+      if (lineIndex < this.offsets.length) {
+        this.offsets.length = lineIndex
+      }
+
+      if (originalOffsets.length !== newOffsets.length) {
+        offsetAdjusted = true
+      }
+
+      this.indexOffset = lastLineEnd
+
+      if (offsetAdjusted) {
+        this.shouldSave = true
+        try {
+          await this._saveIndexDataToFile()
+        } catch (error) {
+          if (this.opts.debugMode) {
+            console.warn(`⚠️ Failed to persist recovered offsets: ${error.message}`)
+          }
+        }
+      }
+    }
+  }
+
   /**
    * Walk through records using streaming (real implementation)
    */
@@ -3547,6 +4628,7 @@ class Database extends EventEmitter {
     if (this.indexOffset === 0 && this.writeBuffer.length === 0) return
     
     let count = 0
+    let remainingSkip = options.skip || 0
     
     let map
     if (!Array.isArray(criteria)) {
@@ -3557,8 +4639,9 @@ class Database extends EventEmitter {
         map = [...this.indexManager.query(criteria, options)]
       } else {
         // For empty criteria {} or null/undefined, get all records
-        // Use writeBuffer length when indexOffset is 0 (data not saved yet)
-        const totalRecords = this.indexOffset > 0 ? this.indexOffset : this.writeBuffer.length
+        const totalRecords = this.offsets && this.offsets.length > 0
+          ? this.offsets.length
+          : this.writeBuffer.length
         map = [...Array(totalRecords).keys()]
       }
     } else {
@@ -3577,6 +4660,10 @@ class Database extends EventEmitter {
           }
           const entry = this.writeBuffer[i]
           if (entry && this.queryManager.matchesCriteria(entry, criteria, options)) {
+            if (remainingSkip > 0) {
+              remainingSkip--
+              continue
+            }
             count++
             if (options.includeOffsets) {
               yield { entry, start: 0, _: i }
@@ -3597,6 +4684,10 @@ class Database extends EventEmitter {
           if (lineNumber < this.writeBuffer.length) {
             const entry = this.writeBuffer[lineNumber]
             if (entry) {
+              if (remainingSkip > 0) {
+                remainingSkip--
+                continue
+              }
               count++
               if (options.includeOffsets) {
                 yield { entry, start: 0, _: lineNumber }
@@ -3657,6 +4748,11 @@ class Database extends EventEmitter {
                   // SPACE OPTIMIZATION: Restore term IDs to terms for user
                   const recordWithTerms = this.restoreTermIdsAfterDeserialization(record)
                   
+                  if (remainingSkip > 0) {
+                    remainingSkip--
+                    continue
+                  }
+
                   count++
                   if (options.includeOffsets) {
                     yield { entry: recordWithTerms, start: row.start, _: row._ || 0 }
@@ -3667,7 +4763,21 @@ class Database extends EventEmitter {
                     yield recordWithTerms
                   }
                 } catch (error) {
-                  // Skip invalid lines
+                  // CRITICAL FIX: Log deserialization errors instead of silently ignoring them
+                  // This helps identify data corruption issues
+                  if (1||this.opts.debugMode) {
+                    console.warn(`⚠️ walk(): Failed to deserialize record at offset ${row.start}: ${error.message}`)
+                    console.warn(`⚠️ walk(): Problematic line (first 200 chars): ${row.line.substring(0, 200)}`)
+                  }
+                  if (!this._offsetRecoveryInProgress) {
+                    for await (const recoveredEntry of this._streamingRecoveryGenerator(criteria, options, count, map, remainingSkip)) {
+                      yield recoveredEntry
+                      count++
+                    }
+                    return
+                  }
+                  // Skip invalid lines but continue processing
+                  // This prevents one corrupted record from stopping the entire walk operation
                 }
               }
               if (options.limit && count >= options.limit) {
@@ -3696,6 +4806,12 @@ class Database extends EventEmitter {
             if (options.limit && count >= options.limit) {
               break
             }
+
+            if (remainingSkip > 0) {
+              remainingSkip--
+              continue
+            }
+
             count++
             
             // SPACE OPTIMIZATION: Restore term IDs to terms for user
@@ -3732,20 +4848,44 @@ class Database extends EventEmitter {
           if (options.limit && count >= options.limit) {
             break
           }
-          const entry = await this.serializer.deserialize(row.line, { compress: this.opts.compress, v8: this.opts.v8 })
-          if (entry === null) continue
           
-          // SPACE OPTIMIZATION: Restore term IDs to terms for user
-          const entryWithTerms = this.restoreTermIdsAfterDeserialization(entry)
+          try {
+            const entry = await this.serializer.deserialize(row.line, { compress: this.opts.compress, v8: this.opts.v8 })
+            if (entry === null) continue
+            
+            // SPACE OPTIMIZATION: Restore term IDs to terms for user
+            const entryWithTerms = this.restoreTermIdsAfterDeserialization(entry)
           
-          count++
-          if (options.includeOffsets) {
-            yield { entry: entryWithTerms, start: row.start, _: row._ || this.offsets.findIndex(n => n === row.start) }
-          } else {
-            if (this.opts.includeLinePosition) {
-              entryWithTerms._ = row._ || this.offsets.findIndex(n => n === row.start)
+            if (remainingSkip > 0) {
+              remainingSkip--
+              continue
+            }
+
+            count++
+            if (options.includeOffsets) {
+              yield { entry: entryWithTerms, start: row.start, _: row._ || this.offsets.findIndex(n => n === row.start) }
+            } else {
+              if (this.opts.includeLinePosition) {
+                entryWithTerms._ = row._ || this.offsets.findIndex(n => n === row.start)
+              }
+              yield entryWithTerms
             }
-            yield entryWithTerms
+          } catch (error) {
+            // CRITICAL FIX: Log deserialization errors instead of silently ignoring them
+            // This helps identify data corruption issues
+            if (1||this.opts.debugMode) {
+              console.warn(`⚠️ walk(): Failed to deserialize record at offset ${row.start}: ${error.message}`)
+              console.warn(`⚠️ walk(): Problematic line (first 200 chars): ${row.line.substring(0, 200)}`)
+            }
+            if (!this._offsetRecoveryInProgress) {
+              for await (const recoveredEntry of this._streamingRecoveryGenerator(criteria, options, count, map, remainingSkip)) {
+                yield recoveredEntry
+                count++
+              }
+              return
+            }
+            // Skip invalid lines but continue processing
+            // This prevents one corrupted record from stopping the entire walk operation
           }
         }
       }
@@ -3899,16 +5039,20 @@ class Database extends EventEmitter {
           
           // Update record in writeBuffer or add to writeBuffer
           const index = this.writeBuffer.findIndex(r => r.id === record.id)
+          let targetIndex
           if (index !== -1) {
             // Record is already in writeBuffer, update it
             this.writeBuffer[index] = record
+            targetIndex = index
           } else {
             // Record is in file, add updated version to writeBuffer
             this.writeBuffer.push(record)
+            targetIndex = this.writeBuffer.length - 1
           }
           
           // Update index
-          await this.indexManager.update(record, record, this.writeBuffer.length - 1)
+          const absoluteLineNumber = this._getAbsoluteLineNumber(targetIndex)
+          await this.indexManager.update(record, record, absoluteLineNumber)
         }
         
         if (this.opts.debugMode) {
@@ -3982,8 +5126,24 @@ class Database extends EventEmitter {
           this.writeBufferSizes = []
         }
       } else {
-        // Even if no data to save, ensure index data is persisted
-        await this._saveIndexDataToFile()
+        // Only save index data if it actually has content
+        // Don't overwrite a valid index with an empty one
+        if (this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
+          let hasIndexData = false
+          for (const field of this.indexManager.indexedFields) {
+            if (this.indexManager.hasUsableIndexData(field)) {
+              hasIndexData = true
+              break
+            }
+          }
+          // Only save if we have actual index data OR if offsets are populated
+          // (offsets being populated means we've processed data)
+          if (hasIndexData || (this.offsets && this.offsets.length > 0)) {
+            await this._saveIndexDataToFile()
+          } else if (this.opts.debugMode) {
+            console.log('⚠️ close(): Skipping index save - index is empty and no offsets')
+          }
+        }
       }
       
       // 2. Mark as closed (but not destroyed) to allow reopening
@@ -4019,8 +5179,43 @@ class Database extends EventEmitter {
     if (this.indexManager) {
       try {
         const idxPath = this.normalizedFile.replace('.jdb', '.idx.jdb')
+        const indexJSON = this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0 ? this.indexManager.toJSON() : {}
+        
+        // Check if index is empty
+        const isEmpty = !indexJSON || Object.keys(indexJSON).length === 0 || 
+          (this.indexManager.indexedFields && this.indexManager.indexedFields.every(field => {
+            const fieldIndex = indexJSON[field]
+            return !fieldIndex || (typeof fieldIndex === 'object' && Object.keys(fieldIndex).length === 0)
+          }))
+        
+        // PROTECTION: Don't overwrite a valid index file with empty data
+        // If the .idx.jdb file exists and has data, and we're trying to save empty index,
+        // skip the save to prevent corruption
+        if (isEmpty && !this.offsets?.length) {
+          const fs = await import('fs')
+          if (fs.existsSync(idxPath)) {
+            try {
+              const existingData = JSON.parse(await fs.promises.readFile(idxPath, 'utf8'))
+              const existingHasData = existingData.index && Object.keys(existingData.index).length > 0
+              const existingHasOffsets = existingData.offsets && existingData.offsets.length > 0
+              
+              if (existingHasData || existingHasOffsets) {
+                if (this.opts.debugMode) {
+                  console.log(`⚠️ _saveIndexDataToFile: Skipping save - would overwrite valid index with empty data`)
+                }
+                return // Don't overwrite valid index with empty one
+              }
+            } catch (error) {
+              // If we can't read existing file, proceed with save (might be corrupted)
+              if (this.opts.debugMode) {
+                console.log(`⚠️ _saveIndexDataToFile: Could not read existing index file, proceeding with save`)
+              }
+            }
+          }
+        }
+        
         const indexData = {
-          index: this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0 ? this.indexManager.toJSON() : {},
+          index: indexJSON,
           offsets: this.offsets, // Save actual offsets for efficient file operations
           indexOffset: this.indexOffset, // Save file size for proper range calculations
           // Save configuration for reuse when database exists