jexidb 2.1.3 → 2.1.5

package/src/Database.mjs

@@ -403,7 +403,19 @@ class Database extends EventEmitter {
       saveTime: 0,
       loadTime: 0
     }
-    
+
+    // Initialize integrity correction tracking
+    this.integrityCorrections = {
+      indexSync: 0,        // index.totalLines vs offsets.length corrections
+      indexInconsistency: 0, // Index record count vs offsets mismatch
+      writeBufferForced: 0, // WriteBuffer not cleared after save
+      indexSaveFailures: 0, // Failed to save index data
+      dataIntegrity: 0,    // General data integrity issues
+      utf8Recovery: 0,     // UTF-8 decoding failures recovered
+      jsonRecovery: 0      // JSON parsing failures recovered
+    }
+
+
     // Initialize usage stats for QueryManager
     this.usageStats = {
       totalQueries: 0,
@@ -596,7 +608,9 @@ class Database extends EventEmitter {
         }
       }
       if (arrayStringFields.length > 0) {
-        console.warn(`⚠️  Warning: The following array:string indexed fields were not added to term mapping: ${arrayStringFields.join(', ')}. This may impact performance.`)
+        if (this.opts.debugMode) {
+          console.warn(`⚠️  Warning: The following array:string indexed fields were not added to term mapping: ${arrayStringFields.join(', ')}. This may impact performance.`)
+        }
       }
     }
     
@@ -638,22 +652,27 @@ class Database extends EventEmitter {
   }
 
   /**
-   * Get term mapping fields from indexes (auto-detected)
+   * Get term mapping fields from configuration or indexes (auto-detected)
    * @returns {string[]} Array of field names that use term mapping
    */
   getTermMappingFields() {
+    // If termMappingFields is explicitly configured, use it
+    if (this.opts.termMappingFields && Array.isArray(this.opts.termMappingFields)) {
+      return [...this.opts.termMappingFields]
+    }
+
+    // Auto-detect fields that benefit from term mapping from indexes
     if (!this.opts.indexes) return []
-    
-    // Auto-detect fields that benefit from term mapping
+
     const termMappingFields = []
-    
+
     for (const [field, type] of Object.entries(this.opts.indexes)) {
       // Fields that should use term mapping (only array fields)
       if (type === 'array:string') {
         termMappingFields.push(field)
       }
     }
-    
+
     return termMappingFields
   }
 
@@ -745,7 +764,7 @@ class Database extends EventEmitter {
       
       // Reset closed state when reinitializing
       this.closed = false
-      
+
       // Initialize managers (protected against double initialization)
       this.initializeManagers()
       
@@ -768,12 +787,33 @@ class Database extends EventEmitter {
           await this.load()
         }
       }
-      
+
+      // CRITICAL INTEGRITY CHECK: Ensure IndexManager is consistent with loaded offsets
+      // This must happen immediately after load() to prevent any subsequent operations from seeing inconsistent state
+      if (this.indexManager && this.offsets && this.offsets.length > 0) {
+        const currentTotalLines = this.indexManager.totalLines || 0
+        if (currentTotalLines !== this.offsets.length) {
+          this.indexManager.setTotalLines(this.offsets.length)
+          if (this.opts.debugMode) {
+            console.log(`🔧 Post-load integrity sync: IndexManager totalLines ${currentTotalLines} → ${this.offsets.length}`)
+          }
+        }
+      }
+
       // Manual save is now the default behavior
-      
+
+      // CRITICAL FIX: Ensure IndexManager totalLines is consistent with offsets
+      // This prevents data integrity issues when database is initialized without existing data
+      if (this.indexManager && this.offsets) {
+        this.indexManager.setTotalLines(this.offsets.length)
+        if (this.opts.debugMode) {
+          console.log(`🔧 Initialized index totalLines to ${this.offsets.length}`)
+        }
+      }
+
       this.initialized = true
       this.emit('initialized')
-      
+
       if (this.opts.debugMode) {
         console.log(`✅ Database initialized with ${this.writeBuffer.length} records`)
       }
@@ -927,11 +967,11 @@ class Database extends EventEmitter {
               this.offsets = parsedIdxData.offsets
               // CRITICAL FIX: Update IndexManager totalLines to match offsets length
               // This ensures queries and length property work correctly even if offsets are reset later
-              if (this.indexManager && this.offsets.length > 0) {
+              if (this.indexManager) {
                 this.indexManager.setTotalLines(this.offsets.length)
-              }
-              if (this.opts.debugMode) {
-                console.log(`📂 Loaded ${this.offsets.length} offsets from ${idxPath}`)
+                if (this.opts.debugMode) {
+                  console.log(`📂 Loaded ${this.offsets.length} offsets from ${idxPath}, synced IndexManager totalLines`)
+                }
               }
             }
             
@@ -1606,6 +1646,7 @@ class Database extends EventEmitter {
       
       if (this.opts.debugMode) {
         console.log(`💾 Save: allData.length=${allData.length}, cleanedData.length=${cleanedData.length}`)
+        console.log(`💾 Save: Current offsets.length before recalculation: ${this.offsets.length}`)
         console.log(`💾 Save: All records in allData before serialization:`, allData.map(r => r && r.id ? { id: String(r.id), price: r.price, app_id: r.app_id, currency: r.currency } : 'no-id'))
         console.log(`💾 Save: Sample cleaned record:`, cleanedData[0] ? Object.keys(cleanedData[0]) : 'null')
       }
@@ -1624,6 +1665,8 @@ class Database extends EventEmitter {
         }
       }
       
+      // CRITICAL FIX: Always recalculate offsets from serialized data to ensure consistency
+      // Even if _streamExistingRecords updated offsets, we need to recalculate based on actual serialized data
       this.offsets = []
       let currentOffset = 0
       for (let i = 0; i < lines.length; i++) {
@@ -1634,6 +1677,10 @@ class Database extends EventEmitter {
         currentOffset += Buffer.byteLength(lineWithNewline, 'utf8')
       }
       
+      if (this.opts.debugMode) {
+        console.log(`💾 Save: Recalculated offsets.length=${this.offsets.length}, should match lines.length=${lines.length}`)
+      }
+      
       // CRITICAL FIX: Ensure indexOffset matches actual file size
       this.indexOffset = currentOffset
       
@@ -1655,11 +1702,15 @@ class Database extends EventEmitter {
       this.shouldSave = false
       this.lastSaveTime = Date.now()
       
-      // Clear writeBuffer and deletedIds after successful save only if we had data to save
-      if (allData.length > 0) {
-        // Rebuild index when records were deleted or updated to maintain consistency
+      // CRITICAL FIX: Always clear deletedIds and rebuild index if there were deletions,
+      // even if allData.length === 0 (all records were deleted)
         const hadDeletedRecords = deletedIdsSnapshot.size > 0
         const hadUpdatedRecords = writeBufferSnapshot.length > 0
+      
+      // Clear writeBuffer and deletedIds after successful save
+      // Also rebuild index if records were deleted or updated, even if allData is empty
+      if (allData.length > 0 || hadDeletedRecords || hadUpdatedRecords) {
+        // Rebuild index when records were deleted or updated to maintain consistency
         if (this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
           if (hadDeletedRecords || hadUpdatedRecords) {
             // Clear the index and rebuild it from the saved records
@@ -1713,8 +1764,26 @@ class Database extends EventEmitter {
               
               await this.indexManager.add(record, i)
             }
+
+            // VALIDATION: Ensure index consistency after rebuild
+            // Check that all indexed records have valid line numbers
+            const indexedRecordCount = this.indexManager.getIndexedRecordCount?.() || allData.length
+            if (indexedRecordCount !== this.offsets.length) {
+              this.integrityCorrections.indexInconsistency++
+              console.log(`🔧 Auto-corrected index consistency: ${indexedRecordCount} indexed → ${this.offsets.length} offsets`)
+
+              if (this.integrityCorrections.indexInconsistency > 5) {
+                console.warn(`⚠️ Frequent index inconsistencies detected (${this.integrityCorrections.indexInconsistency} times)`)
+              }
+
+              // Force consistency by setting totalLines to match offsets
+              this.indexManager.setTotalLines(this.offsets.length)
+            } else {
+              this.indexManager.setTotalLines(this.offsets.length)
+            }
+
             if (this.opts.debugMode) {
-              console.log(`💾 Save: Index rebuilt with ${allData.length} records`)
+              console.log(`💾 Save: Index rebuilt with ${allData.length} records, totalLines set to ${this.offsets.length}`)
             }
           }
         }
@@ -1738,6 +1807,22 @@ class Database extends EventEmitter {
         for (const deletedId of deletedIdsSnapshot) {
           this.deletedIds.delete(deletedId)
         }
+      } else if (hadDeletedRecords) {
+        // CRITICAL FIX: Even if allData is empty, clear deletedIds and rebuild index
+        // when records were deleted to ensure consistency
+        if (this.indexManager && this.indexManager.indexedFields && this.indexManager.indexedFields.length > 0) {
+          // Clear the index since all records were deleted
+          this.indexManager.clear()
+          this.indexManager.setTotalLines(0)
+          if (this.opts.debugMode) {
+            console.log(`🧹 Cleared index after removing all ${deletedIdsSnapshot.size} deleted records`)
+          }
+        }
+        
+        // Clear deletedIds even when allData is empty
+        for (const deletedId of deletedIdsSnapshot) {
+          this.deletedIds.delete(deletedId)
+        }
         
         // CRITICAL FIX: Ensure writeBuffer is completely cleared after successful save
         if (this.writeBuffer.length > 0) {
@@ -2244,7 +2329,7 @@ class Database extends EventEmitter {
    */
   async find(criteria = {}, options = {}) {
     this._validateInitialization('find')
-    
+
     // CRITICAL FIX: Validate state before find operation
     this.validateState()
     
@@ -2257,6 +2342,38 @@ class Database extends EventEmitter {
     }
     
     try {
+      // INTEGRITY CHECK: Validate data consistency before querying
+      // This is a safety net for unexpected inconsistencies - should rarely trigger
+      if (this.indexManager && this.offsets && this.offsets.length > 0) {
+        const indexTotalLines = this.indexManager.totalLines || 0
+        const offsetsLength = this.offsets.length
+
+        if (indexTotalLines !== offsetsLength) {
+          // This should be extremely rare - indicates a real bug if it happens frequently
+          this.integrityCorrections.dataIntegrity++
+
+          // Only show in debug mode - these corrections indicate real issues
+          if (this.opts.debugMode) {
+            console.log(`🔧 Integrity correction needed: index.totalLines ${indexTotalLines} → ${offsetsLength} (${this.integrityCorrections.dataIntegrity} total)`)
+          }
+
+          // Warn if corrections are becoming frequent (indicates a real problem)
+          if (this.integrityCorrections.dataIntegrity > 5) {
+            console.warn(`⚠️ Frequent integrity corrections (${this.integrityCorrections.dataIntegrity} times) - this indicates a systemic issue`)
+          }
+
+          this.indexManager.setTotalLines(offsetsLength)
+
+          // Try to persist the fix, but don't fail the operation if it doesn't work
+          try {
+            await this._saveIndexDataToFile()
+          } catch (error) {
+            // Just track the failure - don't throw since this is a safety net
+            this.integrityCorrections.indexSaveFailures++
+          }
+        }
+      }
+
       // Validate indexed query mode if enabled
       if (this.opts.indexedQueryMode === 'strict') {
         this._validateIndexedQuery(criteria, options)
@@ -2279,36 +2396,25 @@ class Database extends EventEmitter {
       
       
       // Combine results, removing duplicates (writeBuffer takes precedence)
-      // OPTIMIZATION: Use parallel processing for better performance when writeBuffer has many records
+      // OPTIMIZATION: Unified efficient approach with consistent precedence rules
       let allResults
-      if (writeBufferResults.length > 50) {
-        // Parallel approach for large writeBuffer
-        const [fileResultsSet, writeBufferSet] = await Promise.all([
-          Promise.resolve(new Set(fileResultsWithTerms.map(r => r.id))),
-          Promise.resolve(new Set(writeBufferResultsWithTerms.map(r => r.id)))
-        ])
+
+      // Create efficient lookup map for writeBuffer records
+      const writeBufferMap = new Map()
+      writeBufferResultsWithTerms.forEach(record => {
+        if (record && record.id) {
+          writeBufferMap.set(record.id, record)
+        }
+      })
         
-        // Merge efficiently: keep file results not in writeBuffer, then add all writeBuffer results
-        const filteredFileResults = await Promise.resolve(
-          fileResultsWithTerms.filter(r => !writeBufferSet.has(r.id))
-        )
+      // Filter file results to exclude any records that exist in writeBuffer
+      // This ensures writeBuffer always takes precedence
+      const filteredFileResults = fileResultsWithTerms.filter(record =>
+        record && record.id && !writeBufferMap.has(record.id)
+      )
+
+      // Combine results: file results (filtered) + all writeBuffer results
         allResults = [...filteredFileResults, ...writeBufferResultsWithTerms]
-      } else {
-        // Sequential approach for small writeBuffer (original logic)
-        allResults = [...fileResultsWithTerms]
-        
-        // Replace file records with writeBuffer records and add new writeBuffer records
-        for (const record of writeBufferResultsWithTerms) {
-          const existingIndex = allResults.findIndex(r => r.id === record.id)
-          if (existingIndex !== -1) {
-            // Replace existing record with writeBuffer version
-            allResults[existingIndex] = record
-          } else {
-            // Add new record from writeBuffer
-            allResults.push(record)
-          }
-        }
-      }
       
       // Remove records that are marked as deleted
       const finalResults = allResults.filter(record => !this.deletedIds.has(record.id))
@@ -2566,19 +2672,6 @@ class Database extends EventEmitter {
         
         // CRITICAL FIX: Validate state before update operation
         this.validateState()
-        
-        // CRITICAL FIX: If there's data to save, call save() to persist it
-        // Only save if there are actual records in writeBuffer
-        if (this.shouldSave && this.writeBuffer.length > 0) {
-          if (this.opts.debugMode) {
-            console.log(`🔄 UPDATE: Calling save() before update - writeBuffer.length=${this.writeBuffer.length}`)
-          }
-          const saveStart = Date.now()
-          await this.save(false) // Use save(false) since we're already in queue
-          if (this.opts.debugMode) {
-            console.log(`🔄 UPDATE: Save completed in ${Date.now() - saveStart}ms`)
-          }
-        }
       
       if (this.opts.debugMode) {
         console.log(`🔄 UPDATE: Starting find() - writeBuffer=${this.writeBuffer.length}`)
@@ -2591,7 +2684,12 @@ class Database extends EventEmitter {
       }
       
       const updatedRecords = []
-      
+
+      if (this.opts.debugMode) {
+        console.log(`🔄 UPDATE: About to process ${records.length} records`)
+        console.log(`🔄 UPDATE: Records:`, records.map(r => ({ id: r.id, value: r.value })))
+      }
+
       for (const record of records) {
         const recordStart = Date.now()
         if (this.opts.debugMode) {
@@ -2628,13 +2726,20 @@ class Database extends EventEmitter {
         // For records in the file, we need to ensure they are properly marked for replacement
         const index = this.writeBuffer.findIndex(r => r.id === record.id)
         let lineNumber = null
-        
+
+        if (this.opts.debugMode) {
+          console.log(`🔄 UPDATE: writeBuffer.findIndex for ${record.id} returned ${index}`)
+          console.log(`🔄 UPDATE: writeBuffer length: ${this.writeBuffer.length}`)
+          console.log(`🔄 UPDATE: writeBuffer IDs:`, this.writeBuffer.map(r => r.id))
+        }
+
         if (index !== -1) {
           // Record is already in writeBuffer, update it
           this.writeBuffer[index] = updated
           lineNumber = this._getAbsoluteLineNumber(index)
           if (this.opts.debugMode) {
             console.log(`🔄 UPDATE: Updated existing writeBuffer record at index ${index}`)
+            console.log(`🔄 UPDATE: writeBuffer now has ${this.writeBuffer.length} records`)
           }
         } else {
           // Record is in file, add updated version to writeBuffer
@@ -2644,6 +2749,7 @@ class Database extends EventEmitter {
           lineNumber = this._getAbsoluteLineNumber(this.writeBuffer.length - 1)
           if (this.opts.debugMode) {
             console.log(`🔄 UPDATE: Added updated record to writeBuffer (will replace file record ${record.id})`)
+            console.log(`🔄 UPDATE: writeBuffer now has ${this.writeBuffer.length} records`)
           }
         }
         
@@ -2679,13 +2785,32 @@ class Database extends EventEmitter {
    */
   async delete(criteria) {
     this._validateInitialization('delete')
-    
+
     return this.operationQueue.enqueue(async () => {
       this.isInsideOperationQueue = true
       try {
         // CRITICAL FIX: Validate state before delete operation
         this.validateState()
-      
+
+        // 🔧 NEW: Validate indexed query mode for delete operations
+        if (this.opts.indexedQueryMode === 'strict') {
+          this._validateIndexedQuery(criteria, { operation: 'delete' })
+        }
+
+        // ⚠️ NEW: Warn about non-indexed fields in permissive mode
+        if (this.opts.indexedQueryMode !== 'strict') {
+          const indexedFields = Object.keys(this.opts.indexes || {})
+          const queryFields = this._extractQueryFields(criteria)
+          const nonIndexedFields = queryFields.filter(field => !indexedFields.includes(field))
+
+          if (nonIndexedFields.length > 0) {
+            if (this.opts.debugMode) {
+              console.warn(`⚠️ Delete operation using non-indexed fields: ${nonIndexedFields.join(', ')}`)
+              console.warn(`   This may be slow or fail silently. Consider indexing these fields.`)
+            }
+          }
+        }
+
       const records = await this.find(criteria)
       const deletedIds = []
       
@@ -4289,14 +4414,30 @@ class Database extends EventEmitter {
         try {
           const arrayData = JSON.parse(trimmedLine)
           if (Array.isArray(arrayData) && arrayData.length > 0) {
-            // For arrays without explicit ID, use the first element as a fallback
-            // or try to find the ID field if it exists
+            // CRITICAL FIX: Use schema to find ID position, not hardcoded position
+            // The schema defines the order of fields in the array
+            if (this.serializer && this.serializer.schemaManager && this.serializer.schemaManager.isInitialized) {
+              const schema = this.serializer.schemaManager.getSchema()
+              const idIndex = schema.indexOf('id')
+              if (idIndex !== -1 && arrayData.length > idIndex) {
+                // ID is at the position defined by schema
+                recordId = arrayData[idIndex]
+              } else if (arrayData.length > schema.length) {
+                // ID might be appended after schema fields (for backward compatibility)
+                recordId = arrayData[schema.length]
+              } else {
+                // Fallback: use first element
+                recordId = arrayData[0]
+              }
+            } else {
+              // No schema available, try common positions
             if (arrayData.length > 2) {
-              // ID is typically at position 2 in array format [age, city, id, name]
+                // Try position 2 (common in older formats)
               recordId = arrayData[2]
             } else {
-              // For arrays without ID field, use first element as fallback
+                // Fallback: use first element
               recordId = arrayData[0]
+              }
             }
             if (recordId !== undefined && recordId !== null) {
               recordId = String(recordId)
@@ -4369,7 +4510,7 @@ class Database extends EventEmitter {
         } else if (!deletedIdsSnapshot.has(String(recordWithIds.id))) {
           // Keep existing record if not deleted
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Kept record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'})`)
+            console.log(`💾 Save: Kept record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - not in deletedIdsSnapshot`)
           }
           return { 
             type: 'kept', 
@@ -4380,7 +4521,9 @@ class Database extends EventEmitter {
         } else {
           // Skip deleted record
           if (this.opts.debugMode) {
-            console.log(`💾 Save: Skipped record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - deleted`)
+            console.log(`💾 Save: Skipped record ${recordWithIds.id} (${recordWithIds.name || 'Unnamed'}) - deleted (found in deletedIdsSnapshot)`)
+            console.log(`💾 Save: deletedIdsSnapshot contains:`, Array.from(deletedIdsSnapshot))
+            console.log(`💾 Save: Record ID check: String(${recordWithIds.id}) = "${String(recordWithIds.id)}", has() = ${deletedIdsSnapshot.has(String(recordWithIds.id))}`)
           }
           return { 
             type: 'deleted', 
@@ -4426,6 +4569,54 @@ class Database extends EventEmitter {
       
       switch (result.type) {
         case 'unchanged':
+          // CRITICAL FIX: Verify that unchanged records are not deleted
+          // Extract ID from the line to check against deletedIdsSnapshot
+          let unchangedRecordId = null
+          try {
+            if (result.line.startsWith('[') && result.line.endsWith(']')) {
+              const arrayData = JSON.parse(result.line)
+              if (Array.isArray(arrayData) && arrayData.length > 0) {
+                // CRITICAL FIX: Use schema to find ID position, not hardcoded position
+                if (this.serializer && this.serializer.schemaManager && this.serializer.schemaManager.isInitialized) {
+                  const schema = this.serializer.schemaManager.getSchema()
+                  const idIndex = schema.indexOf('id')
+                  if (idIndex !== -1 && arrayData.length > idIndex) {
+                    unchangedRecordId = String(arrayData[idIndex])
+                  } else if (arrayData.length > schema.length) {
+                    unchangedRecordId = String(arrayData[schema.length])
+                  } else {
+                    unchangedRecordId = String(arrayData[0])
+                  }
+                } else {
+                  // No schema, try common positions
+                  if (arrayData.length > 2) {
+                    unchangedRecordId = String(arrayData[2])
+                  } else {
+                    unchangedRecordId = String(arrayData[0])
+                  }
+                }
+              }
+            } else {
+              const obj = JSON.parse(result.line)
+              unchangedRecordId = obj.id ? String(obj.id) : null
+            }
+          } catch (e) {
+            // If we can't parse, skip this record to be safe
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Could not parse unchanged record to check deletion: ${e.message}`)
+            }
+            continue
+          }
+          
+          // Skip if this record is deleted
+          if (unchangedRecordId && deletedIdsSnapshot.has(unchangedRecordId)) {
+            if (this.opts.debugMode) {
+              console.log(`💾 Save: Skipping unchanged record ${unchangedRecordId} - deleted`)
+            }
+            deletedOffsets.add(offset)
+            break
+          }
+          
           // Collect unchanged lines for batch processing
           unchangedLines.push(result.line)
           keptRecords.push({ offset, type: 'unchanged', line: result.line })
@@ -4620,6 +4811,78 @@ class Database extends EventEmitter {
     return this._getWriteBufferBaseLineNumber() + writeBufferIndex
   }
 
+
+  /**
+   * Attempts to recover a corrupted line by cleaning invalid characters and fixing common JSON issues
+   * @param {string} line - The corrupted line to recover
+   * @returns {string|null} - The recovered line or null if recovery is not possible
+   */
+  _tryRecoverLine(line) {
+    if (!line || typeof line !== 'string') {
+      return null
+    }
+
+    try {
+      // Try parsing as-is first
+      JSON.parse(line)
+      return line // Line is already valid
+    } catch (e) {
+      // Line is corrupted, attempt recovery
+    }
+
+    let recovered = line.trim()
+
+    // Remove invalid control characters (except \n, \r, \t)
+    recovered = recovered.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '')
+
+    // Try to close unclosed strings
+    // Count quotes and ensure they're balanced
+    const quoteCount = (recovered.match(/"/g) || []).length
+    if (quoteCount % 2 !== 0) {
+      // Odd number of quotes - try to close the string
+      const lastQuoteIndex = recovered.lastIndexOf('"')
+      if (lastQuoteIndex > 0) {
+        // Check if we're inside a string (not escaped)
+        const beforeLastQuote = recovered.substring(0, lastQuoteIndex)
+        const escapedQuotes = (beforeLastQuote.match(/\\"/g) || []).length
+        const unescapedQuotes = (beforeLastQuote.match(/"/g) || []).length - escapedQuotes
+        
+        if (unescapedQuotes % 2 !== 0) {
+          // We're inside an unclosed string - try to close it
+          recovered = recovered + '"'
+        }
+      }
+    }
+
+    // Try to close unclosed arrays/objects
+    const openBraces = (recovered.match(/\{/g) || []).length
+    const closeBraces = (recovered.match(/\}/g) || []).length
+    const openBrackets = (recovered.match(/\[/g) || []).length
+    const closeBrackets = (recovered.match(/\]/g) || []).length
+
+    // Remove trailing commas before closing braces/brackets
+    recovered = recovered.replace(/,\s*([}\]])/g, '$1')
+
+    // Try to close arrays
+    if (openBrackets > closeBrackets) {
+      recovered = recovered + ']'.repeat(openBrackets - closeBrackets)
+    }
+
+    // Try to close objects
+    if (openBraces > closeBraces) {
+      recovered = recovered + '}'.repeat(openBraces - closeBraces)
+    }
+
+    // Final validation - try to parse
+    try {
+      JSON.parse(recovered)
+      return recovered
+    } catch (e) {
+      // Recovery failed
+      return null
+    }
+  }
+
   async *_streamingRecoveryGenerator(criteria, options, alreadyYielded = 0, map = null, remainingSkipValue = 0) {
     if (this._offsetRecoveryInProgress) {
       return
@@ -4845,6 +5108,17 @@ class Database extends EventEmitter {
     // If no data at all, return empty
     if (this.indexOffset === 0 && this.writeBuffer.length === 0) return
     
+    // CRITICAL FIX: Wait for any ongoing save operations to complete
+    // This prevents reading partially written data
+    if (this.isSaving) {
+      if (this.opts.debugMode) {
+        console.log('🔍 walk(): waiting for save operation to complete')
+      }
+      while (this.isSaving) {
+        await new Promise(resolve => setTimeout(resolve, 10))
+      }
+    }
+    
     let count = 0
     let remainingSkip = options.skip || 0
     
@@ -4983,10 +5257,50 @@ class Database extends EventEmitter {
                 } catch (error) {
                   // CRITICAL FIX: Log deserialization errors instead of silently ignoring them
                   // This helps identify data corruption issues
-                  if (1||this.opts.debugMode) {
+                  if (this.opts.debugMode) {
                     console.warn(`⚠️ walk(): Failed to deserialize record at offset ${row.start}: ${error.message}`)
                     console.warn(`⚠️ walk(): Problematic line (first 200 chars): ${row.line.substring(0, 200)}`)
                   }
+                  
+                  // CRITICAL FIX: Attempt to recover corrupted line before giving up
+                  const recoveredLine = this._tryRecoverLine(row.line)
+                  if (recoveredLine) {
+                    try {
+                      const record = this.serializer.deserialize(recoveredLine)
+                      if (record !== null) {
+                        this.integrityCorrections.jsonRecovery++
+                        console.log(`🔧 Recovered corrupted JSON line (${this.integrityCorrections.jsonRecovery} recoveries)`)
+
+                        if (this.integrityCorrections.jsonRecovery > 20) {
+                          console.warn(`⚠️ Frequent JSON recovery detected (${this.integrityCorrections.jsonRecovery} times) - may indicate data corruption`)
+                        }
+
+                        const recordWithTerms = this.restoreTermIdsAfterDeserialization(record)
+
+                        if (remainingSkip > 0) {
+                          remainingSkip--
+                          continue
+                        }
+
+                        count++
+                        if (options.includeOffsets) {
+                          yield { entry: recordWithTerms, start: row.start, _: row._ || 0 }
+                        } else {
+                          if (this.opts.includeLinePosition) {
+                            recordWithTerms._ = row._ || 0
+                          }
+                          yield recordWithTerms
+                        }
+                        continue // Successfully recovered and yielded
+                      }
+                    } catch (recoveryError) {
+                      // Recovery attempt failed, continue with normal error handling
+                      if (this.opts.debugMode) {
+                        console.warn(`⚠️ walk(): Line recovery failed: ${recoveryError.message}`)
+                      }
+                    }
+                  }
+                  
                   if (!this._offsetRecoveryInProgress) {
                     for await (const recoveredEntry of this._streamingRecoveryGenerator(criteria, options, count, map, remainingSkip)) {
                       yield recoveredEntry
@@ -5091,10 +5405,50 @@ class Database extends EventEmitter {
           } catch (error) {
             // CRITICAL FIX: Log deserialization errors instead of silently ignoring them
             // This helps identify data corruption issues
-            if (1||this.opts.debugMode) {
+            if (this.opts.debugMode) {
               console.warn(`⚠️ walk(): Failed to deserialize record at offset ${row.start}: ${error.message}`)
               console.warn(`⚠️ walk(): Problematic line (first 200 chars): ${row.line.substring(0, 200)}`)
             }
+            
+            // CRITICAL FIX: Attempt to recover corrupted line before giving up
+            const recoveredLine = this._tryRecoverLine(row.line)
+            if (recoveredLine) {
+              try {
+                const entry = await this.serializer.deserialize(recoveredLine, { compress: this.opts.compress, v8: this.opts.v8 })
+                if (entry !== null) {
+                  this.integrityCorrections.jsonRecovery++
+                  console.log(`🔧 Recovered corrupted JSON line (${this.integrityCorrections.jsonRecovery} recoveries)`)
+
+                  if (this.integrityCorrections.jsonRecovery > 20) {
+                    console.warn(`⚠️ Frequent JSON recovery detected (${this.integrityCorrections.jsonRecovery} times) - may indicate data corruption`)
+                  }
+
+                  const entryWithTerms = this.restoreTermIdsAfterDeserialization(entry)
+
+                  if (remainingSkip > 0) {
+                    remainingSkip--
+                    continue
+                  }
+
+                  count++
+                  if (options.includeOffsets) {
+                    yield { entry: entryWithTerms, start: row.start, _: row._ || this.offsets.findIndex(n => n === row.start) }
+                  } else {
+                    if (this.opts.includeLinePosition) {
+                      entryWithTerms._ = row._ || this.offsets.findIndex(n => n === row.start)
+                    }
+                    yield entryWithTerms
+                  }
+                  continue // Successfully recovered and yielded
+                }
+              } catch (recoveryError) {
+                // Recovery attempt failed, continue with normal error handling
+                if (this.opts.debugMode) {
+                  console.warn(`⚠️ walk(): Line recovery failed: ${recoveryError.message}`)
+                }
+              }
+            }
+            
             if (!this._offsetRecoveryInProgress) {
               for await (const recoveredEntry of this._streamingRecoveryGenerator(criteria, options, count, map, remainingSkip)) {
                 yield recoveredEntry
@@ -5338,7 +5692,13 @@ class Database extends EventEmitter {
         await this.save()
         // Ensure writeBuffer is cleared after save
         if (this.writeBuffer.length > 0) {
-          console.warn('⚠️ WriteBuffer not cleared after save() - forcing clear')
+          this.integrityCorrections.writeBufferForced++
+          console.log(`🔧 Forced WriteBuffer clear after save (${this.writeBuffer.length} items remaining)`)
+
+          if (this.integrityCorrections.writeBufferForced > 3) {
+            console.warn(`⚠️ Frequent WriteBuffer clear issues detected (${this.integrityCorrections.writeBufferForced} times)`)
+          }
+
           this.writeBuffer = []
           this.writeBufferOffsets = []
           this.writeBufferSizes = []
@@ -5463,7 +5823,8 @@ class Database extends EventEmitter {
           console.log(`💾 Index data saved to ${idxPath}`)
         }
       } catch (error) {
-        console.warn('Failed to save index data:', error.message)
+        this.integrityCorrections.indexSaveFailures++
+        console.warn(`⚠️ Index save failure (${this.integrityCorrections.indexSaveFailures} times): ${error.message}`)
         throw error // Re-throw to let caller handle
       }
     }
@@ -5539,5 +5900,4 @@ class Database extends EventEmitter {
 }
 
 export { Database }
-export default Database